Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM updates from Paolo Bonzini: "s390: - ioctl hardening - selftests ARM: - ITS translation cache - support for 512 vCPUs - various cleanups and bugfixes PPC: - various minor fixes and preparation x86: - bugfixes all over the place (posted interrupts, SVM, emulation corner cases, blocked INIT) - some IPI optimizations" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (75 commits) KVM: X86: Use IPI shorthands in kvm guest when support KVM: x86: Fix INIT signal handling in various CPU states KVM: VMX: Introduce exit reason for receiving INIT signal on guest-mode KVM: VMX: Stop the preemption timer during vCPU reset KVM: LAPIC: Micro optimize IPI latency kvm: Nested KVM MMUs need PAE root too KVM: x86: set ctxt->have_exception in x86_decode_insn() KVM: x86: always stop emulation on page fault KVM: nVMX: trace nested VM-Enter failures detected by H/W KVM: nVMX: add tracepoint for failed nested VM-Enter x86: KVM: svm: Fix a check in nested_svm_vmrun() KVM: x86: Return to userspace with internal error on unexpected exit reason KVM: x86: Add kvm_emulate_{rd,wr}msr() to consolidate VXM/SVM code KVM: x86: Refactor up kvm_{g,s}et_msr() to simplify callers doc: kvm: Fix return description of KVM_SET_MSRS KVM: X86: Tune PLE Window tracepoint KVM: VMX: Change ple_window type to unsigned int KVM: X86: Remove tailing newline for tracepoints KVM: X86: Trace vcpu_id for vmexit KVM: x86: Manually calculate reserved bits when loading PDPTRS ...

Linus Torvalds committed 7 years ago

Commit fe38bd62 parents 404e634 fb3925d63 files · +1694 −797Browse files

Unified · Split

63 files

Documentation/virt/kvm/api.txt +24 −9
Documentation/virt/kvm/mmu.txt +2 −2
arch/arm/include/uapi/asm/kvm.h +3 −1
arch/arm64/include/asm/pgtable-prot.h +1 −1
arch/arm64/include/uapi/asm/kvm.h +3 −1
arch/arm64/kvm/hyp/tlb.c +13 −1
arch/powerpc/include/asm/kvm_host.h +18 −4
arch/powerpc/include/asm/kvm_ppc.h +1 −0
arch/powerpc/include/asm/xive.h +9 −0
arch/powerpc/kvm/book3s.c +5 −3
arch/powerpc/kvm/book3s_hv.c +18 −6
arch/powerpc/kvm/book3s_hv_rm_mmu.c +1 −1
arch/powerpc/kvm/book3s_hv_rmhandlers.S +25 −13
arch/powerpc/kvm/book3s_xive.c +51 −9
arch/powerpc/kvm/book3s_xive.h +2 −0
arch/powerpc/kvm/book3s_xive_native.c +18 −5
arch/powerpc/kvm/e500.c +3 −0
arch/powerpc/kvm/emulate.c +1 −0
arch/powerpc/kvm/emulate_loadstore.c +0 −6
arch/powerpc/kvm/powerpc.c +2 −1
arch/powerpc/sysdev/xive/common.c +64 −23
arch/powerpc/sysdev/xive/native.c +7 −0
arch/s390/include/uapi/asm/kvm.h +6 −0
arch/s390/kvm/kvm-s390.c +5 −1
arch/x86/include/asm/kvm_emulate.h +2 −1
arch/x86/include/asm/kvm_host.h +15 −4
arch/x86/include/asm/vmx.h +14 −0
arch/x86/include/uapi/asm/vmx.h +2 −0
arch/x86/kernel/kvm.c +0 −12
arch/x86/kvm/cpuid.c +19 −8
arch/x86/kvm/emulate.c +25 −2
arch/x86/kvm/lapic.c +11 −9
arch/x86/kvm/mmu.c +39 −22
arch/x86/kvm/mmu.h +1 −1
arch/x86/kvm/svm.c +93 −105
arch/x86/kvm/trace.h +54 −20
arch/x86/kvm/vmx/nested.c +162 −143
arch/x86/kvm/vmx/vmenter.S +2 −2
arch/x86/kvm/vmx/vmx.c +48 −46
arch/x86/kvm/vmx/vmx.h +1 −1
arch/x86/kvm/x86.c +128 −69
arch/x86/kvm/x86.h +1 −1
include/kvm/arm_vgic.h +3 −1
include/uapi/linux/kvm.h +3 −0
tools/testing/selftests/kvm/Makefile +6 −4
tools/testing/selftests/kvm/dirty_log_test.c +53 −8
tools/testing/selftests/kvm/include/kvm_util.h +1 −7
tools/testing/selftests/kvm/lib/aarch64/ucall.c +112 −0
tools/testing/selftests/kvm/lib/s390x/ucall.c +56 −0
tools/testing/selftests/kvm/lib/ucall.c +0 −157
tools/testing/selftests/kvm/lib/x86_64/ucall.c +56 −0
tools/testing/selftests/kvm/s390x/memop.c +166 −0
tools/testing/selftests/kvm/s390x/sync_regs_test.c +34 −2
virt/kvm/arm/arm.c +2 −0
virt/kvm/arm/vgic/vgic-init.c +6 −2
virt/kvm/arm/vgic/vgic-irqfd.c +30 −6
virt/kvm/arm/vgic/vgic-its.c +207 −0
virt/kvm/arm/vgic/vgic-mmio-v3.c +27 −58
virt/kvm/arm/vgic/vgic-v2.c +4 −3
virt/kvm/arm/vgic/vgic-v3.c +4 −3
virt/kvm/arm/vgic/vgic.c +17 −9
virt/kvm/arm/vgic/vgic.h +5 −0
virt/kvm/kvm_main.c +3 −4

Documentation/virt/kvm/api.txt+24−9

@@ -586,7 +586,7 @@ Capability: basic · 585 unmodified lines
586	586	Architectures: x86
587	587	Type: vcpu ioctl
588	588	Parameters: struct kvm_msrs (in)
589		−Returns: 0 on success, -1 on error
	589	+Returns: number of msrs successfully set (see below), -1 on error
590	590
591	591	Writes model-specific registers to the vcpu. See KVM_GET_MSRS for the
592	592	data structures.
@@ -595,6 +595,11 @@ Application code should set the 'nmsrs' member (which indicates the · 2 unmodified lines
595	595	size of the entries array), and the 'index' and 'data' members of each
596	596	array entry.
597	597
	598	+It tries to set the MSRs in array entries[] one by one. If setting an MSR
	599	+fails, e.g., due to setting reserved bits, the MSR isn't supported/emulated
	600	+by KVM, etc..., it stops processing the MSR list and returns the number of
	601	+MSRs that have been set successfully.
	602	+
598	603
599	604	4.20 KVM_SET_CPUID
600	605
@@ -753,8 +758,8 @@ in-kernel irqchip (GIC), and for in-kernel irqchip can tell the GIC to · 152 unmodified lines
753	758	use PPIs designated for specific cpus. The irq field is interpreted
754	759	like this:
755	760
756		− bits: \| 31 ... 24 \| 23 ... 16 \| 15 ... 0 \|
757		− field: \| irq_type \| vcpu_index \| irq_id \|
	761	+ bits: \| 31 ... 28 \| 27 ... 24 \| 23 ... 16 \| 15 ... 0 \|
	762	+ field: \| vcpu2_index \| irq_type \| vcpu_index \| irq_id \|
758	763
759	764	The irq_type field has the following values:
760	765	- irq_type[0]: out-of-kernel GIC: irq_id 0 is IRQ, irq_id 1 is FIQ
@@ -766,6 +771,14 @@ The irq_type field has the following values: · 5 unmodified lines
766	771
767	772	In both cases, level is used to assert/deassert the line.
768	773
	774	+When KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 is supported, the target vcpu is
	775	+identified as (256 * vcpu2_index + vcpu_index). Otherwise, vcpu2_index
	776	+must be zero.
	777	+
	778	+Note that on arm/arm64, the KVM_CAP_IRQCHIP capability only conditions
	779	+injection of interrupts for the in-kernel irqchip. KVM_IRQ_LINE can always
	780	+be used for a userspace interrupt controller.
	781	+
769	782	struct kvm_irq_level {
770	783	union {
771	784	__u32 irq; /* GSI */
@@ -3079,12 +3092,14 @@ This exception is also raised directly at the corresponding VCPU if the · 2307 unmodified lines
3079	3092	flag KVM_S390_MEMOP_F_INJECT_EXCEPTION is set in the "flags" field.
3080	3093
3081	3094	The start address of the memory region has to be specified in the "gaddr"
3082		−field, and the length of the region in the "size" field. "buf" is the buffer
3083		−supplied by the userspace application where the read data should be written
3084		−to for KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written
3085		−is stored for a KVM_S390_MEMOP_LOGICAL_WRITE. "buf" is unused and can be NULL
3086		−when KVM_S390_MEMOP_F_CHECK_ONLY is specified. "ar" designates the access
3087		−register number to be used.
	3095	+field, and the length of the region in the "size" field (which must not
	3096	+be 0). The maximum value for "size" can be obtained by checking the
	3097	+KVM_CAP_S390_MEM_OP capability. "buf" is the buffer supplied by the
	3098	+userspace application where the read data should be written to for
	3099	+KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written is
	3100	+stored for a KVM_S390_MEMOP_LOGICAL_WRITE. When KVM_S390_MEMOP_F_CHECK_ONLY
	3101	+is specified, "buf" is unused and can be NULL. "ar" designates the access
	3102	+register number to be used; the valid range is 0..15.
3088	3103
3089	3104	The "reserved" field is meant for future extensions. It is not used by
3090	3105	KVM with the currently defined set of flags.

Documentation/virt/kvm/mmu.txt+2−2

@@ -294,7 +294,7 @@ Handling a page fault is performed as follows: · 293 unmodified lines
294	294	- walk shadow page table
295	295	- check for valid generation number in the spte (see "Fast invalidation of
296	296	MMIO sptes" below)
297		− - cache the information to vcpu->arch.mmio_gva, vcpu->arch.access and
	297	+ - cache the information to vcpu->arch.mmio_gva, vcpu->arch.mmio_access and
298	298	vcpu->arch.mmio_gfn, and call the emulator
299	299	- If both P bit and R/W bit of error code are set, this could possibly
300	300	be handled as a "fast page fault" (fixed without taking the MMU lock). See
@@ -304,7 +304,7 @@ Handling a page fault is performed as follows: · 3 unmodified lines
304	304	- if permissions are insufficient, reflect the fault back to the guest
305	305	- determine the host page
306	306	- if this is an mmio request, there is no host page; cache the info to
307		− vcpu->arch.mmio_gva, vcpu->arch.access and vcpu->arch.mmio_gfn
	307	+ vcpu->arch.mmio_gva, vcpu->arch.mmio_access and vcpu->arch.mmio_gfn
308	308	- walk the shadow page table to find the spte for the translation,
309	309	instantiating missing intermediate page tables as necessary
310	310	- If this is an mmio request, cache the mmio info to the spte and set some

arch/arm/include/uapi/asm/kvm.h+3−1

@@ -266,8 +266,10 @@ struct kvm_vcpu_events { · 265 unmodified lines
266	266	#define KVM_DEV_ARM_ITS_CTRL_RESET 4
267	267
268	268	/* KVM_IRQ_LINE irq field index values */
	269	+#define KVM_ARM_IRQ_VCPU2_SHIFT 28
	270	+#define KVM_ARM_IRQ_VCPU2_MASK 0xf
269	271	#define KVM_ARM_IRQ_TYPE_SHIFT 24
270		−#define KVM_ARM_IRQ_TYPE_MASK 0xff
	272	+#define KVM_ARM_IRQ_TYPE_MASK 0xf
271	273	#define KVM_ARM_IRQ_VCPU_SHIFT 16
272	274	#define KVM_ARM_IRQ_VCPU_MASK 0xff
273	275	#define KVM_ARM_IRQ_NUM_SHIFT 0

arch/arm64/include/asm/pgtable-prot.h+1−1

@@ -77,7 +77,7 @@ · 76 unmodified lines
77	77	})
78	78
79	79	#define PAGE_S2 __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(NORMAL) \| PTE_S2_RDONLY \| PAGE_S2_XN)
80		−#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(DEVICE_nGnRE) \| PTE_S2_RDONLY \| PAGE_S2_XN)
	80	+#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(DEVICE_nGnRE) \| PTE_S2_RDONLY \| PTE_S2_XN)
81	81
82	82	#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) \| PTE_PROT_NONE \| PTE_RDONLY \| PTE_NG \| PTE_PXN \| PTE_UXN)
83	83	#define PAGE_SHARED __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN \| PTE_WRITE)

arch/arm64/include/uapi/asm/kvm.h+3−1

@@ -325,8 +325,10 @@ struct kvm_vcpu_events { · 324 unmodified lines
325	325	#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER 1
326	326
327	327	/* KVM_IRQ_LINE irq field index values */
	328	+#define KVM_ARM_IRQ_VCPU2_SHIFT 28
	329	+#define KVM_ARM_IRQ_VCPU2_MASK 0xf
328	330	#define KVM_ARM_IRQ_TYPE_SHIFT 24
329		−#define KVM_ARM_IRQ_TYPE_MASK 0xff
	331	+#define KVM_ARM_IRQ_TYPE_MASK 0xf
330	332	#define KVM_ARM_IRQ_VCPU_SHIFT 16
331	333	#define KVM_ARM_IRQ_VCPU_MASK 0xff
332	334	#define KVM_ARM_IRQ_NUM_SHIFT 0

arch/arm64/kvm/hyp/tlb.c+13−1

@@ -193,6 +193,18 @@ void __hyp_text __kvm_flush_vm_context(void) · 192 unmodified lines
193	193	{
194	194	dsb(ishst);
195	195	__tlbi(alle1is);
196		− asm volatile("ic ialluis" : : );
	196	+
	197	+ /*
	198	+ * VIPT and PIPT caches are not affected by VMID, so no maintenance
	199	+ * is necessary across a VMID rollover.
	200	+ *
	201	+ * VPIPT caches constrain lookup and maintenance to the active VMID,
	202	+ * so we need to invalidate lines with a stale VMID to avoid an ABA
	203	+ * race after multiple rollovers.
	204	+ *
	205	+ */
	206	+ if (icache_is_vpipt())
	207	+ asm volatile("ic ialluis");
	208	+
197	209	dsb(ish);
198	210	}

arch/powerpc/include/asm/kvm_host.h+18−4

@@ -232,11 +232,25 @@ struct revmap_entry { · 231 unmodified lines
232	232	};
233	233
234	234	/*
235		− * We use the top bit of each memslot->arch.rmap entry as a lock bit,
236		− * and bit 32 as a present flag. The bottom 32 bits are the
237		− * index in the guest HPT of a HPTE that points to the page.
	235	+ * The rmap array of size number of guest pages is allocated for each memslot.
	236	+ * This array is used to store usage specific information about the guest page.
	237	+ * Below are the encodings of the various possible usage types.
238	238	*/
239		−#define KVMPPC_RMAP_LOCK_BIT 63
	239	+/* Free bits which can be used to define a new usage */
	240	+#define KVMPPC_RMAP_TYPE_MASK 0xff00000000000000
	241	+#define KVMPPC_RMAP_NESTED 0xc000000000000000 /* Nested rmap array */
	242	+#define KVMPPC_RMAP_HPT 0x0100000000000000 /* HPT guest */
	243	+
	244	+/*
	245	+ * rmap usage definition for a hash page table (hpt) guest:
	246	+ * 0x0000080000000000 Lock bit
	247	+ * 0x0000018000000000 RC bits
	248	+ * 0x0000000100000000 Present bit
	249	+ * 0x00000000ffffffff HPT index bits
	250	+ * The bottom 32 bits are the index in the guest HPT of a HPTE that points to
	251	+ * the page.
	252	+ */
	253	+#define KVMPPC_RMAP_LOCK_BIT 43
240	254	#define KVMPPC_RMAP_RC_SHIFT 32
241	255	#define KVMPPC_RMAP_REFERENCED (HPTE_R_R << KVMPPC_RMAP_RC_SHIFT)
242	256	#define KVMPPC_RMAP_PRESENT 0x100000000ul

arch/powerpc/include/asm/kvm_ppc.h+1−0

@@ -598,6 +598,7 @@ extern int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, · 597 unmodified lines
598	598	union kvmppc_one_reg *val);
599	599	extern int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu,
600	600	union kvmppc_one_reg *val);
	601	+extern bool kvmppc_xive_native_supported(void);
601	602
602	603	#else
603	604	static inline int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,

arch/powerpc/include/asm/xive.h+9−0

@@ -46,7 +46,15 @@ struct xive_irq_data { · 45 unmodified lines
46	46
47	47	/* Setup/used by frontend */
48	48	int target;
	49	+ /*
	50	+ * saved_p means that there is a queue entry for this interrupt
	51	+ * in some CPU's queue (not including guest vcpu queues), even
	52	+ * if P is not set in the source ESB.
	53	+ * stale_p means that there is no queue entry for this interrupt
	54	+ * in some CPU's queue, even if P is set in the source ESB.
	55	+ */
49	56	bool saved_p;
	57	+ bool stale_p;
50	58	};
51	59	#define XIVE_IRQ_FLAG_STORE_EOI 0x01
52	60	#define XIVE_IRQ_FLAG_LSI 0x02
@@ -127,6 +135,7 @@ extern int xive_native_get_queue_state(u32 vp_id, uint32_t prio, u32 *qtoggle, · 74 unmodified lines
127	135	extern int xive_native_set_queue_state(u32 vp_id, uint32_t prio, u32 qtoggle,
128	136	u32 qindex);
129	137	extern int xive_native_get_vp_state(u32 vp_id, u64 *out_state);
	138	+extern bool xive_native_has_queue_state_support(void);
130	139
131	140	#else
132	141

arch/powerpc/kvm/book3s.c+5−3

@@ -1083,9 +1083,11 @@ static int kvmppc_book3s_init(void) · 1082 unmodified lines
1083	1083	if (xics_on_xive()) {
1084	1084	kvmppc_xive_init_module();
1085	1085	kvm_register_device_ops(&kvm_xive_ops, KVM_DEV_TYPE_XICS);
1086		− kvmppc_xive_native_init_module();
1087		− kvm_register_device_ops(&kvm_xive_native_ops,
1088		− KVM_DEV_TYPE_XIVE);
	1086	+ if (kvmppc_xive_native_supported()) {
	1087	+ kvmppc_xive_native_init_module();
	1088	+ kvm_register_device_ops(&kvm_xive_native_ops,
	1089	+ KVM_DEV_TYPE_XIVE);
	1090	+ }
1089	1091	} else
1090	1092	#endif
1091	1093	kvm_register_device_ops(&kvm_xics_ops, KVM_DEV_TYPE_XICS);

arch/powerpc/kvm/book3s_hv.c+18−6

@@ -1678,7 +1678,14 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, · 1677 unmodified lines
1678	1678	*val = get_reg_val(id, vcpu->arch.pspb);
1679	1679	break;
1680	1680	case KVM_REG_PPC_DPDES:
1681		− *val = get_reg_val(id, vcpu->arch.vcore->dpdes);
	1681	+ /*
	1682	+ * On POWER9, where we are emulating msgsndp etc.,
	1683	+ * we return 1 bit for each vcpu, which can come from
	1684	+ * either vcore->dpdes or doorbell_request.
	1685	+ * On POWER8, doorbell_request is 0.
	1686	+ */
	1687	+ *val = get_reg_val(id, vcpu->arch.vcore->dpdes \|
	1688	+ vcpu->arch.doorbell_request);
1682	1689	break;
1683	1690	case KVM_REG_PPC_VTB:
1684	1691	*val = get_reg_val(id, vcpu->arch.vcore->vtb);
@@ -2860,7 +2867,7 @@ static void collect_piggybacks(struct core_info *cip, int target_threads) · 1175 unmodified lines
2860	2867	if (!spin_trylock(&pvc->lock))
2861	2868	continue;
2862	2869	prepare_threads(pvc);
2863		− if (!pvc->n_runnable) {
	2870	+ if (!pvc->n_runnable \|\| !pvc->kvm->arch.mmu_ready) {
2864	2871	list_del_init(&pvc->preempt_list);
2865	2872	if (pvc->runner == NULL) {
2866	2873	pvc->vcore_state = VCORE_INACTIVE;
@@ -2881,15 +2888,20 @@ static void collect_piggybacks(struct core_info *cip, int target_threads) · 14 unmodified lines
2881	2888	spin_unlock(&lp->lock);
2882	2889	}
2883	2890
2884		−static bool recheck_signals(struct core_info *cip)
	2891	+static bool recheck_signals_and_mmu(struct core_info *cip)
2885	2892	{
2886	2893	int sub, i;
2887	2894	struct kvm_vcpu *vcpu;
	2895	+ struct kvmppc_vcore *vc;
2888	2896
2889		− for (sub = 0; sub < cip->n_subcores; ++sub)
2890		− for_each_runnable_thread(i, vcpu, cip->vc[sub])
	2897	+ for (sub = 0; sub < cip->n_subcores; ++sub) {
	2898	+ vc = cip->vc[sub];
	2899	+ if (!vc->kvm->arch.mmu_ready)
	2900	+ return true;
	2901	+ for_each_runnable_thread(i, vcpu, vc)
2891	2902	if (signal_pending(vcpu->arch.run_task))
2892	2903	return true;
	2904	+ }
2893	2905	return false;
2894	2906	}
2895	2907
@@ -3119,7 +3131,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) · 223 unmodified lines
3119	3131	local_irq_disable();
3120	3132	hard_irq_disable();
3121	3133	if (lazy_irq_pending() \|\| need_resched() \|\|
3122		− recheck_signals(&core_info) \|\| !vc->kvm->arch.mmu_ready) {
	3134	+ recheck_signals_and_mmu(&core_info)) {
3123	3135	local_irq_enable();
3124	3136	vc->vcore_state = VCORE_INACTIVE;
3125	3137	/* Unlock all except the primary vcore */

arch/powerpc/kvm/book3s_hv_rm_mmu.c+1−1

@@ -99,7 +99,7 @@ void kvmppc_add_revmap_chain(struct kvm kvm, struct revmap_entry rev, · 98 unmodified lines
99	99	} else {
100	100	rev->forw = rev->back = pte_index;
101	101	rmap = (rmap & ~KVMPPC_RMAP_INDEX) \|
102		− pte_index \| KVMPPC_RMAP_PRESENT;
	102	+ pte_index \| KVMPPC_RMAP_PRESENT \| KVMPPC_RMAP_HPT;
103	103	}
104	104	unlock_rmap(rmap);
105	105	}

arch/powerpc/kvm/book3s_hv_rmhandlers.S+25−13

@@ -942,6 +942,8 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) · 941 unmodified lines
942	942	ld r11, VCPU_XIVE_SAVED_STATE(r4)
943	943	li r9, TM_QW1_OS
944	944	lwz r8, VCPU_XIVE_CAM_WORD(r4)
	945	+ cmpwi r8, 0
	946	+ beq no_xive
945	947	li r7, TM_QW1_OS + TM_WORD2
946	948	mfmsr r0
947	949	andi. r0, r0, MSR_DR /* in real mode? */
@@ -2831,29 +2833,39 @@ kvm_cede_prodded: · 1883 unmodified lines
2831	2833	kvm_cede_exit:
2832	2834	ld r9, HSTATE_KVM_VCPU(r13)
2833	2835	#ifdef CONFIG_KVM_XICS
2834		− /* Abort if we still have a pending escalation */
	2836	+ /* are we using XIVE with single escalation? */
	2837	+ ld r10, VCPU_XIVE_ESC_VADDR(r9)
	2838	+ cmpdi r10, 0
	2839	+ beq 3f
	2840	+ li r6, XIVE_ESB_SET_PQ_00
	2841	+ /*
	2842	+ * If we still have a pending escalation, abort the cede,
	2843	+ * and we must set PQ to 10 rather than 00 so that we don't
	2844	+ * potentially end up with two entries for the escalation
	2845	+ * interrupt in the XIVE interrupt queue. In that case
	2846	+ * we also don't want to set xive_esc_on to 1 here in
	2847	+ * case we race with xive_esc_irq().
	2848	+ */
2835	2849	lbz r5, VCPU_XIVE_ESC_ON(r9)
2836	2850	cmpwi r5, 0
2837		− beq 1f
	2851	+ beq 4f
2838	2852	li r0, 0
2839	2853	stb r0, VCPU_CEDED(r9)
2840		−1: /* Enable XIVE escalation */
2841		− li r5, XIVE_ESB_SET_PQ_00
	2854	+ li r6, XIVE_ESB_SET_PQ_10
	2855	+ b 5f
	2856	+4: li r0, 1
	2857	+ stb r0, VCPU_XIVE_ESC_ON(r9)
	2858	+ /* make sure store to xive_esc_on is seen before xive_esc_irq runs */
	2859	+ sync
	2860	+5: /* Enable XIVE escalation */
2842	2861	mfmsr r0
2843	2862	andi. r0, r0, MSR_DR /* in real mode? */
2844	2863	beq 1f
2845		− ld r10, VCPU_XIVE_ESC_VADDR(r9)
2846		− cmpdi r10, 0
2847		− beq 3f
2848		− ldx r0, r10, r5
	2864	+ ldx r0, r10, r6
2849	2865	b 2f
2850	2866	1: ld r10, VCPU_XIVE_ESC_RADDR(r9)
2851		− cmpdi r10, 0
2852		− beq 3f
2853		− ldcix r0, r10, r5
	2867	+ ldcix r0, r10, r6
2854	2868	2: sync
2855		− li r0, 1
2856		− stb r0, VCPU_XIVE_ESC_ON(r9)
2857	2869	#endif /* CONFIG_KVM_XICS */
2858	2870	3: b guest_exit_cont
2859	2871

arch/powerpc/kvm/book3s_xive.c+51−9

@@ -67,8 +67,14 @@ void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) · 66 unmodified lines
67	67	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
68	68	u64 pq;
69	69
70		− if (!tima)
	70	+ /*
	71	+ * Nothing to do if the platform doesn't have a XIVE
	72	+ * or this vCPU doesn't have its own XIVE context
	73	+ * (e.g. because it's not using an in-kernel interrupt controller).
	74	+ */
	75	+ if (!tima \|\| !vcpu->arch.xive_cam_word)
71	76	return;
	77	+
72	78	eieio();
73	79	__raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
74	80	__raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
@@ -160,6 +166,9 @@ static irqreturn_t xive_esc_irq(int irq, void *data) · 85 unmodified lines
160	166	*/
161	167	vcpu->arch.xive_esc_on = false;
162	168
	169	+ /* This orders xive_esc_on = false vs. subsequent stale_p = true */
	170	+ smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
	171	+
163	172	return IRQ_HANDLED;
164	173	}
165	174
@@ -1113,6 +1122,31 @@ void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu) · 947 unmodified lines
1113	1122	vcpu->arch.xive_esc_raddr = 0;
1114	1123	}
1115	1124
	1125	+/*
	1126	+ * In single escalation mode, the escalation interrupt is marked so
	1127	+ * that EOI doesn't re-enable it, but just sets the stale_p flag to
	1128	+ * indicate that the P bit has already been dealt with. However, the
	1129	+ * assembly code that enters the guest sets PQ to 00 without clearing
	1130	+ * stale_p (because it has no easy way to address it). Hence we have
	1131	+ * to adjust stale_p before shutting down the interrupt.
	1132	+ */
	1133	+void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
	1134	+ struct kvmppc_xive_vcpu *xc, int irq)
	1135	+{
	1136	+ struct irq_data *d = irq_get_irq_data(irq);
	1137	+ struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
	1138	+
	1139	+ /*
	1140	+ * This slightly odd sequence gives the right result
	1141	+ * (i.e. stale_p set if xive_esc_on is false) even if
	1142	+ * we race with xive_esc_irq() and xive_irq_eoi().
	1143	+ */
	1144	+ xd->stale_p = false;
	1145	+ smp_mb(); /* paired with smb_wmb in xive_esc_irq */
	1146	+ if (!vcpu->arch.xive_esc_on)
	1147	+ xd->stale_p = true;
	1148	+}
	1149	+
1116	1150	void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1117	1151	{
1118	1152	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
@@ -1134,20 +1168,28 @@ void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) · 15 unmodified lines
1134	1168	/* Mask the VP IPI */
1135	1169	xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1136	1170
1137		− /* Disable the VP */
1138		− xive_native_disable_vp(xc->vp_id);
1139		−
1140		− /* Free the queues & associated interrupts */
	1171	+ /* Free escalations */
1141	1172	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1142		− struct xive_q *q = &xc->queues[i];
1143		−
1144		− /* Free the escalation irq */
1145	1173	if (xc->esc_virq[i]) {
	1174	+ if (xc->xive->single_escalation)
	1175	+ xive_cleanup_single_escalation(vcpu, xc,
	1176	+ xc->esc_virq[i]);
1146	1177	free_irq(xc->esc_virq[i], vcpu);
1147	1178	irq_dispose_mapping(xc->esc_virq[i]);
1148	1179	kfree(xc->esc_virq_names[i]);
1149	1180	}
1150		− /* Free the queue */
	1181	+ }
	1182	+
	1183	+ /* Disable the VP */
	1184	+ xive_native_disable_vp(xc->vp_id);
	1185	+
	1186	+ /* Clear the cam word so guest entry won't try to push context */
	1187	+ vcpu->arch.xive_cam_word = 0;
	1188	+
	1189	+ /* Free the queues */
	1190	+ for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
	1191	+ struct xive_q *q = &xc->queues[i];
	1192	+
1151	1193	xive_native_disable_queue(xc->vp_id, q, i);
1152	1194	if (q->qpage) {
1153	1195	free_pages((unsigned long)q->qpage,

arch/powerpc/kvm/book3s_xive.h+2−0

@@ -282,6 +282,8 @@ int kvmppc_xive_select_target(struct kvm kvm, u32 server, u8 prio); · 281 unmodified lines
282	282	int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
283	283	bool single_escalation);
284	284	struct kvmppc_xive kvmppc_xive_get_device(struct kvm kvm, u32 type);
	285	+void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
	286	+ struct kvmppc_xive_vcpu *xc, int irq);
285	287
286	288	#endif /* CONFIG_KVM_XICS */
287	289	#endif /* _KVM_PPC_BOOK3S_XICS_H */

arch/powerpc/kvm/book3s_xive_native.c+18−5

@@ -67,20 +67,28 @@ void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) · 66 unmodified lines
67	67	xc->valid = false;
68	68	kvmppc_xive_disable_vcpu_interrupts(vcpu);
69	69
70		− /* Disable the VP */
71		− xive_native_disable_vp(xc->vp_id);
72		−
73		− /* Free the queues & associated interrupts */
	70	+ /* Free escalations */
74	71	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
75	72	/* Free the escalation irq */
76	73	if (xc->esc_virq[i]) {
	74	+ if (xc->xive->single_escalation)
	75	+ xive_cleanup_single_escalation(vcpu, xc,
	76	+ xc->esc_virq[i]);
77	77	free_irq(xc->esc_virq[i], vcpu);
78	78	irq_dispose_mapping(xc->esc_virq[i]);
79	79	kfree(xc->esc_virq_names[i]);
80	80	xc->esc_virq[i] = 0;
81	81	}
	82	+ }
	83	+
	84	+ /* Disable the VP */
	85	+ xive_native_disable_vp(xc->vp_id);
	86	+
	87	+ /* Clear the cam word so guest entry won't try to push context */
	88	+ vcpu->arch.xive_cam_word = 0;
82	89
83		− /* Free the queue */
	90	+ /* Free the queues */
	91	+ for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
84	92	kvmppc_xive_native_cleanup_queue(vcpu, i);
85	93	}
86	94
@@ -1171,6 +1179,11 @@ int kvmppc_xive_native_set_vp(struct kvm_vcpu vcpu, union kvmppc_one_reg val) · 1084 unmodified lines
1171	1179	return 0;
1172	1180	}
1173	1181
	1182	+bool kvmppc_xive_native_supported(void)
	1183	+{
	1184	+ return xive_native_has_queue_state_support();
	1185	+}
	1186	+
1174	1187	static int xive_native_debug_show(struct seq_file m, void private)
1175	1188	{
1176	1189	struct kvmppc_xive *xive = m->private;

arch/powerpc/kvm/e500.c+3−0

@@ -440,6 +440,9 @@ static struct kvm_vcpu kvmppc_core_vcpu_create_e500(struct kvm kvm, · 439 unmodified lines
440	440	struct kvm_vcpu *vcpu;
441	441	int err;
442	442
	443	+ BUILD_BUG_ON_MSG(offsetof(struct kvmppc_vcpu_e500, vcpu) != 0,
	444	+ "struct kvm_vcpu must be at offset 0 for arch usercopy region");
	445	+
443	446	vcpu_e500 = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
444	447	if (!vcpu_e500) {
445	448	err = -ENOMEM;

arch/powerpc/kvm/emulate.c+1−0

@@ -271,6 +271,7 @@ int kvmppc_emulate_instruction(struct kvm_run run, struct kvm_vcpu vcpu) · 270 unmodified lines
271	271	*/
272	272	if (inst == KVMPPC_INST_SW_BREAKPOINT) {
273	273	run->exit_reason = KVM_EXIT_DEBUG;
	274	+ run->debug.arch.status = 0;
274	275	run->debug.arch.address = kvmppc_get_pc(vcpu);
275	276	emulated = EMULATE_EXIT_USER;
276	277	advance = 0;

arch/powerpc/kvm/emulate_loadstore.c+0−6

@@ -89,12 +89,6 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) · 88 unmodified lines
89	89	rs = get_rs(inst);
90	90	rt = get_rt(inst);
91	91
92		− /*
93		− * if mmio_vsx_tx_sx_enabled == 0, copy data between
94		− * VSR[0..31] and memory
95		− * if mmio_vsx_tx_sx_enabled == 1, copy data between
96		− * VSR[32..63] and memory
97		− */
98	92	vcpu->arch.mmio_vsx_copy_nums = 0;
99	93	vcpu->arch.mmio_vsx_offset = 0;
100	94	vcpu->arch.mmio_copy_type = KVMPPC_VSX_COPY_NONE;

arch/powerpc/kvm/powerpc.c+2−1

@@ -561,7 +561,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 560 unmodified lines
561	561	* a POWER9 processor) and the PowerNV platform, as
562	562	* nested is not yet supported.
563	563	*/
564		− r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE);
	564	+ r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE) &&
	565	+ kvmppc_xive_native_supported();
565	566	break;
566	567	#endif
567	568

arch/powerpc/sysdev/xive/common.c+64−23

@@ -135,7 +135,7 @@ static u32 xive_read_eq(struct xive_q *q, bool just_peek) · 134 unmodified lines
135	135	static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek)
136	136	{
137	137	u32 irq = 0;
138		− u8 prio;
	138	+ u8 prio = 0;
139	139
140	140	/* Find highest pending priority */
141	141	while (xc->pending_prio != 0) {
@@ -148,8 +148,19 @@ static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek) · 6 unmodified lines
148	148	irq = xive_read_eq(&xc->queue[prio], just_peek);
149	149
150	150	/* Found something ? That's it */
151		− if (irq)
152		− break;
	151	+ if (irq) {
	152	+ if (just_peek \|\| irq_to_desc(irq))
	153	+ break;
	154	+ /*
	155	+ * We should never get here; if we do then we must
	156	+ * have failed to synchronize the interrupt properly
	157	+ * when shutting it down.
	158	+ */
	159	+ pr_crit("xive: got interrupt %d without descriptor, dropping\n",
	160	+ irq);
	161	+ WARN_ON(1);
	162	+ continue;
	163	+ }
153	164
154	165	/* Clear pending bits */
155	166	xc->pending_prio &= ~(1 << prio);
@@ -307,6 +318,7 @@ static void xive_do_queue_eoi(struct xive_cpu *xc) · 151 unmodified lines
307	318	*/
308	319	static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
309	320	{
	321	+ xd->stale_p = false;
310	322	/* If the XIVE supports the new "store EOI facility, use it */
311	323	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
312	324	xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);
@@ -350,7 +362,7 @@ static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd) · 37 unmodified lines
350	362	}
351	363	}
352	364
353		−/* irq_chip eoi callback */
	365	+/* irq_chip eoi callback, called with irq descriptor lock held */
354	366	static void xive_irq_eoi(struct irq_data *d)
355	367	{
356	368	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
@@ -366,6 +378,8 @@ static void xive_irq_eoi(struct irq_data *d) · 9 unmodified lines
366	378	if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) &&
367	379	!(xd->flags & XIVE_IRQ_NO_EOI))
368	380	xive_do_source_eoi(irqd_to_hwirq(d), xd);
	381	+ else
	382	+ xd->stale_p = true;
369	383
370	384	/*
371	385	* Clear saved_p to indicate that it's no longer occupying
@@ -397,11 +411,16 @@ static void xive_do_source_set_mask(struct xive_irq_data *xd, · 25 unmodified lines
397	411	*/
398	412	if (mask) {
399	413	val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01);
400		− xd->saved_p = !!(val & XIVE_ESB_VAL_P);
401		− } else if (xd->saved_p)
	414	+ if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P))
	415	+ xd->saved_p = true;
	416	+ xd->stale_p = false;
	417	+ } else if (xd->saved_p) {
402	418	xive_esb_read(xd, XIVE_ESB_SET_PQ_10);
403		− else
	419	+ xd->saved_p = false;
	420	+ } else {
404	421	xive_esb_read(xd, XIVE_ESB_SET_PQ_00);
	422	+ xd->stale_p = false;
	423	+ }
405	424	}
406	425
407	426	/*
@@ -541,6 +560,8 @@ static unsigned int xive_irq_startup(struct irq_data *d) · 133 unmodified lines
541	560	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
542	561	int target, rc;
543	562
	563	+ xd->saved_p = false;
	564	+ xd->stale_p = false;
544	565	pr_devel("xive_irq_startup: irq %d [0x%x] data @%p\n",
545	566	d->irq, hw_irq, d);
546	567
@@ -587,6 +608,7 @@ static unsigned int xive_irq_startup(struct irq_data *d) · 40 unmodified lines
587	608	return 0;
588	609	}
589	610
	611	+/* called with irq descriptor lock held */
590	612	static void xive_irq_shutdown(struct irq_data *d)
591	613	{
592	614	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
@@ -601,16 +623,6 @@ static void xive_irq_shutdown(struct irq_data *d) · 8 unmodified lines
601	623	/* Mask the interrupt at the source */
602	624	xive_do_source_set_mask(xd, true);
603	625
604		− /*
605		− * The above may have set saved_p. We clear it otherwise it
606		− * will prevent re-enabling later on. It is ok to forget the
607		− * fact that the interrupt might be in a queue because we are
608		− * accounting that already in xive_dec_target_count() and will
609		− * be re-routing it to a new queue with proper accounting when
610		− * it's started up again
611		− */
612		− xd->saved_p = false;
613		−
614	626	/*
615	627	* Mask the interrupt in HW in the IVT/EAS and set the number
616	628	* to be the "bad" IRQ number
@@ -797,6 +809,10 @@ static int xive_irq_retrigger(struct irq_data *d) · 180 unmodified lines
797	809	return 1;
798	810	}
799	811
	812	+/*
	813	+ * Caller holds the irq descriptor lock, so this won't be called
	814	+ * concurrently with xive_get_irqchip_state on the same interrupt.
	815	+ */
800	816	static int xive_irq_set_vcpu_affinity(struct irq_data d, void state)
801	817	{
802	818	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
@@ -820,6 +836,10 @@ static int xive_irq_set_vcpu_affinity(struct irq_data d, void state) · 17 unmodified lines
820	836
821	837	/* Set it to PQ=10 state to prevent further sends */
822	838	pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10);
	839	+ if (!xd->stale_p) {
	840	+ xd->saved_p = !!(pq & XIVE_ESB_VAL_P);
	841	+ xd->stale_p = !xd->saved_p;
	842	+ }
823	843
824	844	/* No target ? nothing to do */
825	845	if (xd->target == XIVE_INVALID_TARGET) {
@@ -827,7 +847,7 @@ static int xive_irq_set_vcpu_affinity(struct irq_data d, void state) · 1 unmodified line
827	847	* An untargetted interrupt should have been
828	848	* also masked at the source
829	849	*/
830		− WARN_ON(pq & 2);
	850	+ WARN_ON(xd->saved_p);
831	851
832	852	return 0;
833	853	}
@@ -847,9 +867,8 @@ static int xive_irq_set_vcpu_affinity(struct irq_data d, void state) · 13 unmodified lines
847	867	* This saved_p is cleared by the host EOI, when we know
848	868	* for sure the queue slot is no longer in use.
849	869	*/
850		− if (pq & 2) {
851		− pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_11);
852		− xd->saved_p = true;
	870	+ if (xd->saved_p) {
	871	+ xive_esb_read(xd, XIVE_ESB_SET_PQ_11);
853	872
854	873	/*
855	874	* Sync the XIVE source HW to ensure the interrupt
@@ -862,8 +881,7 @@ static int xive_irq_set_vcpu_affinity(struct irq_data d, void state) · 6 unmodified lines
862	881	*/
863	882	if (xive_ops->sync_source)
864	883	xive_ops->sync_source(hw_irq);
865		− } else
866		− xd->saved_p = false;
	884	+ }
867	885	} else {
868	886	irqd_clr_forwarded_to_vcpu(d);
869	887
@@ -914,6 +932,23 @@ static int xive_irq_set_vcpu_affinity(struct irq_data d, void state) · 44 unmodified lines
914	932	return 0;
915	933	}
916	934
	935	+/* Called with irq descriptor lock held. */
	936	+static int xive_get_irqchip_state(struct irq_data *data,
	937	+ enum irqchip_irq_state which, bool *state)
	938	+{
	939	+ struct xive_irq_data *xd = irq_data_get_irq_handler_data(data);
	940	+
	941	+ switch (which) {
	942	+ case IRQCHIP_STATE_ACTIVE:
	943	+ *state = !xd->stale_p &&
	944	+ (xd->saved_p \|\|
	945	+ !!(xive_esb_read(xd, XIVE_ESB_GET) & XIVE_ESB_VAL_P));
	946	+ return 0;
	947	+ default:
	948	+ return -EINVAL;
	949	+ }
	950	+}
	951	+
917	952	static struct irq_chip xive_irq_chip = {
918	953	.name = "XIVE-IRQ",
919	954	.irq_startup = xive_irq_startup,
@@ -925,6 +960,7 @@ static struct irq_chip xive_irq_chip = { · 5 unmodified lines
925	960	.irq_set_type = xive_irq_set_type,
926	961	.irq_retrigger = xive_irq_retrigger,
927	962	.irq_set_vcpu_affinity = xive_irq_set_vcpu_affinity,
	963	+ .irq_get_irqchip_state = xive_get_irqchip_state,
928	964	};
929	965
930	966	bool is_xive_irq(struct irq_chip *chip)
@@ -1337,6 +1373,11 @@ static void xive_flush_cpu_queue(unsigned int cpu, struct xive_cpu *xc) · 406 unmodified lines
1337	1373	raw_spin_lock(&desc->lock);
1338	1374	xd = irq_desc_get_handler_data(desc);
1339	1375
	1376	+ /*
	1377	+ * Clear saved_p to indicate that it's no longer pending
	1378	+ */
	1379	+ xd->saved_p = false;
	1380	+
1340	1381	/*
1341	1382	* For LSIs, we EOI, this will cause a resend if it's
1342	1383	* still asserted. Otherwise do an MSI retrigger.

arch/powerpc/sysdev/xive/native.c+7−0

@@ -800,6 +800,13 @@ int xive_native_set_queue_state(u32 vp_id, u32 prio, u32 qtoggle, u32 qindex) · 799 unmodified lines
800	800	}
801	801	EXPORT_SYMBOL_GPL(xive_native_set_queue_state);
802	802
	803	+bool xive_native_has_queue_state_support(void)
	804	+{
	805	+ return opal_check_token(OPAL_XIVE_GET_QUEUE_STATE) &&
	806	+ opal_check_token(OPAL_XIVE_SET_QUEUE_STATE);
	807	+}
	808	+EXPORT_SYMBOL_GPL(xive_native_has_queue_state_support);
	809	+
803	810	int xive_native_get_vp_state(u32 vp_id, u64 *out_state)
804	811	{
805	812	__be64 state;

arch/s390/include/uapi/asm/kvm.h+6−0

@@ -231,6 +231,12 @@ struct kvm_guest_debug_arch { · 230 unmodified lines
231	231	#define KVM_SYNC_GSCB (1UL << 9)
232	232	#define KVM_SYNC_BPBC (1UL << 10)
233	233	#define KVM_SYNC_ETOKEN (1UL << 11)
	234	+
	235	+#define KVM_SYNC_S390_VALID_FIELDS \
	236	+ (KVM_SYNC_PREFIX \| KVM_SYNC_GPRS \| KVM_SYNC_ACRS \| KVM_SYNC_CRS \| \
	237	+ KVM_SYNC_ARCH0 \| KVM_SYNC_PFAULT \| KVM_SYNC_VRS \| KVM_SYNC_RICCB \| \
	238	+ KVM_SYNC_FPRS \| KVM_SYNC_GSCB \| KVM_SYNC_BPBC \| KVM_SYNC_ETOKEN)
	239	+
234	240	/* length and alignment of the sdnx as a power of two */
235	241	#define SDNXC 8
236	242	#define SDNXL (1UL << SDNXC)

arch/s390/kvm/kvm-s390.c+5−1

@@ -4000,6 +4000,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 3999 unmodified lines
4000	4000	if (kvm_run->immediate_exit)
4001	4001	return -EINTR;
4002	4002
	4003	+ if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS \|\|
	4004	+ kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
	4005	+ return -EINVAL;
	4006	+
4003	4007	vcpu_load(vcpu);
4004	4008
4005	4009	if (guestdbg_exit_pending(vcpu)) {
@@ -4257,7 +4261,7 @@ static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, · 251 unmodified lines
4257	4261	const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4258	4262	\| KVM_S390_MEMOP_F_CHECK_ONLY;
4259	4263
4260		− if (mop->flags & ~supported_flags)
	4264	+ if (mop->flags & ~supported_flags \|\| mop->ar >= NUM_ACRS \|\| !mop->size)
4261	4265	return -EINVAL;
4262	4266
4263	4267	if (mop->size > MEM_OP_MAX_SIZE)

arch/x86/include/asm/kvm_emulate.h+2−1

@@ -229,7 +229,7 @@ struct x86_emulate_ops { · 228 unmodified lines
229	229	int (pre_leave_smm)(struct x86_emulate_ctxt ctxt,
230	230	const char *smstate);
231	231	void (post_leave_smm)(struct x86_emulate_ctxt ctxt);
232		−
	232	+ int (set_xcr)(struct x86_emulate_ctxt ctxt, u32 index, u64 xcr);
233	233	};
234	234
235	235	typedef u32 __attribute__((vector_size(16))) sse128_t;
@@ -429,6 +429,7 @@ enum x86_intercept { · 193 unmodified lines
429	429	x86_intercept_ins,
430	430	x86_intercept_out,
431	431	x86_intercept_outs,
	432	+ x86_intercept_xsetbv,
432	433
433	434	nr_x86_intercepts
434	435	};

arch/x86/include/asm/kvm_host.h+15−4

@@ -718,7 +718,7 @@ struct kvm_vcpu_arch { · 717 unmodified lines
718	718
719	719	/* Cache MMIO info */
720	720	u64 mmio_gva;
721		− unsigned access;
	721	+ unsigned mmio_access;
722	722	gfn_t mmio_gfn;
723	723	u64 mmio_gen;
724	724
@@ -1072,7 +1072,7 @@ struct kvm_x86_ops { · 347 unmodified lines
1072	1072
1073	1073	void (run)(struct kvm_vcpu vcpu);
1074	1074	int (handle_exit)(struct kvm_vcpu vcpu);
1075		− void (skip_emulated_instruction)(struct kvm_vcpu vcpu);
	1075	+ int (skip_emulated_instruction)(struct kvm_vcpu vcpu);
1076	1076	void (set_interrupt_shadow)(struct kvm_vcpu vcpu, int mask);
1077	1077	u32 (get_interrupt_shadow)(struct kvm_vcpu vcpu);
1078	1078	void (patch_hypercall)(struct kvm_vcpu vcpu,
@@ -1211,6 +1211,8 @@ struct kvm_x86_ops { · 132 unmodified lines
1211	1211	uint16_t (nested_get_evmcs_version)(struct kvm_vcpu vcpu);
1212	1212
1213	1213	bool (need_emulation_on_page_fault)(struct kvm_vcpu vcpu);
	1214	+
	1215	+ bool (apic_init_signal_blocked)(struct kvm_vcpu vcpu);
1214	1216	};
1215	1217
1216	1218	struct kvm_arch_async_pf {
@@ -1328,8 +1330,10 @@ int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, · 111 unmodified lines
1328	1330
1329	1331	void kvm_enable_efer_bits(u64);
1330	1332	bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
1331		−int kvm_get_msr(struct kvm_vcpu vcpu, struct msr_data msr);
1332		−int kvm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr);
	1333	+int kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data);
	1334	+int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data);
	1335	+int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
	1336	+int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
1333	1337
1334	1338	struct x86_emulate_ctxt;
1335	1339
@@ -1583,6 +1587,13 @@ bool kvm_intr_is_single_vcpu(struct kvm kvm, struct kvm_lapic_irq irq, · 247 unmodified lines
1583	1587	void kvm_set_msi_irq(struct kvm kvm, struct kvm_kernel_irq_routing_entry e,
1584	1588	struct kvm_lapic_irq *irq);
1585	1589
	1590	+static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
	1591	+{
	1592	+ /* We can only post Fixed and LowPrio IRQs */
	1593	+ return (irq->delivery_mode == dest_Fixed \|\|
	1594	+ irq->delivery_mode == dest_LowestPrio);
	1595	+}
	1596	+
1586	1597	static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
1587	1598	{
1588	1599	if (kvm_x86_ops->vcpu_blocking)

arch/x86/include/asm/vmx.h+14−0

@@ -562,6 +562,20 @@ enum vm_instruction_error_number { · 561 unmodified lines
562	562	VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,
563	563	};
564	564
	565	+/*
	566	+ * VM-instruction errors that can be encountered on VM-Enter, used to trace
	567	+ * nested VM-Enter failures reported by hardware. Errors unique to VM-Enter
	568	+ * from a SMI Transfer Monitor are not included as things have gone seriously
	569	+ * sideways if we get one of those...
	570	+ */
	571	+#define VMX_VMENTER_INSTRUCTION_ERRORS \
	572	+ { VMXERR_VMLAUNCH_NONCLEAR_VMCS, "VMLAUNCH_NONCLEAR_VMCS" }, \
	573	+ { VMXERR_VMRESUME_NONLAUNCHED_VMCS, "VMRESUME_NONLAUNCHED_VMCS" }, \
	574	+ { VMXERR_VMRESUME_AFTER_VMXOFF, "VMRESUME_AFTER_VMXOFF" }, \
	575	+ { VMXERR_ENTRY_INVALID_CONTROL_FIELD, "VMENTRY_INVALID_CONTROL_FIELD" }, \
	576	+ { VMXERR_ENTRY_INVALID_HOST_STATE_FIELD, "VMENTRY_INVALID_HOST_STATE_FIELD" }, \
	577	+ { VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS, "VMENTRY_EVENTS_BLOCKED_BY_MOV_SS" }
	578	+
565	579	enum vmx_l1d_flush_state {
566	580	VMENTER_L1D_FLUSH_AUTO,
567	581	VMENTER_L1D_FLUSH_NEVER,

arch/x86/include/uapi/asm/vmx.h+2−0

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#define EXIT_REASON_EXCEPTION_NMI 0
32	32	#define EXIT_REASON_EXTERNAL_INTERRUPT 1
33	33	#define EXIT_REASON_TRIPLE_FAULT 2
	34	+#define EXIT_REASON_INIT_SIGNAL 3
34	35
35	36	#define EXIT_REASON_PENDING_INTERRUPT 7
36	37	#define EXIT_REASON_NMI_WINDOW 8
@@ -90,6 +91,7 @@ · 53 unmodified lines
90	91	{ EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
91	92	{ EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \
92	93	{ EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \
	94	+ { EXIT_REASON_INIT_SIGNAL, "INIT_SIGNAL" }, \
93	95	{ EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \
94	96	{ EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \
95	97	{ EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \

arch/x86/kernel/kvm.c+0−12

@@ -502,16 +502,6 @@ static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) · 501 unmodified lines
502	502	__send_ipi_mask(local_mask, vector);
503	503	}
504	504
505		−static void kvm_send_ipi_allbutself(int vector)
506		−{
507		− kvm_send_ipi_mask_allbutself(cpu_online_mask, vector);
508		−}
509		−
510		−static void kvm_send_ipi_all(int vector)
511		−{
512		− __send_ipi_mask(cpu_online_mask, vector);
513		−}
514		−
515	505	/*
516	506	* Set the IPI entry points
517	507	*/
@@ -519,8 +509,6 @@ static void kvm_setup_pv_ipi(void) · 1 unmodified line
519	509	{
520	510	apic->send_IPI_mask = kvm_send_ipi_mask;
521	511	apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself;
522		− apic->send_IPI_allbutself = kvm_send_ipi_allbutself;
523		− apic->send_IPI_all = kvm_send_ipi_all;
524	512	pr_info("KVM setup pv IPIs\n");
525	513	}
526	514

arch/x86/kvm/cpuid.c+19−8

@@ -392,6 +392,12 @@ static inline void do_cpuid_7_mask(struct kvm_cpuid_entry2 *entry, int index) · 391 unmodified lines
392	392
393	393	entry->edx &= kvm_cpuid_7_0_edx_x86_features;
394	394	cpuid_mask(&entry->edx, CPUID_7_EDX);
	395	+ if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
	396	+ entry->edx \|= F(SPEC_CTRL);
	397	+ if (boot_cpu_has(X86_FEATURE_STIBP))
	398	+ entry->edx \|= F(INTEL_STIBP);
	399	+ if (boot_cpu_has(X86_FEATURE_SSBD))
	400	+ entry->edx \|= F(SPEC_CTRL_SSBD);
395	401	/*
396	402	* We emulate ARCH_CAPABILITIES in software even
397	403	* if the host doesn't support it.
@@ -729,18 +735,23 @@ static inline int __do_cpuid_func(struct kvm_cpuid_entry2 *entry, u32 function, · 331 unmodified lines
729	735	g_phys_as = phys_as;
730	736	entry->eax = g_phys_as \| (virt_as << 8);
731	737	entry->edx = 0;
	738	+ entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
	739	+ cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
732	740	/*
733		− * IBRS, IBPB and VIRT_SSBD aren't necessarily present in
734		− * hardware cpuid
	741	+ * AMD has separate bits for each SPEC_CTRL bit.
	742	+ * arch/x86/kernel/cpu/bugs.c is kind enough to
	743	+ * record that in cpufeatures so use them.
735	744	*/
736		− if (boot_cpu_has(X86_FEATURE_AMD_IBPB))
	745	+ if (boot_cpu_has(X86_FEATURE_IBPB))
737	746	entry->ebx \|= F(AMD_IBPB);
738		− if (boot_cpu_has(X86_FEATURE_AMD_IBRS))
	747	+ if (boot_cpu_has(X86_FEATURE_IBRS))
739	748	entry->ebx \|= F(AMD_IBRS);
740		− if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
741		− entry->ebx \|= F(VIRT_SSBD);
742		− entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
743		− cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
	749	+ if (boot_cpu_has(X86_FEATURE_STIBP))
	750	+ entry->ebx \|= F(AMD_STIBP);
	751	+ if (boot_cpu_has(X86_FEATURE_SSBD))
	752	+ entry->ebx \|= F(AMD_SSBD);
	753	+ if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
	754	+ entry->ebx \|= F(AMD_SSB_NO);
744	755	/*
745	756	* The preference is to use SPEC CTRL MSR instead of the
746	757	* VIRT_SPEC MSR.

arch/x86/kvm/emulate.c+25−2

@@ -4156,6 +4156,20 @@ out: · 4155 unmodified lines
4156	4156	return rc;
4157	4157	}
4158	4158
	4159	+static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
	4160	+{
	4161	+ u32 eax, ecx, edx;
	4162	+
	4163	+ eax = reg_read(ctxt, VCPU_REGS_RAX);
	4164	+ edx = reg_read(ctxt, VCPU_REGS_RDX);
	4165	+ ecx = reg_read(ctxt, VCPU_REGS_RCX);
	4166	+
	4167	+ if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) \| eax))
	4168	+ return emulate_gp(ctxt, 0);
	4169	+
	4170	+ return X86EMUL_CONTINUE;
	4171	+}
	4172	+
4159	4173	static bool valid_cr(int nr)
4160	4174	{
4161	4175	switch (nr) {
@@ -4409,6 +4423,12 @@ static const struct opcode group7_rm1[] = { · 247 unmodified lines
4409	4423	N, N, N, N, N, N,
4410	4424	};
4411	4425
	4426	+static const struct opcode group7_rm2[] = {
	4427	+ N,
	4428	+ II(ImplicitOps \| Priv, em_xsetbv, xsetbv),
	4429	+ N, N, N, N, N, N,
	4430	+};
	4431	+
4412	4432	static const struct opcode group7_rm3[] = {
4413	4433	DIP(SrcNone \| Prot \| Priv, vmrun, check_svme_pa),
4414	4434	II(SrcNone \| Prot \| EmulateOnUD, em_hypercall, vmmcall),
@@ -4498,7 +4518,8 @@ static const struct group_dual group7 = { { · 83 unmodified lines
4498	4518	}, {
4499	4519	EXT(0, group7_rm0),
4500	4520	EXT(0, group7_rm1),
4501		− N, EXT(0, group7_rm3),
	4521	+ EXT(0, group7_rm2),
	4522	+ EXT(0, group7_rm3),
4502	4523	II(SrcNone \| DstMem \| Mov, em_smsw, smsw), N,
4503	4524	II(SrcMem16 \| Mov \| Priv, em_lmsw, lmsw),
4504	4525	EXT(0, group7_rm7),
@@ -5144,7 +5165,7 @@ int x86_decode_insn(struct x86_emulate_ctxt ctxt, void insn, int insn_len) · 639 unmodified lines
5144	5165	else {
5145	5166	rc = __do_insn_fetch_bytes(ctxt, 1);
5146	5167	if (rc != X86EMUL_CONTINUE)
5147		− return rc;
	5168	+ goto done;
5148	5169	}
5149	5170
5150	5171	switch (mode) {
@@ -5395,6 +5416,8 @@ done_prefixes: · 244 unmodified lines
5395	5416	ctxt->memopp->addr.mem.ea + ctxt->_eip);
5396	5417
5397	5418	done:
	5419	+ if (rc == X86EMUL_PROPAGATE_FAULT)
	5420	+ ctxt->have_exception = true;
5398	5421	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5399	5422	}
5400	5423

arch/x86/kvm/lapic.c+11−9

@@ -1198,10 +1198,8 @@ void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) · 1197 unmodified lines
1198	1198	}
1199	1199	EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1200	1200
1201		−static void apic_send_ipi(struct kvm_lapic *apic)
	1201	+static void apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1202	1202	{
1203		− u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1204		− u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
1205	1203	struct kvm_lapic_irq irq;
1206	1204
1207	1205	irq.vector = icr_low & APIC_VECTOR_MASK;
@@ -1914,8 +1912,9 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) · 706 unmodified lines
1914	1912	}
1915	1913	case APIC_ICR:
1916	1914	/* No delay here, so we always clear the pending bit */
1917		− kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
1918		− apic_send_ipi(apic);
	1915	+ val &= ~(1 << 12);
	1916	+ apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
	1917	+ kvm_lapic_set_reg(apic, APIC_ICR, val);
1919	1918	break;
1920	1919
1921	1920	case APIC_ICR2:
@@ -2707,11 +2706,14 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) · 785 unmodified lines
2707	2706	return;
2708	2707
2709	2708	/*
2710		− * INITs are latched while in SMM. Because an SMM CPU cannot
2711		− * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2712		− * and delay processing of INIT until the next RSM.
	2709	+ * INITs are latched while CPU is in specific states
	2710	+ * (SMM, VMX non-root mode, SVM with GIF=0).
	2711	+ * Because a CPU cannot be in these states immediately
	2712	+ * after it has processed an INIT signal (and thus in
	2713	+ * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
	2714	+ * and leave the INIT pending.
2713	2715	*/
2714		− if (is_smm(vcpu)) {
	2716	+ if (is_smm(vcpu) \|\| kvm_x86_ops->apic_init_signal_blocked(vcpu)) {
2715	2717	WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2716	2718	if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2717	2719	clear_bit(KVM_APIC_SIPI, &apic->pending_events);

arch/x86/kvm/mmu.c+39−22

@@ -214,6 +214,7 @@ static u64 __read_mostly shadow_accessed_mask; · 213 unmodified lines
214	214	static u64 __read_mostly shadow_dirty_mask;
215	215	static u64 __read_mostly shadow_mmio_mask;
216	216	static u64 __read_mostly shadow_mmio_value;
	217	+static u64 __read_mostly shadow_mmio_access_mask;
217	218	static u64 __read_mostly shadow_present_mask;
218	219	static u64 __read_mostly shadow_me_mask;
219	220
@@ -299,14 +300,21 @@ static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, · 79 unmodified lines
299	300	kvm_flush_remote_tlbs_with_range(kvm, &range);
300	301	}
301	302
302		−void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)
	303	+void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask)
303	304	{
	305	+ BUG_ON((u64)(unsigned)access_mask != access_mask);
304	306	BUG_ON((mmio_mask & mmio_value) != mmio_value);
305	307	shadow_mmio_value = mmio_value \| SPTE_SPECIAL_MASK;
306	308	shadow_mmio_mask = mmio_mask \| SPTE_SPECIAL_MASK;
	309	+ shadow_mmio_access_mask = access_mask;
307	310	}
308	311	EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
309	312
	313	+static bool is_mmio_spte(u64 spte)
	314	+{
	315	+ return (spte & shadow_mmio_mask) == shadow_mmio_value;
	316	+}
	317	+
310	318	static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
311	319	{
312	320	return sp->role.ad_disabled;
@@ -314,19 +322,19 @@ static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) · 1 unmodified line
314	322
315	323	static inline bool spte_ad_enabled(u64 spte)
316	324	{
317		− MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	325	+ MMU_WARN_ON(is_mmio_spte(spte));
318	326	return !(spte & shadow_acc_track_value);
319	327	}
320	328
321	329	static inline u64 spte_shadow_accessed_mask(u64 spte)
322	330	{
323		− MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	331	+ MMU_WARN_ON(is_mmio_spte(spte));
324	332	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
325	333	}
326	334
327	335	static inline u64 spte_shadow_dirty_mask(u64 spte)
328	336	{
329		− MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);
	337	+ MMU_WARN_ON(is_mmio_spte(spte));
330	338	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
331	339	}
332	340
@@ -389,7 +397,7 @@ static void mark_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, u64 gfn, · 56 unmodified lines
389	397	u64 mask = generation_mmio_spte_mask(gen);
390	398	u64 gpa = gfn << PAGE_SHIFT;
391	399
392		− access &= ACC_WRITE_MASK \| ACC_USER_MASK;
	400	+ access &= shadow_mmio_access_mask;
393	401	mask \|= shadow_mmio_value \| access;
394	402	mask \|= gpa \| shadow_nonpresent_or_rsvd_mask;
395	403	mask \|= (gpa & shadow_nonpresent_or_rsvd_mask)
@@ -401,11 +409,6 @@ static void mark_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, u64 gfn, · 5 unmodified lines
401	409	mmu_spte_set(sptep, mask);
402	410	}
403	411
404		−static bool is_mmio_spte(u64 spte)
405		−{
406		− return (spte & shadow_mmio_mask) == shadow_mmio_value;
407		−}
408		−
409	412	static gfn_t get_mmio_spte_gfn(u64 spte)
410	413	{
411	414	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
@@ -418,8 +421,7 @@ static gfn_t get_mmio_spte_gfn(u64 spte) · 6 unmodified lines
418	421
419	422	static unsigned get_mmio_spte_access(u64 spte)
420	423	{
421		− u64 mask = generation_mmio_spte_mask(MMIO_SPTE_GEN_MASK) \| shadow_mmio_mask;
422		− return (spte & ~mask) & ~PAGE_MASK;
	424	+ return spte & shadow_mmio_access_mask;
423	425	}
424	426
425	427	static bool set_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, gfn_t gfn,
@@ -3302,7 +3304,8 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, · 2876 unmodified lines
3302	3304	}
3303	3305
3304	3306	if (unlikely(is_noslot_pfn(pfn)))
3305		− vcpu_cache_mmio_info(vcpu, gva, gfn, access);
	3307	+ vcpu_cache_mmio_info(vcpu, gva, gfn,
	3308	+ access & shadow_mmio_access_mask);
3306	3309
3307	3310	return false;
3308	3311	}
@@ -5611,13 +5614,13 @@ slot_handle_leaf(struct kvm kvm, struct kvm_memory_slot memslot, · 2302 unmodified lines
5611	5614	PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
5612	5615	}
5613	5616
5614		−static void free_mmu_pages(struct kvm_vcpu *vcpu)
	5617	+static void free_mmu_pages(struct kvm_mmu *mmu)
5615	5618	{
5616		− free_page((unsigned long)vcpu->arch.mmu->pae_root);
5617		− free_page((unsigned long)vcpu->arch.mmu->lm_root);
	5619	+ free_page((unsigned long)mmu->pae_root);
	5620	+ free_page((unsigned long)mmu->lm_root);
5618	5621	}
5619	5622
5620		−static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
	5623	+static int alloc_mmu_pages(struct kvm_vcpu vcpu, struct kvm_mmu mmu)
5621	5624	{
5622	5625	struct page *page;
5623	5626	int i;
@@ -5638,9 +5641,9 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) · 14 unmodified lines
5638	5641	if (!page)
5639	5642	return -ENOMEM;
5640	5643
5641		− vcpu->arch.mmu->pae_root = page_address(page);
	5644	+ mmu->pae_root = page_address(page);
5642	5645	for (i = 0; i < 4; ++i)
5643		− vcpu->arch.mmu->pae_root[i] = INVALID_PAGE;
	5646	+ mmu->pae_root[i] = INVALID_PAGE;
5644	5647
5645	5648	return 0;
5646	5649	}
@@ -5648,6 +5651,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) · 1 unmodified line
5648	5651	int kvm_mmu_create(struct kvm_vcpu *vcpu)
5649	5652	{
5650	5653	uint i;
	5654	+ int ret;
5651	5655
5652	5656	vcpu->arch.mmu = &vcpu->arch.root_mmu;
5653	5657	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
@@ -5665,7 +5669,19 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) · 11 unmodified lines
5665	5669	vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5666	5670
5667	5671	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
5668		− return alloc_mmu_pages(vcpu);
	5672	+
	5673	+ ret = alloc_mmu_pages(vcpu, &vcpu->arch.guest_mmu);
	5674	+ if (ret)
	5675	+ return ret;
	5676	+
	5677	+ ret = alloc_mmu_pages(vcpu, &vcpu->arch.root_mmu);
	5678	+ if (ret)
	5679	+ goto fail_allocate_root;
	5680	+
	5681	+ return ret;
	5682	+ fail_allocate_root:
	5683	+ free_mmu_pages(&vcpu->arch.guest_mmu);
	5684	+ return ret;
5669	5685	}
5670	5686
5671	5687
@@ -6094,7 +6110,7 @@ static void kvm_set_mmio_spte_mask(void) · 422 unmodified lines
6094	6110	if (IS_ENABLED(CONFIG_X86_64) && shadow_phys_bits == 52)
6095	6111	mask &= ~1ull;
6096	6112
6097		− kvm_mmu_set_mmio_spte_mask(mask, mask);
	6113	+ kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK \| ACC_USER_MASK);
6098	6114	}
6099	6115
6100	6116	int kvm_mmu_module_init(void)
@@ -6168,7 +6184,8 @@ unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) · 67 unmodified lines
6168	6184	void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
6169	6185	{
6170	6186	kvm_mmu_unload(vcpu);
6171		− free_mmu_pages(vcpu);
	6187	+ free_mmu_pages(&vcpu->arch.root_mmu);
	6188	+ free_mmu_pages(&vcpu->arch.guest_mmu);
6172	6189	mmu_free_memory_caches(vcpu);
6173	6190	}
6174	6191

arch/x86/kvm/mmu.h+1−1

@@ -51,7 +51,7 @@ static inline u64 rsvd_bits(int s, int e) · 50 unmodified lines
51	51	return ((1ULL << (e - s + 1)) - 1) << s;
52	52	}
53	53
54		−void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value);
	54	+void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask);
55	55
56	56	void
57	57	reset_shadow_zero_bits_mask(struct kvm_vcpu vcpu, struct kvm_mmu context);

arch/x86/kvm/svm.c+93−105

@@ -68,10 +68,8 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); · 67 unmodified lines
68	68	#define SEG_TYPE_LDT 2
69	69	#define SEG_TYPE_BUSY_TSS16 3
70	70
71		−#define SVM_FEATURE_NPT (1 << 0)
72	71	#define SVM_FEATURE_LBRV (1 << 1)
73	72	#define SVM_FEATURE_SVML (1 << 2)
74		−#define SVM_FEATURE_NRIP (1 << 3)
75	73	#define SVM_FEATURE_TSC_RATE (1 << 4)
76	74	#define SVM_FEATURE_VMCB_CLEAN (1 << 5)
77	75	#define SVM_FEATURE_FLUSH_ASID (1 << 6)
@@ -770,7 +768,7 @@ static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) · 692 unmodified lines
770	768
771	769	}
772	770
773		−static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
	771	+static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
774	772	{
775	773	struct vcpu_svm *svm = to_svm(vcpu);
776	774
@@ -779,18 +777,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) · 2 unmodified lines
779	777	svm->next_rip = svm->vmcb->control.next_rip;
780	778	}
781	779
782		− if (!svm->next_rip) {
783		− if (kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) !=
784		− EMULATE_DONE)
785		− printk(KERN_DEBUG "%s: NOP\n", __func__);
786		− return;
787		− }
	780	+ if (!svm->next_rip)
	781	+ return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP);
	782	+
788	783	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
789	784	printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
790	785	__func__, kvm_rip_read(vcpu), svm->next_rip);
791	786
792	787	kvm_rip_write(vcpu, svm->next_rip);
793	788	svm_set_interrupt_shadow(vcpu, 0);
	789	+
	790	+ return EMULATE_DONE;
794	791	}
795	792
796	793	static void svm_queue_exception(struct kvm_vcpu *vcpu)
@@ -821,7 +818,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu) · 24 unmodified lines
821	818	* raises a fault that is not intercepted. Still better than
822	819	* failing in all cases.
823	820	*/
824		− skip_emulated_instruction(&svm->vcpu);
	821	+ (void)skip_emulated_instruction(&svm->vcpu);
825	822	rip = kvm_rip_read(&svm->vcpu);
826	823	svm->int3_rip = rip + svm->vmcb->save.cs.base;
827	824	svm->int3_injected = rip - old_rip;
@@ -1269,11 +1266,11 @@ static void grow_ple_window(struct kvm_vcpu *vcpu) · 441 unmodified lines
1269	1266	pause_filter_count_grow,
1270	1267	pause_filter_count_max);
1271	1268
1272		− if (control->pause_filter_count != old)
	1269	+ if (control->pause_filter_count != old) {
1273	1270	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1274		−
1275		− trace_kvm_ple_window_grow(vcpu->vcpu_id,
1276		− control->pause_filter_count, old);
	1271	+ trace_kvm_ple_window_update(vcpu->vcpu_id,
	1272	+ control->pause_filter_count, old);
	1273	+ }
1277	1274	}
1278	1275
1279	1276	static void shrink_ple_window(struct kvm_vcpu *vcpu)
@@ -1287,11 +1284,11 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu) · 7 unmodified lines
1287	1284	pause_filter_count,
1288	1285	pause_filter_count_shrink,
1289	1286	pause_filter_count);
1290		− if (control->pause_filter_count != old)
	1287	+ if (control->pause_filter_count != old) {
1291	1288	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1292		−
1293		− trace_kvm_ple_window_shrink(vcpu->vcpu_id,
1294		− control->pause_filter_count, old);
	1289	+ trace_kvm_ple_window_update(vcpu->vcpu_id,
	1290	+ control->pause_filter_count, old);
	1291	+ }
1295	1292	}
1296	1293
1297	1294	static __init int svm_hardware_setup(void)
@@ -2136,6 +2133,9 @@ static struct kvm_vcpu svm_create_vcpu(struct kvm kvm, unsigned int id) · 838 unmodified lines
2136	2133	struct page *nested_msrpm_pages;
2137	2134	int err;
2138	2135
	2136	+ BUILD_BUG_ON_MSG(offsetof(struct vcpu_svm, vcpu) != 0,
	2137	+ "struct kvm_vcpu must be at offset 0 for arch usercopy region");
	2138	+
2139	2139	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
2140	2140	if (!svm) {
2141	2141	err = -ENOMEM;
@@ -2903,13 +2903,11 @@ static int nop_on_interception(struct vcpu_svm *svm) · 761 unmodified lines
2903	2903
2904	2904	static int halt_interception(struct vcpu_svm *svm)
2905	2905	{
2906		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2907	2906	return kvm_emulate_halt(&svm->vcpu);
2908	2907	}
2909	2908
2910	2909	static int vmmcall_interception(struct vcpu_svm *svm)
2911	2910	{
2912		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2913	2911	return kvm_emulate_hypercall(&svm->vcpu);
2914	2912	}
2915	2913
@@ -3588,9 +3586,9 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, · 672 unmodified lines
3588	3586	mark_all_dirty(svm->vmcb);
3589	3587	}
3590	3588
3591		−static bool nested_svm_vmrun(struct vcpu_svm *svm)
	3589	+static int nested_svm_vmrun(struct vcpu_svm *svm)
3592	3590	{
3593		− int rc;
	3591	+ int ret;
3594	3592	struct vmcb *nested_vmcb;
3595	3593	struct vmcb *hsave = svm->nested.hsave;
3596	3594	struct vmcb *vmcb = svm->vmcb;
@@ -3599,13 +3597,16 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 2 unmodified lines
3599	3597
3600	3598	vmcb_gpa = svm->vmcb->save.rax;
3601	3599
3602		− rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
3603		− if (rc) {
3604		− if (rc == -EINVAL)
3605		− kvm_inject_gp(&svm->vcpu, 0);
3606		− return false;
	3600	+ ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
	3601	+ if (ret == -EINVAL) {
	3602	+ kvm_inject_gp(&svm->vcpu, 0);
	3603	+ return 1;
	3604	+ } else if (ret) {
	3605	+ return kvm_skip_emulated_instruction(&svm->vcpu);
3607	3606	}
3608	3607
	3608	+ ret = kvm_skip_emulated_instruction(&svm->vcpu);
	3609	+
3609	3610	nested_vmcb = map.hva;
3610	3611
3611	3612	if (!nested_vmcb_checks(nested_vmcb)) {
@@ -3616,7 +3617,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 4 unmodified lines
3616	3617
3617	3618	kvm_vcpu_unmap(&svm->vcpu, &map, true);
3618	3619
3619		− return false;
	3620	+ return ret;
3620	3621	}
3621	3622
3622	3623	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
@@ -3660,7 +3661,16 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 37 unmodified lines
3660	3661
3661	3662	enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
3662	3663
3663		− return true;
	3664	+ if (!nested_svm_vmrun_msrpm(svm)) {
	3665	+ svm->vmcb->control.exit_code = SVM_EXIT_ERR;
	3666	+ svm->vmcb->control.exit_code_hi = 0;
	3667	+ svm->vmcb->control.exit_info_1 = 0;
	3668	+ svm->vmcb->control.exit_info_2 = 0;
	3669	+
	3670	+ nested_svm_vmexit(svm);
	3671	+ }
	3672	+
	3673	+ return ret;
3664	3674	}
3665	3675
3666	3676	static void nested_svm_vmloadsave(struct vmcb from_vmcb, struct vmcb to_vmcb)
@@ -3697,7 +3707,6 @@ static int vmload_interception(struct vcpu_svm *svm) · 30 unmodified lines
3697	3707
3698	3708	nested_vmcb = map.hva;
3699	3709
3700		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3701	3710	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3702	3711
3703	3712	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
@@ -3724,7 +3733,6 @@ static int vmsave_interception(struct vcpu_svm *svm) · 20 unmodified lines
3724	3733
3725	3734	nested_vmcb = map.hva;
3726	3735
3727		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3728	3736	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3729	3737
3730	3738	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
@@ -3738,27 +3746,7 @@ static int vmrun_interception(struct vcpu_svm *svm) · 7 unmodified lines
3738	3746	if (nested_svm_check_permissions(svm))
3739	3747	return 1;
3740	3748
3741		− /* Save rip after vmrun instruction */
3742		− kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3743		−
3744		− if (!nested_svm_vmrun(svm))
3745		− return 1;
3746		−
3747		− if (!nested_svm_vmrun_msrpm(svm))
3748		− goto failed;
3749		−
3750		− return 1;
3751		−
3752		−failed:
3753		−
3754		− svm->vmcb->control.exit_code = SVM_EXIT_ERR;
3755		− svm->vmcb->control.exit_code_hi = 0;
3756		− svm->vmcb->control.exit_info_1 = 0;
3757		− svm->vmcb->control.exit_info_2 = 0;
3758		−
3759		− nested_svm_vmexit(svm);
3760		−
3761		− return 1;
	3749	+ return nested_svm_vmrun(svm);
3762	3750	}
3763	3751
3764	3752	static int stgi_interception(struct vcpu_svm *svm)
@@ -3775,7 +3763,6 @@ static int stgi_interception(struct vcpu_svm *svm) · 10 unmodified lines
3775	3763	if (vgif_enabled(svm))
3776	3764	clr_intercept(svm, INTERCEPT_STGI);
3777	3765
3778		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3779	3766	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3780	3767	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3781	3768
@@ -3791,7 +3778,6 @@ static int clgi_interception(struct vcpu_svm *svm) · 9 unmodified lines
3791	3778	if (nested_svm_check_permissions(svm))
3792	3779	return 1;
3793	3780
3794		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3795	3781	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3796	3782
3797	3783	disable_gif(svm);
@@ -3816,7 +3802,6 @@ static int invlpga_interception(struct vcpu_svm *svm) · 18 unmodified lines
3816	3802	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3817	3803	kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
3818	3804
3819		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3820	3805	return kvm_skip_emulated_instruction(&svm->vcpu);
3821	3806	}
3822	3807
@@ -3839,7 +3824,6 @@ static int xsetbv_interception(struct vcpu_svm *svm) · 16 unmodified lines
3839	3824	u32 index = kvm_rcx_read(&svm->vcpu);
3840	3825
3841	3826	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3842		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3843	3827	return kvm_skip_emulated_instruction(&svm->vcpu);
3844	3828	}
3845	3829
@@ -3898,25 +3882,29 @@ static int task_switch_interception(struct vcpu_svm *svm) · 52 unmodified lines
3898	3882	if (reason != TASK_SWITCH_GATE \|\|
3899	3883	int_type == SVM_EXITINTINFO_TYPE_SOFT \|\|
3900	3884	(int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3901		− (int_vec == OF_VECTOR \|\| int_vec == BP_VECTOR)))
3902		− skip_emulated_instruction(&svm->vcpu);
	3885	+ (int_vec == OF_VECTOR \|\| int_vec == BP_VECTOR))) {
	3886	+ if (skip_emulated_instruction(&svm->vcpu) != EMULATE_DONE)
	3887	+ goto fail;
	3888	+ }
3903	3889
3904	3890	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3905	3891	int_vec = -1;
3906	3892
3907	3893	if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3908		− has_error_code, error_code) == EMULATE_FAIL) {
3909		− svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
3910		− svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
3911		− svm->vcpu.run->internal.ndata = 0;
3912		− return 0;
3913		− }
	3894	+ has_error_code, error_code) == EMULATE_FAIL)
	3895	+ goto fail;
	3896	+
3914	3897	return 1;
	3898	+
	3899	+fail:
	3900	+ svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
	3901	+ svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
	3902	+ svm->vcpu.run->internal.ndata = 0;
	3903	+ return 0;
3915	3904	}
3916	3905
3917	3906	static int cpuid_interception(struct vcpu_svm *svm)
3918	3907	{
3919		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3920	3908	return kvm_emulate_cpuid(&svm->vcpu);
3921	3909	}
3922	3910
@@ -4232,23 +4220,7 @@ static int svm_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 309 unmodified lines
4232	4220
4233	4221	static int rdmsr_interception(struct vcpu_svm *svm)
4234	4222	{
4235		− u32 ecx = kvm_rcx_read(&svm->vcpu);
4236		− struct msr_data msr_info;
4237		−
4238		− msr_info.index = ecx;
4239		− msr_info.host_initiated = false;
4240		− if (svm_get_msr(&svm->vcpu, &msr_info)) {
4241		− trace_kvm_msr_read_ex(ecx);
4242		− kvm_inject_gp(&svm->vcpu, 0);
4243		− return 1;
4244		− } else {
4245		− trace_kvm_msr_read(ecx, msr_info.data);
4246		−
4247		− kvm_rax_write(&svm->vcpu, msr_info.data & 0xffffffff);
4248		− kvm_rdx_write(&svm->vcpu, msr_info.data >> 32);
4249		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4250		− return kvm_skip_emulated_instruction(&svm->vcpu);
4251		− }
	4223	+ return kvm_emulate_rdmsr(&svm->vcpu);
4252	4224	}
4253	4225
4254	4226	static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
@@ -4438,23 +4410,7 @@ static int svm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) · 183 unmodified lines
4438	4410
4439	4411	static int wrmsr_interception(struct vcpu_svm *svm)
4440	4412	{
4441		− struct msr_data msr;
4442		− u32 ecx = kvm_rcx_read(&svm->vcpu);
4443		− u64 data = kvm_read_edx_eax(&svm->vcpu);
4444		−
4445		− msr.data = data;
4446		− msr.index = ecx;
4447		− msr.host_initiated = false;
4448		−
4449		− svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4450		− if (kvm_set_msr(&svm->vcpu, &msr)) {
4451		− trace_kvm_msr_write_ex(ecx, data);
4452		− kvm_inject_gp(&svm->vcpu, 0);
4453		− return 1;
4454		− } else {
4455		− trace_kvm_msr_write(ecx, data);
4456		− return kvm_skip_emulated_instruction(&svm->vcpu);
4457		− }
	4413	+ return kvm_emulate_wrmsr(&svm->vcpu);
4458	4414	}
4459	4415
4460	4416	static int msr_interception(struct vcpu_svm *svm)
@@ -5025,9 +4981,14 @@ static int handle_exit(struct kvm_vcpu *vcpu) · 564 unmodified lines
5025	4981
5026	4982	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
5027	4983	\|\| !svm_exit_handlers[exit_code]) {
5028		− WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
5029		− kvm_queue_exception(vcpu, UD_VECTOR);
5030		− return 1;
	4984	+ vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
	4985	+ dump_vmcb(vcpu);
	4986	+ vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
	4987	+ vcpu->run->internal.suberror =
	4988	+ KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
	4989	+ vcpu->run->internal.ndata = 1;
	4990	+ vcpu->run->internal.data[0] = exit_code;
	4991	+ return 0;
5031	4992	}
5032	4993
5033	4994	return svm_exit_handlers[exit_code](svm);
@@ -5274,7 +5235,8 @@ get_pi_vcpu_info(struct kvm kvm, struct kvm_kernel_irq_routing_entry e, · 240 unmodified lines
5274	5235
5275	5236	kvm_set_msi_irq(kvm, e, &irq);
5276	5237
5277		− if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
	5238	+ if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) \|\|
	5239	+ !kvm_irq_is_postable(&irq)) {
5278	5240	pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5279	5241	__func__, irq.vector);
5280	5242	return -1;
@@ -5328,6 +5290,7 @@ static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, · 47 unmodified lines
5328	5290	* 1. When cannot target interrupt to a specific vcpu.
5329	5291	* 2. Unsetting posted interrupt.
5330	5292	* 3. APIC virtialization is disabled for the vcpu.
	5293	+ * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
5331	5294	*/
5332	5295	if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5333	5296	kvm_vcpu_apicv_active(&svm->vcpu)) {
@@ -5933,6 +5896,8 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) · 599 unmodified lines
5933	5896	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5934	5897	}
5935	5898
	5899	+#define F(x) bit(X86_FEATURE_##x)
	5900	+
5936	5901	static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5937	5902	{
5938	5903	switch (func) {
@@ -5944,6 +5909,11 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) · 5 unmodified lines
5944	5909	if (nested)
5945	5910	entry->ecx \|= (1 << 2); /* Set SVM bit */
5946	5911	break;
	5912	+ case 0x80000008:
	5913	+ if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) \|\|
	5914	+ boot_cpu_has(X86_FEATURE_AMD_SSBD))
	5915	+ entry->ebx \|= F(VIRT_SSBD);
	5916	+ break;
5947	5917	case 0x8000000A:
5948	5918	entry->eax = 1; /* SVM revision 1 */
5949	5919	entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
@@ -5954,11 +5924,11 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) · 4 unmodified lines
5954	5924
5955	5925	/* Support next_rip if host supports it */
5956	5926	if (boot_cpu_has(X86_FEATURE_NRIPS))
5957		− entry->edx \|= SVM_FEATURE_NRIP;
	5927	+ entry->edx \|= F(NRIPS);
5958	5928
5959	5929	/* Support NPT for the guest if enabled */
5960	5930	if (npt_enabled)
5961		− entry->edx \|= SVM_FEATURE_NPT;
	5931	+ entry->edx \|= F(NPT);
5962	5932
5963	5933	break;
5964	5934	case 0x8000001F:
@@ -6067,6 +6037,7 @@ static const struct __x86_intercept { · 102 unmodified lines
6067	6037	[x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
6068	6038	[x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
6069	6039	[x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
	6040	+ [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV),
6070	6041	};
6071	6042
6072	6043	#undef PRE_EX
@@ -7193,6 +7164,21 @@ static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) · 1120 unmodified lines
7193	7164	return false;
7194	7165	}
7195	7166
	7167	+static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
	7168	+{
	7169	+ struct vcpu_svm *svm = to_svm(vcpu);
	7170	+
	7171	+ /*
	7172	+ * TODO: Last condition latch INIT signals on vCPU when
	7173	+ * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
	7174	+ * To properly emulate the INIT intercept, SVM should implement
	7175	+ * kvm_x86_ops->check_nested_events() and call nested_svm_vmexit()
	7176	+ * there if an INIT signal is pending.
	7177	+ */
	7178	+ return !gif_set(svm) \|\|
	7179	+ (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
	7180	+}
	7181	+
7196	7182	static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7197	7183	.cpu_has_kvm_support = has_svm,
7198	7184	.disabled_by_bios = is_disabled,
@@ -7329,6 +7315,8 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · 130 unmodified lines
7329	7315	.nested_get_evmcs_version = NULL,
7330	7316
7331	7317	.need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
	7318	+
	7319	+ .apic_init_signal_blocked = svm_apic_init_signal_blocked,
7332	7320	};
7333	7321
7334	7322	static int __init svm_init(void)

arch/x86/kvm/trace.h+54−20

@@ -232,17 +232,20 @@ TRACE_EVENT(kvm_exit, · 231 unmodified lines
232	232	__field( u32, isa )
233	233	__field( u64, info1 )
234	234	__field( u64, info2 )
	235	+ __field( unsigned int, vcpu_id )
235	236	),
236	237
237	238	TP_fast_assign(
238	239	__entry->exit_reason = exit_reason;
239	240	__entry->guest_rip = kvm_rip_read(vcpu);
240	241	__entry->isa = isa;
	242	+ __entry->vcpu_id = vcpu->vcpu_id;
241	243	kvm_x86_ops->get_exit_info(vcpu, &__entry->info1,
242	244	&__entry->info2);
243	245	),
244	246
245		− TP_printk("reason %s rip 0x%lx info %llx %llx",
	247	+ TP_printk("vcpu %u reason %s rip 0x%lx info %llx %llx",
	248	+ __entry->vcpu_id,
246	249	(__entry->isa == KVM_ISA_VMX) ?
247	250	__print_symbolic(__entry->exit_reason, VMX_EXIT_REASONS) :
248	251	__print_symbolic(__entry->exit_reason, SVM_EXIT_REASONS),
@@ -887,36 +890,27 @@ TRACE_EVENT(kvm_pml_full, · 638 unmodified lines
887	890	TP_printk("vcpu %d: PML full", __entry->vcpu_id)
888	891	);
889	892
890		−TRACE_EVENT(kvm_ple_window,
891		− TP_PROTO(bool grow, unsigned int vcpu_id, int new, int old),
892		− TP_ARGS(grow, vcpu_id, new, old),
	893	+TRACE_EVENT(kvm_ple_window_update,
	894	+ TP_PROTO(unsigned int vcpu_id, unsigned int new, unsigned int old),
	895	+ TP_ARGS(vcpu_id, new, old),
893	896
894	897	TP_STRUCT__entry(
895		− __field( bool, grow )
896	898	__field( unsigned int, vcpu_id )
897		− __field( int, new )
898		− __field( int, old )
	899	+ __field( unsigned int, new )
	900	+ __field( unsigned int, old )
899	901	),
900	902
901	903	TP_fast_assign(
902		− __entry->grow = grow;
903	904	__entry->vcpu_id = vcpu_id;
904	905	__entry->new = new;
905	906	__entry->old = old;
906	907	),
907	908
908		− TP_printk("vcpu %u: ple_window %d (%s %d)",
909		− __entry->vcpu_id,
910		− __entry->new,
911		− __entry->grow ? "grow" : "shrink",
912		− __entry->old)
	909	+ TP_printk("vcpu %u old %u new %u (%s)",
	910	+ __entry->vcpu_id, __entry->old, __entry->new,
	911	+ __entry->old < __entry->new ? "growed" : "shrinked")
913	912	);
914	913
915		−#define trace_kvm_ple_window_grow(vcpu_id, new, old) \
916		− trace_kvm_ple_window(true, vcpu_id, new, old)
917		−#define trace_kvm_ple_window_shrink(vcpu_id, new, old) \
918		− trace_kvm_ple_window(false, vcpu_id, new, old)
919		−
920	914	TRACE_EVENT(kvm_pvclock_update,
921	915	TP_PROTO(unsigned int vcpu_id, struct pvclock_vcpu_time_info *pvclock),
922	916	TP_ARGS(vcpu_id, pvclock),
@@ -1320,7 +1314,7 @@ TRACE_EVENT(kvm_avic_incomplete_ipi, · 397 unmodified lines
1320	1314	__entry->index = index;
1321	1315	),
1322	1316
1323		− TP_printk("vcpu=%u, icrh:icrl=%#010x:%08x, id=%u, index=%u\n",
	1317	+ TP_printk("vcpu=%u, icrh:icrl=%#010x:%08x, id=%u, index=%u",
1324	1318	__entry->vcpu, __entry->icrh, __entry->icrl,
1325	1319	__entry->id, __entry->index)
1326	1320	);
@@ -1345,7 +1339,7 @@ TRACE_EVENT(kvm_avic_unaccelerated_access, · 18 unmodified lines
1345	1339	__entry->vec = vec;
1346	1340	),
1347	1341
1348		− TP_printk("vcpu=%u, offset=%#x(%s), %s, %s, vec=%#x\n",
	1342	+ TP_printk("vcpu=%u, offset=%#x(%s), %s, %s, vec=%#x",
1349	1343	__entry->vcpu,
1350	1344	__entry->offset,
1351	1345	__print_symbolic(__entry->offset, kvm_trace_symbol_apic),
@@ -1462,6 +1456,46 @@ TRACE_EVENT(kvm_hv_send_ipi_ex, · 110 unmodified lines
1462	1456	__entry->vector, __entry->format,
1463	1457	__entry->valid_bank_mask)
1464	1458	);
	1459	+
	1460	+TRACE_EVENT(kvm_pv_tlb_flush,
	1461	+ TP_PROTO(unsigned int vcpu_id, bool need_flush_tlb),
	1462	+ TP_ARGS(vcpu_id, need_flush_tlb),
	1463	+
	1464	+ TP_STRUCT__entry(
	1465	+ __field( unsigned int, vcpu_id )
	1466	+ __field( bool, need_flush_tlb )
	1467	+ ),
	1468	+
	1469	+ TP_fast_assign(
	1470	+ __entry->vcpu_id = vcpu_id;
	1471	+ __entry->need_flush_tlb = need_flush_tlb;
	1472	+ ),
	1473	+
	1474	+ TP_printk("vcpu %u need_flush_tlb %s", __entry->vcpu_id,
	1475	+ __entry->need_flush_tlb ? "true" : "false")
	1476	+);
	1477	+
	1478	+/*
	1479	+ * Tracepoint for failed nested VMX VM-Enter.
	1480	+ */
	1481	+TRACE_EVENT(kvm_nested_vmenter_failed,
	1482	+ TP_PROTO(const char *msg, u32 err),
	1483	+ TP_ARGS(msg, err),
	1484	+
	1485	+ TP_STRUCT__entry(
	1486	+ __field(const char *, msg)
	1487	+ __field(u32, err)
	1488	+ ),
	1489	+
	1490	+ TP_fast_assign(
	1491	+ __entry->msg = msg;
	1492	+ __entry->err = err;
	1493	+ ),
	1494	+
	1495	+ TP_printk("%s%s", __entry->msg, !__entry->err ? "" :
	1496	+ __print_symbolic(__entry->err, VMX_VMENTER_INSTRUCTION_ERRORS))
	1497	+);
	1498	+
1465	1499	#endif /* _TRACE_KVM_H */
1466	1500
1467	1501	#undef TRACE_INCLUDE_PATH

arch/x86/kvm/vmx/nested.c+162−143

@@ -19,6 +19,14 @@ module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); · 18 unmodified lines
19	19	static bool __read_mostly nested_early_check = 0;
20	20	module_param(nested_early_check, bool, S_IRUGO);
21	21
	22	+#define CC(consistency_check) \
	23	+({ \
	24	+ bool failed = (consistency_check); \
	25	+ if (failed) \
	26	+ trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
	27	+ failed; \
	28	+})
	29	+
22	30	/*
23	31	* Hyper-V requires all of these, so mark them as supported even though
24	32	* they are just treated the same as all-context.
@@ -430,8 +438,8 @@ static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, · 405 unmodified lines
430	438	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
431	439	return 0;
432	440
433		− if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) \|\|
434		− !page_address_valid(vcpu, vmcs12->io_bitmap_b))
	441	+ if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) \|\|
	442	+ CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
435	443	return -EINVAL;
436	444
437	445	return 0;
@@ -443,7 +451,7 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
443	451	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
444	452	return 0;
445	453
446		− if (!page_address_valid(vcpu, vmcs12->msr_bitmap))
	454	+ if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
447	455	return -EINVAL;
448	456
449	457	return 0;
@@ -455,7 +463,7 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
455	463	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
456	464	return 0;
457	465
458		− if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))
	466	+ if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
459	467	return -EINVAL;
460	468
461	469	return 0;
@@ -688,7 +696,7 @@ static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, · 226 unmodified lines
688	696	struct vmcs12 *vmcs12)
689	697	{
690	698	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
691		− !page_address_valid(vcpu, vmcs12->apic_access_addr))
	699	+ CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
692	700	return -EINVAL;
693	701	else
694	702	return 0;
@@ -707,16 +715,15 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 12 unmodified lines
707	715	* If virtualize x2apic mode is enabled,
708	716	* virtualize apic access must be disabled.
709	717	*/
710		− if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
711		− nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
	718	+ if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
	719	+ nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
712	720	return -EINVAL;
713	721
714	722	/*
715	723	* If virtual interrupt delivery is enabled,
716	724	* we must exit on external interrupts.
717	725	*/
718		− if (nested_cpu_has_vid(vmcs12) &&
719		− !nested_exit_on_intr(vcpu))
	726	+ if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
720	727	return -EINVAL;
721	728
722	729	/*
@@ -727,15 +734,15 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
727	734	* bits 5:0 of posted_intr_desc_addr should be zero.
728	735	*/
729	736	if (nested_cpu_has_posted_intr(vmcs12) &&
730		− (!nested_cpu_has_vid(vmcs12) \|\|
731		− !nested_exit_intr_ack_set(vcpu) \|\|
732		− (vmcs12->posted_intr_nv & 0xff00) \|\|
733		− (vmcs12->posted_intr_desc_addr & 0x3f) \|\|
734		− (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu))))
	737	+ (CC(!nested_cpu_has_vid(vmcs12)) \|\|
	738	+ CC(!nested_exit_intr_ack_set(vcpu)) \|\|
	739	+ CC((vmcs12->posted_intr_nv & 0xff00)) \|\|
	740	+ CC((vmcs12->posted_intr_desc_addr & 0x3f)) \|\|
	741	+ CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
735	742	return -EINVAL;
736	743
737	744	/* tpr shadow is needed by all apicv features. */
738		− if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
	745	+ if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
739	746	return -EINVAL;
740	747
741	748	return 0;
@@ -759,10 +766,12 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, · 17 unmodified lines
759	766	static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
760	767	struct vmcs12 *vmcs12)
761	768	{
762		− if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count,
763		− vmcs12->vm_exit_msr_load_addr) \|\|
764		− nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count,
765		− vmcs12->vm_exit_msr_store_addr))
	769	+ if (CC(nested_vmx_check_msr_switch(vcpu,
	770	+ vmcs12->vm_exit_msr_load_count,
	771	+ vmcs12->vm_exit_msr_load_addr)) \|\|
	772	+ CC(nested_vmx_check_msr_switch(vcpu,
	773	+ vmcs12->vm_exit_msr_store_count,
	774	+ vmcs12->vm_exit_msr_store_addr)))
766	775	return -EINVAL;
767	776
768	777	return 0;
@@ -771,8 +780,9 @@ static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, · 2 unmodified lines
771	780	static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
772	781	struct vmcs12 *vmcs12)
773	782	{
774		− if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count,
775		− vmcs12->vm_entry_msr_load_addr))
	783	+ if (CC(nested_vmx_check_msr_switch(vcpu,
	784	+ vmcs12->vm_entry_msr_load_count,
	785	+ vmcs12->vm_entry_msr_load_addr)))
776	786	return -EINVAL;
777	787
778	788	return 0;
@@ -784,8 +794,8 @@ static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
784	794	if (!nested_cpu_has_pml(vmcs12))
785	795	return 0;
786	796
787		− if (!nested_cpu_has_ept(vmcs12) \|\|
788		− !page_address_valid(vcpu, vmcs12->pml_address))
	797	+ if (CC(!nested_cpu_has_ept(vmcs12)) \|\|
	798	+ CC(!page_address_valid(vcpu, vmcs12->pml_address)))
789	799	return -EINVAL;
790	800
791	801	return 0;
@@ -794,8 +804,8 @@ static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, · 2 unmodified lines
794	804	static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
795	805	struct vmcs12 *vmcs12)
796	806	{
797		− if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
798		− !nested_cpu_has_ept(vmcs12))
	807	+ if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
	808	+ !nested_cpu_has_ept(vmcs12)))
799	809	return -EINVAL;
800	810	return 0;
801	811	}
@@ -803,8 +813,8 @@ static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
803	813	static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
804	814	struct vmcs12 *vmcs12)
805	815	{
806		− if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
807		− !nested_cpu_has_ept(vmcs12))
	816	+ if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
	817	+ !nested_cpu_has_ept(vmcs12)))
808	818	return -EINVAL;
809	819	return 0;
810	820	}
@@ -815,8 +825,8 @@ static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
815	825	if (!nested_cpu_has_shadow_vmcs(vmcs12))
816	826	return 0;
817	827
818		− if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) \|\|
819		− !page_address_valid(vcpu, vmcs12->vmwrite_bitmap))
	828	+ if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) \|\|
	829	+ CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
820	830	return -EINVAL;
821	831
822	832	return 0;
@@ -826,12 +836,12 @@ static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, · 3 unmodified lines
826	836	struct vmx_msr_entry *e)
827	837	{
828	838	/* x2APIC MSR accesses are not allowed */
829		− if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
	839	+ if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
830	840	return -EINVAL;
831		− if (e->index == MSR_IA32_UCODE_WRITE \|\| /* SDM Table 35-2 */
832		− e->index == MSR_IA32_UCODE_REV)
	841	+ if (CC(e->index == MSR_IA32_UCODE_WRITE) \|\| /* SDM Table 35-2 */
	842	+ CC(e->index == MSR_IA32_UCODE_REV))
833	843	return -EINVAL;
834		− if (e->reserved != 0)
	844	+ if (CC(e->reserved != 0))
835	845	return -EINVAL;
836	846	return 0;
837	847	}
@@ -839,9 +849,9 @@ static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, · 1 unmodified line
839	849	static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
840	850	struct vmx_msr_entry *e)
841	851	{
842		− if (e->index == MSR_FS_BASE \|\|
843		− e->index == MSR_GS_BASE \|\|
844		− e->index == MSR_IA32_SMM_MONITOR_CTL \|\| /* SMM is not supported */
	852	+ if (CC(e->index == MSR_FS_BASE) \|\|
	853	+ CC(e->index == MSR_GS_BASE) \|\|
	854	+ CC(e->index == MSR_IA32_SMM_MONITOR_CTL) \|\| /* SMM is not supported */
845	855	nested_vmx_msr_check_common(vcpu, e))
846	856	return -EINVAL;
847	857	return 0;
@@ -850,7 +860,7 @@ static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, · 2 unmodified lines
850	860	static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
851	861	struct vmx_msr_entry *e)
852	862	{
853		− if (e->index == MSR_IA32_SMBASE \|\| /* SMM is not supported */
	863	+ if (CC(e->index == MSR_IA32_SMBASE) \|\| /* SMM is not supported */
854	864	nested_vmx_msr_check_common(vcpu, e))
855	865	return -EINVAL;
856	866	return 0;
@@ -864,9 +874,7 @@ static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 7 unmodified lines
864	874	{
865	875	u32 i;
866	876	struct vmx_msr_entry e;
867		− struct msr_data msr;
868	877
869		− msr.host_initiated = false;
870	878	for (i = 0; i < count; i++) {
871	879	if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
872	880	&e, sizeof(e))) {
@@ -881,9 +889,7 @@ static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 8 unmodified lines
881	889	__func__, i, e.index, e.reserved);
882	890	goto fail;
883	891	}
884		− msr.index = e.index;
885		− msr.data = e.value;
886		− if (kvm_set_msr(vcpu, &msr)) {
	892	+ if (kvm_set_msr(vcpu, e.index, e.value)) {
887	893	pr_debug_ratelimited(
888	894	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
889	895	__func__, i, e.index, e.value);
@@ -897,11 +903,11 @@ fail: · 7 unmodified lines
897	903
898	904	static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
899	905	{
	906	+ u64 data;
900	907	u32 i;
901	908	struct vmx_msr_entry e;
902	909
903	910	for (i = 0; i < count; i++) {
904		− struct msr_data msr_info;
905	911	if (kvm_vcpu_read_guest(vcpu,
906	912	gpa + i * sizeof(e),
907	913	&e, 2 * sizeof(u32))) {
@@ -916,9 +922,7 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 8 unmodified lines
916	922	__func__, i, e.index, e.reserved);
917	923	return -EINVAL;
918	924	}
919		− msr_info.host_initiated = false;
920		− msr_info.index = e.index;
921		− if (kvm_get_msr(vcpu, &msr_info)) {
	925	+ if (kvm_get_msr(vcpu, e.index, &data)) {
922	926	pr_debug_ratelimited(
923	927	"%s cannot read MSR (%u, 0x%x)\n",
924	928	__func__, i, e.index);
@@ -927,10 +931,10 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 2 unmodified lines
927	931	if (kvm_vcpu_write_guest(vcpu,
928	932	gpa + i * sizeof(e) +
929	933	offsetof(struct vmx_msr_entry, value),
930		− &msr_info.data, sizeof(msr_info.data))) {
	934	+ &data, sizeof(data))) {
931	935	pr_debug_ratelimited(
932	936	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
933		− __func__, i, e.index, msr_info.data);
	937	+ __func__, i, e.index, data);
934	938	return -EINVAL;
935	939	}
936	940	}
@@ -955,7 +959,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne · 18 unmodified lines
955	959	u32 *entry_failure_code)
956	960	{
957	961	if (cr3 != kvm_read_cr3(vcpu) \|\| (!nested_ept && pdptrs_changed(vcpu))) {
958		− if (!nested_cr3_valid(vcpu, cr3)) {
	962	+ if (CC(!nested_cr3_valid(vcpu, cr3))) {
959	963	*entry_failure_code = ENTRY_FAIL_DEFAULT;
960	964	return -EINVAL;
961	965	}
@@ -965,7 +969,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne · 3 unmodified lines
965	969	* must not be dereferenced.
966	970	*/
967	971	if (is_pae_paging(vcpu) && !nested_ept) {
968		− if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
	972	+ if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) {
969	973	*entry_failure_code = ENTRY_FAIL_PDPTE;
970	974	return -EINVAL;
971	975	}
@@ -2411,12 +2415,12 @@ static int prepare_vmcs02(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 1439 unmodified lines
2411	2415
2412	2416	static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2413	2417	{
2414		− if (!nested_cpu_has_nmi_exiting(vmcs12) &&
2415		− nested_cpu_has_virtual_nmis(vmcs12))
	2418	+ if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
	2419	+ nested_cpu_has_virtual_nmis(vmcs12)))
2416	2420	return -EINVAL;
2417	2421
2418		− if (!nested_cpu_has_virtual_nmis(vmcs12) &&
2419		− nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING))
	2422	+ if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
	2423	+ nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)))
2420	2424	return -EINVAL;
2421	2425
2422	2426	return 0;
@@ -2430,11 +2434,11 @@ static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) · 7 unmodified lines
2430	2434	/* Check for memory type validity */
2431	2435	switch (address & VMX_EPTP_MT_MASK) {
2432	2436	case VMX_EPTP_MT_UC:
2433		− if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))
	2437	+ if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2434	2438	return false;
2435	2439	break;
2436	2440	case VMX_EPTP_MT_WB:
2437		− if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))
	2441	+ if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2438	2442	return false;
2439	2443	break;
2440	2444	default:
@@ -2442,16 +2446,16 @@ static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) · 1 unmodified line
2442	2446	}
2443	2447
2444	2448	/* only 4 levels page-walk length are valid */
2445		− if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)
	2449	+ if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4))
2446	2450	return false;
2447	2451
2448	2452	/* Reserved bits should not be set */
2449		− if (address >> maxphyaddr \|\| ((address >> 7) & 0x1f))
	2453	+ if (CC(address >> maxphyaddr \|\| ((address >> 7) & 0x1f)))
2450	2454	return false;
2451	2455
2452	2456	/* AD, if set, should be supported */
2453	2457	if (address & VMX_EPTP_AD_ENABLE_BIT) {
2454		− if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))
	2458	+ if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2455	2459	return false;
2456	2460	}
2457	2461
@@ -2466,21 +2470,21 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 8 unmodified lines
2466	2470	{
2467	2471	struct vcpu_vmx *vmx = to_vmx(vcpu);
2468	2472
2469		− if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2470		− vmx->nested.msrs.pinbased_ctls_low,
2471		− vmx->nested.msrs.pinbased_ctls_high) \|\|
2472		− !vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2473		− vmx->nested.msrs.procbased_ctls_low,
2474		− vmx->nested.msrs.procbased_ctls_high))
	2473	+ if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
	2474	+ vmx->nested.msrs.pinbased_ctls_low,
	2475	+ vmx->nested.msrs.pinbased_ctls_high)) \|\|
	2476	+ CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
	2477	+ vmx->nested.msrs.procbased_ctls_low,
	2478	+ vmx->nested.msrs.procbased_ctls_high)))
2475	2479	return -EINVAL;
2476	2480
2477	2481	if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2478		− !vmx_control_verify(vmcs12->secondary_vm_exec_control,
2479		− vmx->nested.msrs.secondary_ctls_low,
2480		− vmx->nested.msrs.secondary_ctls_high))
	2482	+ CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
	2483	+ vmx->nested.msrs.secondary_ctls_low,
	2484	+ vmx->nested.msrs.secondary_ctls_high)))
2481	2485	return -EINVAL;
2482	2486
2483		− if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) \|\|
	2487	+ if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) \|\|
2484	2488	nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) \|\|
2485	2489	nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) \|\|
2486	2490	nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) \|\|
@@ -2491,7 +2495,7 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
2491	2495	nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) \|\|
2492	2496	nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) \|\|
2493	2497	nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) \|\|
2494		− (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
	2498	+ CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2495	2499	return -EINVAL;
2496	2500
2497	2501	if (!nested_cpu_has_preemption_timer(vmcs12) &&
@@ -2499,17 +2503,17 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
2499	2503	return -EINVAL;
2500	2504
2501	2505	if (nested_cpu_has_ept(vmcs12) &&
2502		− !valid_ept_address(vcpu, vmcs12->ept_pointer))
	2506	+ CC(!valid_ept_address(vcpu, vmcs12->ept_pointer)))
2503	2507	return -EINVAL;
2504	2508
2505	2509	if (nested_cpu_has_vmfunc(vmcs12)) {
2506		− if (vmcs12->vm_function_control &
2507		− ~vmx->nested.msrs.vmfunc_controls)
	2510	+ if (CC(vmcs12->vm_function_control &
	2511	+ ~vmx->nested.msrs.vmfunc_controls))
2508	2512	return -EINVAL;
2509	2513
2510	2514	if (nested_cpu_has_eptp_switching(vmcs12)) {
2511		− if (!nested_cpu_has_ept(vmcs12) \|\|
2512		− !page_address_valid(vcpu, vmcs12->eptp_list_address))
	2515	+ if (CC(!nested_cpu_has_ept(vmcs12)) \|\|
	2516	+ CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
2513	2517	return -EINVAL;
2514	2518	}
2515	2519	}
@@ -2525,10 +2529,10 @@ static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, · 9 unmodified lines
2525	2529	{
2526	2530	struct vcpu_vmx *vmx = to_vmx(vcpu);
2527	2531
2528		− if (!vmx_control_verify(vmcs12->vm_exit_controls,
2529		− vmx->nested.msrs.exit_ctls_low,
2530		− vmx->nested.msrs.exit_ctls_high) \|\|
2531		− nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))
	2532	+ if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
	2533	+ vmx->nested.msrs.exit_ctls_low,
	2534	+ vmx->nested.msrs.exit_ctls_high)) \|\|
	2535	+ CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
2532	2536	return -EINVAL;
2533	2537
2534	2538	return 0;
@@ -2542,9 +2546,9 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 7 unmodified lines
2542	2546	{
2543	2547	struct vcpu_vmx *vmx = to_vmx(vcpu);
2544	2548
2545		− if (!vmx_control_verify(vmcs12->vm_entry_controls,
2546		− vmx->nested.msrs.entry_ctls_low,
2547		− vmx->nested.msrs.entry_ctls_high))
	2549	+ if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
	2550	+ vmx->nested.msrs.entry_ctls_low,
	2551	+ vmx->nested.msrs.entry_ctls_high)))
2548	2552	return -EINVAL;
2549	2553
2550	2554	/*
@@ -2564,31 +2568,31 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 13 unmodified lines
2564	2568	bool prot_mode = !urg \|\| vmcs12->guest_cr0 & X86_CR0_PE;
2565	2569
2566	2570	/* VM-entry interruption-info field: interruption type */
2567		− if (intr_type == INTR_TYPE_RESERVED \|\|
2568		− (intr_type == INTR_TYPE_OTHER_EVENT &&
2569		− !nested_cpu_supports_monitor_trap_flag(vcpu)))
	2571	+ if (CC(intr_type == INTR_TYPE_RESERVED) \|\|
	2572	+ CC(intr_type == INTR_TYPE_OTHER_EVENT &&
	2573	+ !nested_cpu_supports_monitor_trap_flag(vcpu)))
2570	2574	return -EINVAL;
2571	2575
2572	2576	/* VM-entry interruption-info field: vector */
2573		− if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) \|\|
2574		− (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) \|\|
2575		− (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
	2577	+ if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) \|\|
	2578	+ CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) \|\|
	2579	+ CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2576	2580	return -EINVAL;
2577	2581
2578	2582	/* VM-entry interruption-info field: deliver error code */
2579	2583	should_have_error_code =
2580	2584	intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2581	2585	x86_exception_has_error_code(vector);
2582		− if (has_error_code != should_have_error_code)
	2586	+ if (CC(has_error_code != should_have_error_code))
2583	2587	return -EINVAL;
2584	2588
2585	2589	/* VM-entry exception error code */
2586		− if (has_error_code &&
2587		− vmcs12->vm_entry_exception_error_code & GENMASK(31, 15))
	2590	+ if (CC(has_error_code &&
	2591	+ vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)))
2588	2592	return -EINVAL;
2589	2593
2590	2594	/* VM-entry interruption-info field: reserved bits */
2591		− if (intr_info & INTR_INFO_RESVD_BITS_MASK)
	2595	+ if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
2592	2596	return -EINVAL;
2593	2597
2594	2598	/* VM-entry instruction length */
@@ -2596,9 +2600,9 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
2596	2600	case INTR_TYPE_SOFT_EXCEPTION:
2597	2601	case INTR_TYPE_SOFT_INTR:
2598	2602	case INTR_TYPE_PRIV_SW_EXCEPTION:
2599		− if ((vmcs12->vm_entry_instruction_len > 15) \|\|
2600		− (vmcs12->vm_entry_instruction_len == 0 &&
2601		− !nested_cpu_has_zero_length_injection(vcpu)))
	2603	+ if (CC(vmcs12->vm_entry_instruction_len > 15) \|\|
	2604	+ CC(vmcs12->vm_entry_instruction_len == 0 &&
	2605	+ CC(!nested_cpu_has_zero_length_injection(vcpu))))
2602	2606	return -EINVAL;
2603	2607	}
2604	2608	}
@@ -2625,40 +2629,40 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 20 unmodified lines
2625	2629	{
2626	2630	bool ia32e;
2627	2631
2628		− if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) \|\|
2629		− !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) \|\|
2630		− !nested_cr3_valid(vcpu, vmcs12->host_cr3))
	2632	+ if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) \|\|
	2633	+ CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) \|\|
	2634	+ CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
2631	2635	return -EINVAL;
2632	2636
2633		− if (is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu) \|\|
2634		− is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))
	2637	+ if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) \|\|
	2638	+ CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
2635	2639	return -EINVAL;
2636	2640
2637	2641	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
2638		− !kvm_pat_valid(vmcs12->host_ia32_pat))
	2642	+ CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
2639	2643	return -EINVAL;
2640	2644
2641	2645	ia32e = (vmcs12->vm_exit_controls &
2642	2646	VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
2643	2647
2644		− if (vmcs12->host_cs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2645		− vmcs12->host_ss_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2646		− vmcs12->host_ds_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2647		− vmcs12->host_es_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2648		− vmcs12->host_fs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2649		− vmcs12->host_gs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2650		− vmcs12->host_tr_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|
2651		− vmcs12->host_cs_selector == 0 \|\|
2652		− vmcs12->host_tr_selector == 0 \|\|
2653		− (vmcs12->host_ss_selector == 0 && !ia32e))
	2648	+ if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2649	+ CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2650	+ CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2651	+ CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2652	+ CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2653	+ CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2654	+ CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
	2655	+ CC(vmcs12->host_cs_selector == 0) \|\|
	2656	+ CC(vmcs12->host_tr_selector == 0) \|\|
	2657	+ CC(vmcs12->host_ss_selector == 0 && !ia32e))
2654	2658	return -EINVAL;
2655	2659
2656	2660	#ifdef CONFIG_X86_64
2657		− if (is_noncanonical_address(vmcs12->host_fs_base, vcpu) \|\|
2658		− is_noncanonical_address(vmcs12->host_gs_base, vcpu) \|\|
2659		− is_noncanonical_address(vmcs12->host_gdtr_base, vcpu) \|\|
2660		− is_noncanonical_address(vmcs12->host_idtr_base, vcpu) \|\|
2661		− is_noncanonical_address(vmcs12->host_tr_base, vcpu))
	2661	+ if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) \|\|
	2662	+ CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) \|\|
	2663	+ CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) \|\|
	2664	+ CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) \|\|
	2665	+ CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)))
2662	2666	return -EINVAL;
2663	2667	#endif
2664	2668
@@ -2669,9 +2673,9 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 4 unmodified lines
2669	2673	* the host address-space size VM-exit control.
2670	2674	*/
2671	2675	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
2672		− if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) \|\|
2673		− ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) \|\|
2674		− ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))
	2676	+ if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) \|\|
	2677	+ CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) \|\|
	2678	+ CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
2675	2679	return -EINVAL;
2676	2680	}
2677	2681
@@ -2688,16 +2692,16 @@ static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, · 10 unmodified lines
2688	2692	if (vmcs12->vmcs_link_pointer == -1ull)
2689	2693	return 0;
2690	2694
2691		− if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))
	2695	+ if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
2692	2696	return -EINVAL;
2693	2697
2694		− if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
	2698	+ if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)))
2695	2699	return -EINVAL;
2696	2700
2697	2701	shadow = map.hva;
2698	2702
2699		− if (shadow->hdr.revision_id != VMCS12_REVISION \|\|
2700		− shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))
	2703	+ if (CC(shadow->hdr.revision_id != VMCS12_REVISION) \|\|
	2704	+ CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
2701	2705	r = -EINVAL;
2702	2706
2703	2707	kvm_vcpu_unmap(vcpu, &map, false);
@@ -2709,8 +2713,8 @@ static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, · 5 unmodified lines
2709	2713	*/
2710	2714	static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
2711	2715	{
2712		− if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
2713		− vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
	2716	+ if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
	2717	+ vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT))
2714	2718	return -EINVAL;
2715	2719
2716	2720	return 0;
@@ -2724,12 +2728,12 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, · 7 unmodified lines
2724	2728
2725	2729	*exit_qual = ENTRY_FAIL_DEFAULT;
2726	2730
2727		− if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) \|\|
2728		− !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))
	2731	+ if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) \|\|
	2732	+ CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
2729	2733	return -EINVAL;
2730	2734
2731	2735	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
2732		− !kvm_pat_valid(vmcs12->guest_ia32_pat))
	2736	+ CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
2733	2737	return -EINVAL;
2734	2738
2735	2739	if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
@@ -2749,16 +2753,16 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, · 13 unmodified lines
2749	2753	if (to_vmx(vcpu)->nested.nested_run_pending &&
2750	2754	(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
2751	2755	ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
2752		− if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) \|\|
2753		− ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) \|\|
2754		− ((vmcs12->guest_cr0 & X86_CR0_PG) &&
2755		− ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))
	2756	+ if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) \|\|
	2757	+ CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) \|\|
	2758	+ CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
	2759	+ ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
2756	2760	return -EINVAL;
2757	2761	}
2758	2762
2759	2763	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
2760		− (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) \|\|
2761		− (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))
	2764	+ (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) \|\|
	2765	+ CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
2762	2766	return -EINVAL;
2763	2767
2764	2768	if (nested_check_guest_non_reg_state(vmcs12))
@@ -2841,9 +2845,13 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) · 76 unmodified lines
2841	2845	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2842	2846
2843	2847	if (vm_fail) {
	2848	+ u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
	2849	+
2844	2850	preempt_enable();
2845		− WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
2846		− VMXERR_ENTRY_INVALID_CONTROL_FIELD);
	2851	+
	2852	+ trace_kvm_nested_vmenter_failed(
	2853	+ "early hardware check VM-instruction error: ", error);
	2854	+ WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
2847	2855	return 1;
2848	2856	}
2849	2857
@@ -3401,6 +3409,15 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) · 551 unmodified lines
3401	3409	unsigned long exit_qual;
3402	3410	bool block_nested_events =
3403	3411	vmx->nested.nested_run_pending \|\| kvm_event_needs_reinjection(vcpu);
	3412	+ struct kvm_lapic *apic = vcpu->arch.apic;
	3413	+
	3414	+ if (lapic_in_kernel(vcpu) &&
	3415	+ test_bit(KVM_APIC_INIT, &apic->pending_events)) {
	3416	+ if (block_nested_events)
	3417	+ return -EBUSY;
	3418	+ nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
	3419	+ return 0;
	3420	+ }
3404	3421
3405	3422	if (vcpu->arch.exception.pending &&
3406	3423	nested_vmx_check_exception(vcpu, &exit_qual)) {
@@ -3889,7 +3906,6 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 482 unmodified lines
3889	3906	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3890	3907	struct vcpu_vmx *vmx = to_vmx(vcpu);
3891	3908	struct vmx_msr_entry g, h;
3892		− struct msr_data msr;
3893	3909	gpa_t gpa;
3894	3910	u32 i, j;
3895	3911
@@ -3949,7 +3965,6 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 53 unmodified lines
3949	3965	* from the guest value. The intent is to stuff host state as
3950	3966	* silently as possible, not to fully process the exit load list.
3951	3967	*/
3952		− msr.host_initiated = false;
3953	3968	for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
3954	3969	gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
3955	3970	if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
@@ -3979,9 +3994,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 23 unmodified lines
3979	3994	goto vmabort;
3980	3995	}
3981	3996
3982		− msr.index = h.index;
3983		− msr.data = h.value;
3984		− if (kvm_set_msr(vcpu, &msr)) {
	3997	+ if (kvm_set_msr(vcpu, h.index, h.value)) {
3985	3998	pr_debug_ratelimited(
3986	3999	"%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
3987	4000	__func__, j, h.index, h.value);
@@ -4466,7 +4479,12 @@ static int handle_vmoff(struct kvm_vcpu *vcpu) · 478 unmodified lines
4466	4479	{
4467	4480	if (!nested_vmx_check_permission(vcpu))
4468	4481	return 1;
	4482	+
4469	4483	free_nested(vcpu);
	4484	+
	4485	+ /* Process a latched INIT during time CPU was in VMX operation */
	4486	+ kvm_make_request(KVM_REQ_EVENT, vcpu);
	4487	+
4470	4488	return nested_vmx_succeed(vcpu);
4471	4489	}
4472	4490
@@ -5261,8 +5279,9 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) · 788 unmodified lines
5261	5279	return false;
5262	5280
5263	5281	if (unlikely(vmx->fail)) {
5264		− pr_info_ratelimited("%s failed vm entry %x\n", __func__,
5265		− vmcs_read32(VM_INSTRUCTION_ERROR));
	5282	+ trace_kvm_nested_vmenter_failed(
	5283	+ "hardware VM-instruction error: ",
	5284	+ vmcs_read32(VM_INSTRUCTION_ERROR));
5266	5285	return true;
5267	5286	}
5268	5287

arch/x86/kvm/vmx/vmenter.S+2−2

@@ -94,7 +94,7 @@ ENDPROC(vmx_vmexit) · 93 unmodified lines
94	94
95	95	/**
96	96	* __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
97		− * @vmx: struct vcpu_vmx *
	97	+ * @vmx: struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
98	98	* @regs: unsigned long * (to guest registers)
99	99	* @launched: %true if the VMCS has been launched
100	100	*
@@ -151,7 +151,7 @@ ENTRY(__vmx_vcpu_run) · 50 unmodified lines
151	151	mov VCPU_R14(%_ASM_AX), %r14
152	152	mov VCPU_R15(%_ASM_AX), %r15
153	153	#endif
154		− /* Load guest RAX. This kills the vmx_vcpu pointer! */
	154	+ /* Load guest RAX. This kills the @regs pointer! */
155	155	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
156	156
157	157	/* Enter guest mode */

arch/x86/kvm/vmx/vmx.c+48−46

@@ -1472,8 +1472,11 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) · 1471 unmodified lines
1472	1472	return 0;
1473	1473	}
1474	1474
1475		−
1476		−static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
	1475	+/*
	1476	+ * Returns an int to be compatible with SVM implementation (which can fail).
	1477	+ * Do not use directly, use skip_emulated_instruction() instead.
	1478	+ */
	1479	+static int __skip_emulated_instruction(struct kvm_vcpu *vcpu)
1477	1480	{
1478	1481	unsigned long rip;
1479	1482
@@ -1483,6 +1486,13 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) · 3 unmodified lines
1483	1486
1484	1487	/* skipping an emulated instruction also counts */
1485	1488	vmx_set_interrupt_shadow(vcpu, 0);
	1489	+
	1490	+ return EMULATE_DONE;
	1491	+}
	1492	+
	1493	+static inline void skip_emulated_instruction(struct kvm_vcpu *vcpu)
	1494	+{
	1495	+ (void)__skip_emulated_instruction(vcpu);
1486	1496	}
1487	1497
1488	1498	static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
@@ -4026,7 +4036,7 @@ static void ept_set_mmio_spte_mask(void) · 2537 unmodified lines
4026	4036	* of an EPT paging-structure entry is 110b (write/execute).
4027	4037	*/
4028	4038	kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
4029		− VMX_EPT_MISCONFIG_WX_VALUE);
	4039	+ VMX_EPT_MISCONFIG_WX_VALUE, 0);
4030	4040	}
4031	4041
4032	4042	#define VMX_XSS_EXIT_BITMAP 0
@@ -4152,6 +4162,7 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 119 unmodified lines
4152	4162
4153	4163	vcpu->arch.microcode_version = 0x100000000ULL;
4154	4164	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
	4165	+ vmx->hv_deadline_tsc = -1;
4155	4166	kvm_set_cr8(vcpu, 0);
4156	4167
4157	4168	if (!init_event) {
@@ -4856,41 +4867,12 @@ static int handle_cpuid(struct kvm_vcpu *vcpu) · 698 unmodified lines
4856	4867
4857	4868	static int handle_rdmsr(struct kvm_vcpu *vcpu)
4858	4869	{
4859		− u32 ecx = kvm_rcx_read(vcpu);
4860		− struct msr_data msr_info;
4861		−
4862		− msr_info.index = ecx;
4863		− msr_info.host_initiated = false;
4864		− if (vmx_get_msr(vcpu, &msr_info)) {
4865		− trace_kvm_msr_read_ex(ecx);
4866		− kvm_inject_gp(vcpu, 0);
4867		− return 1;
4868		− }
4869		−
4870		− trace_kvm_msr_read(ecx, msr_info.data);
4871		−
4872		− kvm_rax_write(vcpu, msr_info.data & -1u);
4873		− kvm_rdx_write(vcpu, (msr_info.data >> 32) & -1u);
4874		− return kvm_skip_emulated_instruction(vcpu);
	4870	+ return kvm_emulate_rdmsr(vcpu);
4875	4871	}
4876	4872
4877	4873	static int handle_wrmsr(struct kvm_vcpu *vcpu)
4878	4874	{
4879		− struct msr_data msr;
4880		− u32 ecx = kvm_rcx_read(vcpu);
4881		− u64 data = kvm_read_edx_eax(vcpu);
4882		−
4883		− msr.data = data;
4884		− msr.index = ecx;
4885		− msr.host_initiated = false;
4886		− if (kvm_set_msr(vcpu, &msr) != 0) {
4887		− trace_kvm_msr_write_ex(ecx, data);
4888		− kvm_inject_gp(vcpu, 0);
4889		− return 1;
4890		− }
4891		−
4892		− trace_kvm_msr_write(ecx, data);
4893		− return kvm_skip_emulated_instruction(vcpu);
	4875	+ return kvm_emulate_wrmsr(vcpu);
4894	4876	}
4895	4877
4896	4878	static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
@@ -5227,31 +5209,33 @@ emulation_error: · 330 unmodified lines
5227	5209	static void grow_ple_window(struct kvm_vcpu *vcpu)
5228	5210	{
5229	5211	struct vcpu_vmx *vmx = to_vmx(vcpu);
5230		− int old = vmx->ple_window;
	5212	+ unsigned int old = vmx->ple_window;
5231	5213
5232	5214	vmx->ple_window = __grow_ple_window(old, ple_window,
5233	5215	ple_window_grow,
5234	5216	ple_window_max);
5235	5217
5236		− if (vmx->ple_window != old)
	5218	+ if (vmx->ple_window != old) {
5237	5219	vmx->ple_window_dirty = true;
5238		−
5239		− trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
	5220	+ trace_kvm_ple_window_update(vcpu->vcpu_id,
	5221	+ vmx->ple_window, old);
	5222	+ }
5240	5223	}
5241	5224
5242	5225	static void shrink_ple_window(struct kvm_vcpu *vcpu)
5243	5226	{
5244	5227	struct vcpu_vmx *vmx = to_vmx(vcpu);
5245		− int old = vmx->ple_window;
	5228	+ unsigned int old = vmx->ple_window;
5246	5229
5247	5230	vmx->ple_window = __shrink_ple_window(old, ple_window,
5248	5231	ple_window_shrink,
5249	5232	ple_window);
5250	5233
5251		− if (vmx->ple_window != old)
	5234	+ if (vmx->ple_window != old) {
5252	5235	vmx->ple_window_dirty = true;
5253		−
5254		− trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
	5236	+ trace_kvm_ple_window_update(vcpu->vcpu_id,
	5237	+ vmx->ple_window, old);
	5238	+ }
5255	5239	}
5256	5240
5257	5241	/*
@@ -5887,8 +5871,13 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) · 629 unmodified lines
5887	5871	else {
5888	5872	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
5889	5873	exit_reason);
5890		− kvm_queue_exception(vcpu, UD_VECTOR);
5891		− return 1;
	5874	+ dump_vmcs();
	5875	+ vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
	5876	+ vcpu->run->internal.suberror =
	5877	+ KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
	5878	+ vcpu->run->internal.ndata = 1;
	5879	+ vcpu->run->internal.data[0] = exit_reason;
	5880	+ return 0;
5892	5881	}
5893	5882	}
5894	5883
@@ -6615,6 +6604,9 @@ static struct kvm_vcpu vmx_create_vcpu(struct kvm kvm, unsigned int id) · 720 unmodified lines
6615	6604	unsigned long *msr_bitmap;
6616	6605	int cpu;
6617	6606
	6607	+ BUILD_BUG_ON_MSG(offsetof(struct vcpu_vmx, vcpu) != 0,
	6608	+ "struct kvm_vcpu must be at offset 0 for arch usercopy region");
	6609	+
6618	6610	vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
6619	6611	if (!vmx)
6620	6612	return ERR_PTR(-ENOMEM);
@@ -7369,10 +7361,14 @@ static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq, · 748 unmodified lines
7369	7361	* irqbalance to make the interrupts single-CPU.
7370	7362	*
7371	7363	* We will support full lowest-priority interrupt later.
	7364	+ *
	7365	+ * In addition, we can only inject generic interrupts using
	7366	+ * the PI mechanism, refuse to route others through it.
7372	7367	*/
7373	7368
7374	7369	kvm_set_msi_irq(kvm, e, &irq);
7375		− if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
	7370	+ if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) \|\|
	7371	+ !kvm_irq_is_postable(&irq)) {
7376	7372	/*
7377	7373	* Make sure the IRTE is in remapped mode if
7378	7374	* we don't handle it in posted mode.
@@ -7474,6 +7470,11 @@ static bool vmx_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) · 95 unmodified lines
7474	7470	return false;
7475	7471	}
7476	7472
	7473	+static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
	7474	+{
	7475	+ return to_vmx(vcpu)->nested.vmxon;
	7476	+}
	7477	+
7477	7478	static __init int hardware_setup(void)
7478	7479	{
7479	7480	unsigned long host_bndcfgs;
@@ -7705,7 +7706,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { · 225 unmodified lines
7705	7706
7706	7707	.run = vmx_vcpu_run,
7707	7708	.handle_exit = vmx_handle_exit,
7708		− .skip_emulated_instruction = skip_emulated_instruction,
	7709	+ .skip_emulated_instruction = __skip_emulated_instruction,
7709	7710	.set_interrupt_shadow = vmx_set_interrupt_shadow,
7710	7711	.get_interrupt_shadow = vmx_get_interrupt_shadow,
7711	7712	.patch_hypercall = vmx_patch_hypercall,
@@ -7799,6 +7800,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { · 87 unmodified lines
7799	7800	.nested_enable_evmcs = NULL,
7800	7801	.nested_get_evmcs_version = NULL,
7801	7802	.need_emulation_on_page_fault = vmx_need_emulation_on_page_fault,
	7803	+ .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
7802	7804	};
7803	7805
7804	7806	static void vmx_cleanup_l1d_flush(void)

arch/x86/kvm/vmx/vmx.h+1−1

@@ -253,7 +253,7 @@ struct vcpu_vmx { · 252 unmodified lines
253	253	struct nested_vmx nested;
254	254
255	255	/* Dynamic PLE window. */
256		− int ple_window;
	256	+ unsigned int ple_window;
257	257	bool ple_window_dirty;
258	258
259	259	bool req_immediate_exit;

arch/x86/kvm/x86.c+128−69

@@ -674,8 +674,14 @@ static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, · 673 unmodified lines
674	674	data, offset, len, access);
675	675	}
676	676
	677	+static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
	678	+{
	679	+ return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) \| rsvd_bits(5, 8) \|
	680	+ rsvd_bits(1, 2);
	681	+}
	682	+
677	683	/*
678		− * Load the pae pdptrs. Return true is they are all valid.
	684	+ * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise.
679	685	*/
680	686	int load_pdptrs(struct kvm_vcpu vcpu, struct kvm_mmu mmu, unsigned long cr3)
681	687	{
@@ -694,8 +700,7 @@ int load_pdptrs(struct kvm_vcpu vcpu, struct kvm_mmu mmu, unsigned long cr3) · 12 unmodified lines
694	700	}
695	701	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
696	702	if ((pdpte[i] & PT_PRESENT_MASK) &&
697		− (pdpte[i] &
698		− vcpu->arch.mmu->guest_rsvd_check.rsvd_bits_mask[0][2])) {
	703	+ (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
699	704	ret = 0;
700	705	goto out;
701	706	}
@@ -1254,6 +1259,13 @@ static u64 kvm_get_arch_capabilities(void) · 552 unmodified lines
1254	1259	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1255	1260	data \|= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1256	1261
	1262	+ if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
	1263	+ data \|= ARCH_CAP_RDCL_NO;
	1264	+ if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
	1265	+ data \|= ARCH_CAP_SSB_NO;
	1266	+ if (!boot_cpu_has_bug(X86_BUG_MDS))
	1267	+ data \|= ARCH_CAP_MDS_NO;
	1268	+
1257	1269	return data;
1258	1270	}
1259	1271
@@ -1351,19 +1363,23 @@ void kvm_enable_efer_bits(u64 mask) · 91 unmodified lines
1351	1363	EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1352	1364
1353	1365	/*
1354		− * Writes msr value into into the appropriate "register".
	1366	+ * Write @data into the MSR specified by @index. Select MSR specific fault
	1367	+ * checks are bypassed if @host_initiated is %true.
1355	1368	* Returns 0 on success, non-0 otherwise.
1356	1369	* Assumes vcpu_load() was already called.
1357	1370	*/
1358		−int kvm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr)
	1371	+static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
	1372	+ bool host_initiated)
1359	1373	{
1360		− switch (msr->index) {
	1374	+ struct msr_data msr;
	1375	+
	1376	+ switch (index) {
1361	1377	case MSR_FS_BASE:
1362	1378	case MSR_GS_BASE:
1363	1379	case MSR_KERNEL_GS_BASE:
1364	1380	case MSR_CSTAR:
1365	1381	case MSR_LSTAR:
1366		− if (is_noncanonical_address(msr->data, vcpu))
	1382	+ if (is_noncanonical_address(data, vcpu))
1367	1383	return 1;
1368	1384	break;
1369	1385	case MSR_IA32_SYSENTER_EIP:
@@ -1380,38 +1396,95 @@ int kvm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) · 10 unmodified lines
1380	1396	* value, and that something deterministic happens if the guest
1381	1397	* invokes 64-bit SYSENTER.
1382	1398	*/
1383		− msr->data = get_canonical(msr->data, vcpu_virt_addr_bits(vcpu));
	1399	+ data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1384	1400	}
1385		− return kvm_x86_ops->set_msr(vcpu, msr);
	1401	+
	1402	+ msr.data = data;
	1403	+ msr.index = index;
	1404	+ msr.host_initiated = host_initiated;
	1405	+
	1406	+ return kvm_x86_ops->set_msr(vcpu, &msr);
1386	1407	}
1387		−EXPORT_SYMBOL_GPL(kvm_set_msr);
1388	1408
1389	1409	/*
1390		− * Adapt set_msr() to msr_io()'s calling convention
	1410	+ * Read the MSR specified by @index into @data. Select MSR specific fault
	1411	+ * checks are bypassed if @host_initiated is %true.
	1412	+ * Returns 0 on success, non-0 otherwise.
	1413	+ * Assumes vcpu_load() was already called.
1391	1414	*/
1392		−static int do_get_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
	1415	+static int __kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data,
	1416	+ bool host_initiated)
1393	1417	{
1394	1418	struct msr_data msr;
1395		− int r;
	1419	+ int ret;
1396	1420
1397	1421	msr.index = index;
1398		− msr.host_initiated = true;
1399		− r = kvm_get_msr(vcpu, &msr);
1400		− if (r)
1401		− return r;
	1422	+ msr.host_initiated = host_initiated;
1402	1423
1403		− *data = msr.data;
1404		− return 0;
	1424	+ ret = kvm_x86_ops->get_msr(vcpu, &msr);
	1425	+ if (!ret)
	1426	+ *data = msr.data;
	1427	+ return ret;
1405	1428	}
1406	1429
1407		−static int do_set_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
	1430	+int kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data)
1408	1431	{
1409		− struct msr_data msr;
	1432	+ return __kvm_get_msr(vcpu, index, data, false);
	1433	+}
	1434	+EXPORT_SYMBOL_GPL(kvm_get_msr);
1410	1435
1411		− msr.data = *data;
1412		− msr.index = index;
1413		− msr.host_initiated = true;
1414		− return kvm_set_msr(vcpu, &msr);
	1436	+int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
	1437	+{
	1438	+ return __kvm_set_msr(vcpu, index, data, false);
	1439	+}
	1440	+EXPORT_SYMBOL_GPL(kvm_set_msr);
	1441	+
	1442	+int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
	1443	+{
	1444	+ u32 ecx = kvm_rcx_read(vcpu);
	1445	+ u64 data;
	1446	+
	1447	+ if (kvm_get_msr(vcpu, ecx, &data)) {
	1448	+ trace_kvm_msr_read_ex(ecx);
	1449	+ kvm_inject_gp(vcpu, 0);
	1450	+ return 1;
	1451	+ }
	1452	+
	1453	+ trace_kvm_msr_read(ecx, data);
	1454	+
	1455	+ kvm_rax_write(vcpu, data & -1u);
	1456	+ kvm_rdx_write(vcpu, (data >> 32) & -1u);
	1457	+ return kvm_skip_emulated_instruction(vcpu);
	1458	+}
	1459	+EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
	1460	+
	1461	+int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
	1462	+{
	1463	+ u32 ecx = kvm_rcx_read(vcpu);
	1464	+ u64 data = kvm_read_edx_eax(vcpu);
	1465	+
	1466	+ if (kvm_set_msr(vcpu, ecx, data)) {
	1467	+ trace_kvm_msr_write_ex(ecx, data);
	1468	+ kvm_inject_gp(vcpu, 0);
	1469	+ return 1;
	1470	+ }
	1471	+
	1472	+ trace_kvm_msr_write(ecx, data);
	1473	+ return kvm_skip_emulated_instruction(vcpu);
	1474	+}
	1475	+EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
	1476	+
	1477	+/*
	1478	+ * Adapt set_msr() to msr_io()'s calling convention
	1479	+ */
	1480	+static int do_get_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
	1481	+{
	1482	+ return __kvm_get_msr(vcpu, index, data, true);
	1483	+}
	1484	+
	1485	+static int do_set_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
	1486	+{
	1487	+ return __kvm_set_msr(vcpu, index, *data, true);
1415	1488	}
1416	1489
1417	1490	#ifdef CONFIG_X86_64
@@ -2452,6 +2525,8 @@ static void record_steal_time(struct kvm_vcpu *vcpu) · 1034 unmodified lines
2452	2525	* Doing a TLB flush here, on the guest's behalf, can avoid
2453	2526	* expensive IPIs.
2454	2527	*/
	2528	+ trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
	2529	+ vcpu->arch.st.steal.preempted & KVM_VCPU_FLUSH_TLB);
2455	2530	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
2456	2531	kvm_vcpu_flush_tlb(vcpu, false);
2457	2532
@@ -2748,18 +2823,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 290 unmodified lines
2748	2823	}
2749	2824	EXPORT_SYMBOL_GPL(kvm_set_msr_common);
2750	2825
2751		−
2752		−/*
2753		− * Reads an msr value (of 'msr_index') into 'pdata'.
2754		− * Returns 0 on success, non-0 otherwise.
2755		− * Assumes vcpu_load() was already called.
2756		− */
2757		−int kvm_get_msr(struct kvm_vcpu vcpu, struct msr_data msr)
2758		−{
2759		− return kvm_x86_ops->get_msr(vcpu, msr);
2760		−}
2761		−EXPORT_SYMBOL_GPL(kvm_get_msr);
2762		−
2763	2826	static int get_msr_mce(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
2764	2827	{
2765	2828	u64 data;
@@ -3506,8 +3569,7 @@ static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, · 740 unmodified lines
3506	3569	for (bank = 0; bank < bank_num; bank++)
3507	3570	vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3508	3571
3509		− if (kvm_x86_ops->setup_mce)
3510		− kvm_x86_ops->setup_mce(vcpu);
	3572	+ kvm_x86_ops->setup_mce(vcpu);
3511	3573	out:
3512	3574	return r;
3513	3575	}
@@ -5377,7 +5439,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, · 1863 unmodified lines
5377	5439	*/
5378	5440	if (vcpu_match_mmio_gva(vcpu, gva)
5379	5441	&& !permission_fault(vcpu, vcpu->arch.walk_mmu,
5380		− vcpu->arch.access, 0, access)) {
	5442	+ vcpu->arch.mmio_access, 0, access)) {
5381	5443	*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT \|
5382	5444	(gva & (PAGE_SIZE - 1));
5383	5445	trace_vcpu_match_mmio(gva, *gpa, write, false);
@@ -5971,28 +6033,13 @@ static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, · 587 unmodified lines
5971	6033	static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
5972	6034	u32 msr_index, u64 *pdata)
5973	6035	{
5974		− struct msr_data msr;
5975		− int r;
5976		−
5977		− msr.index = msr_index;
5978		− msr.host_initiated = false;
5979		− r = kvm_get_msr(emul_to_vcpu(ctxt), &msr);
5980		− if (r)
5981		− return r;
5982		−
5983		− *pdata = msr.data;
5984		− return 0;
	6036	+ return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
5985	6037	}
5986	6038
5987	6039	static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
5988	6040	u32 msr_index, u64 data)
5989	6041	{
5990		− struct msr_data msr;
5991		−
5992		− msr.data = data;
5993		− msr.index = msr_index;
5994		− msr.host_initiated = false;
5995		− return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
	6042	+ return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
5996	6043	}
5997	6044
5998	6045	static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
@@ -6075,6 +6122,11 @@ static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt) · 76 unmodified lines
6075	6122	kvm_smm_changed(emul_to_vcpu(ctxt));
6076	6123	}
6077	6124
	6125	+static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
	6126	+{
	6127	+ return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
	6128	+}
	6129	+
6078	6130	static const struct x86_emulate_ops emulate_ops = {
6079	6131	.read_gpr = emulator_read_gpr,
6080	6132	.write_gpr = emulator_write_gpr,
@@ -6116,6 +6168,7 @@ static const struct x86_emulate_ops emulate_ops = { · 35 unmodified lines
6116	6168	.set_hflags = emulator_set_hflags,
6117	6169	.pre_leave_smm = emulator_pre_leave_smm,
6118	6170	.post_leave_smm = emulator_post_leave_smm,
	6171	+ .set_xcr = emulator_set_xcr,
6119	6172	};
6120	6173
6121	6174	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
@@ -6390,9 +6443,11 @@ static void kvm_vcpu_do_singlestep(struct kvm_vcpu vcpu, int r) · 268 unmodified lines
6390	6443	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
6391	6444	{
6392	6445	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6393		− int r = EMULATE_DONE;
	6446	+ int r;
6394	6447
6395		− kvm_x86_ops->skip_emulated_instruction(vcpu);
	6448	+ r = kvm_x86_ops->skip_emulated_instruction(vcpu);
	6449	+ if (unlikely(r != EMULATE_DONE))
	6450	+ return 0;
6396	6451
6397	6452	/*
6398	6453	* rflags is the old, "raw" value of the flags. The new value has
@@ -6528,8 +6583,16 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, · 129 unmodified lines
6528	6583	if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
6529	6584	emulation_type))
6530	6585	return EMULATE_DONE;
6531		− if (ctxt->have_exception && inject_emulated_exception(vcpu))
	6586	+ if (ctxt->have_exception) {
	6587	+ /*
	6588	+ * #UD should result in just EMULATION_FAILED, and trap-like
	6589	+ * exception should not be encountered during decode.
	6590	+ */
	6591	+ WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR \|\|
	6592	+ exception_type(ctxt->exception.vector) == EXCPT_TRAP);
	6593	+ inject_emulated_exception(vcpu);
6532	6594	return EMULATE_DONE;
	6595	+ }
6533	6596	if (emulation_type & EMULTYPE_SKIP)
6534	6597	return EMULATE_FAIL;
6535	6598	return handle_emulation_failure(vcpu, emulation_type);
@@ -6544,6 +6607,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, · 8 unmodified lines
6544	6607	kvm_rip_write(vcpu, ctxt->_eip);
6545	6608	if (ctxt->eflags & X86_EFLAGS_RF)
6546	6609	kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
	6610	+ kvm_x86_ops->set_interrupt_shadow(vcpu, 0);
6547	6611	return EMULATE_DONE;
6548	6612	}
6549	6613
@@ -9322,10 +9386,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 2772 unmodified lines
9322	9386	kvm_page_track_init(kvm);
9323	9387	kvm_mmu_init_vm(kvm);
9324	9388
9325		− if (kvm_x86_ops->vm_init)
9326		− return kvm_x86_ops->vm_init(kvm);
9327		−
9328		− return 0;
	9389	+ return kvm_x86_ops->vm_init(kvm);
9329	9390	}
9330	9391
9331	9392	static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
@@ -10017,7 +10078,7 @@ EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma); · 685 unmodified lines
10017	10078
10018	10079	bool kvm_arch_has_irq_bypass(void)
10019	10080	{
10020		− return kvm_x86_ops->update_pi_irte != NULL;
	10081	+ return true;
10021	10082	}
10022	10083
10023	10084	int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
@@ -10057,9 +10118,6 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 33 unmodified lines
10057	10118	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
10058	10119	uint32_t guest_irq, bool set)
10059	10120	{
10060		− if (!kvm_x86_ops->update_pi_irte)
10061		− return -EINVAL;
10062		−
10063	10121	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
10064	10122	}
10065	10123
@@ -10086,11 +10144,12 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); · 20 unmodified lines
10086	10144	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10087	10145	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10088	10146	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
	10147	+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10089	10148	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10090	10149	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10091	10150	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10092	10151	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
10093		−EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window);
	10152	+EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
10094	10153	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10095	10154	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10096	10155	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);

arch/x86/kvm/x86.h+1−1

@@ -196,7 +196,7 @@ static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, · 195 unmodified lines
196	196	* actually a nGPA.
197	197	*/
198	198	vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;
199		− vcpu->arch.access = access;
	199	+ vcpu->arch.mmio_access = access;
200	200	vcpu->arch.mmio_gfn = gfn;
201	201	vcpu->arch.mmio_gen = gen;
202	202	}

include/kvm/arm_vgic.h+3−1

@@ -249,6 +249,9 @@ struct vgic_dist { · 248 unmodified lines
249	249	struct list_head lpi_list_head;
250	250	int lpi_list_count;
251	251
	252	+ /* LPI translation cache */
	253	+ struct list_head lpi_translation_cache;
	254	+
252	255	/* used by vgic-debug */
253	256	struct vgic_state_iter *iter;
254	257
@@ -311,7 +314,6 @@ struct vgic_cpu { · 56 unmodified lines
311	314	* parts of the redistributor.
312	315	*/
313	316	struct vgic_io_device rd_iodev;
314		− struct vgic_io_device sgi_iodev;
315	317	struct vgic_redist_region *rdreg;
316	318
317	319	/* Contains the attributes and gpa of the LPI pending tables. */

include/uapi/linux/kvm.h+3−0

@@ -243,6 +243,8 @@ struct kvm_hyperv_exit { · 242 unmodified lines
243	243	#define KVM_INTERNAL_ERROR_SIMUL_EX 2
244	244	/* Encounter unexpected vm-exit due to delivery event. */
245	245	#define KVM_INTERNAL_ERROR_DELIVERY_EV 3
	246	+/* Encounter unexpected vm-exit reason */
	247	+#define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4
246	248
247	249	/* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
248	250	struct kvm_run {
@@ -996,6 +998,7 @@ struct kvm_ppc_resize_hpt { · 747 unmodified lines
996	998	#define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
997	999	#define KVM_CAP_ARM_PTRAUTH_GENERIC 172
998	1000	#define KVM_CAP_PMU_EVENT_FILTER 173
	1001	+#define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
999	1002
1000	1003	#ifdef KVM_CAP_IRQ_ROUTING
1001	1004

tools/testing/selftests/kvm/Makefile+6−4

@@ -7,10 +7,10 @@ top_srcdir = ../../../.. · 6 unmodified lines
7	7	KSFT_KHDR_INSTALL := 1
8	8	UNAME_M := $(shell uname -m)
9	9
10		−LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/ucall.c lib/sparsebit.c
11		−LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c
12		−LIBKVM_aarch64 = lib/aarch64/processor.c
13		−LIBKVM_s390x = lib/s390x/processor.c
	10	+LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c
	11	+LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/ucall.c
	12	+LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c
	13	+LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c
14	14
15	15	TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test
16	16	TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test
@@ -32,7 +32,9 @@ TEST_GEN_PROGS_aarch64 += clear_dirty_log_test · 15 unmodified lines
32	32	TEST_GEN_PROGS_aarch64 += dirty_log_test
33	33	TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus
34	34
	35	+TEST_GEN_PROGS_s390x = s390x/memop
35	36	TEST_GEN_PROGS_s390x += s390x/sync_regs_test
	37	+TEST_GEN_PROGS_s390x += dirty_log_test
36	38	TEST_GEN_PROGS_s390x += kvm_create_max_vcpus
37	39
38	40	TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M))

tools/testing/selftests/kvm/dirty_log_test.c+53−8

@@ -26,8 +26,8 @@ · 25 unmodified lines
26	26	/* The memory slot index to track dirty pages */
27	27	#define TEST_MEM_SLOT_INDEX 1
28	28
29		−/* Default guest test memory offset, 1G */
30		−#define DEFAULT_GUEST_TEST_MEM 0x40000000
	29	+/* Default guest test virtual memory offset */
	30	+#define DEFAULT_GUEST_TEST_MEM 0xc0000000
31	31
32	32	/* How many pages to dirty for each guest loop */
33	33	#define TEST_PAGES_PER_LOOP 1024
@@ -38,6 +38,27 @@ · 4 unmodified lines
38	38	/* Interval for each host loop (ms) */
39	39	#define TEST_HOST_LOOP_INTERVAL 10UL
40	40
	41	+/* Dirty bitmaps are always little endian, so we need to swap on big endian */
	42	+#if defined(__s390x__)
	43	+# define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7)
	44	+# define test_bit_le(nr, addr) \
	45	+ test_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
	46	+# define set_bit_le(nr, addr) \
	47	+ set_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
	48	+# define clear_bit_le(nr, addr) \
	49	+ clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
	50	+# define test_and_set_bit_le(nr, addr) \
	51	+ test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
	52	+# define test_and_clear_bit_le(nr, addr) \
	53	+ test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
	54	+#else
	55	+# define test_bit_le test_bit
	56	+# define set_bit_le set_bit
	57	+# define clear_bit_le clear_bit
	58	+# define test_and_set_bit_le test_and_set_bit
	59	+# define test_and_clear_bit_le test_and_clear_bit
	60	+#endif
	61	+
41	62	/*
42	63	* Guest/Host shared variables. Ensure addr_gva2hva() and/or
43	64	* sync_global_to/from_guest() are used when accessing from
@@ -69,11 +90,23 @@ static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM; · 25 unmodified lines
69	90	*/
70	91	static void guest_code(void)
71	92	{
	93	+ uint64_t addr;
72	94	int i;
73	95
	96	+ /*
	97	+ * On s390x, all pages of a 1M segment are initially marked as dirty
	98	+ * when a page of the segment is written to for the very first time.
	99	+ * To compensate this specialty in this test, we need to touch all
	100	+ * pages during the first iteration.
	101	+ */
	102	+ for (i = 0; i < guest_num_pages; i++) {
	103	+ addr = guest_test_virt_mem + i * guest_page_size;
	104	+ (uint64_t )addr = READ_ONCE(iteration);
	105	+ }
	106	+
74	107	while (true) {
75	108	for (i = 0; i < TEST_PAGES_PER_LOOP; i++) {
76		− uint64_t addr = guest_test_virt_mem;
	109	+ addr = guest_test_virt_mem;
77	110	addr += (READ_ONCE(random_array[i]) % guest_num_pages)
78	111	* guest_page_size;
79	112	addr &= ~(host_page_size - 1);
@@ -158,15 +191,15 @@ static void vm_dirty_log_verify(unsigned long *bmap) · 78 unmodified lines
158	191	value_ptr = host_test_mem + page * host_page_size;
159	192
160	193	/* If this is a special page that we were tracking... */
161		− if (test_and_clear_bit(page, host_bmap_track)) {
	194	+ if (test_and_clear_bit_le(page, host_bmap_track)) {
162	195	host_track_next_count++;
163		− TEST_ASSERT(test_bit(page, bmap),
	196	+ TEST_ASSERT(test_bit_le(page, bmap),
164	197	"Page %"PRIu64" should have its dirty bit "
165	198	"set in this iteration but it is missing",
166	199	page);
167	200	}
168	201
169		− if (test_bit(page, bmap)) {
	202	+ if (test_bit_le(page, bmap)) {
170	203	host_dirty_count++;
171	204	/*
172	205	* If the bit is set, the value written onto
@@ -209,7 +242,7 @@ static void vm_dirty_log_verify(unsigned long *bmap) · 36 unmodified lines
209	242	* should report its dirtyness in the
210	243	* next run
211	244	*/
212		− set_bit(page, host_bmap_track);
	245	+ set_bit_le(page, host_bmap_track);
213	246	}
214	247	}
215	248	}
@@ -293,6 +326,10 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, · 77 unmodified lines
293	326	* case where the size is not aligned to 64 pages.
294	327	*/
295	328	guest_num_pages = (1ul << (30 - guest_page_shift)) + 16;
	329	+#ifdef __s390x__
	330	+ /* Round up to multiple of 1M (segment size) */
	331	+ guest_num_pages = (guest_num_pages + 0xff) & ~0xffUL;
	332	+#endif
296	333	host_page_size = getpagesize();
297	334	host_num_pages = (guest_num_pages * guest_page_size) / host_page_size +
298	335	!!((guest_num_pages * guest_page_size) % host_page_size);
@@ -304,6 +341,11 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, · 5 unmodified lines
304	341	guest_test_phys_mem = phys_offset;
305	342	}
306	343
	344	+#ifdef __s390x__
	345	+ /* Align to 1M (segment size) */
	346	+ guest_test_phys_mem &= ~((1 << 20) - 1);
	347	+#endif
	348	+
307	349	DEBUG("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem);
308	350
309	351	bmap = bitmap_alloc(host_num_pages);
@@ -337,7 +379,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, · 27 unmodified lines
337	379	vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
338	380	#endif
339	381	#ifdef __aarch64__
340		− ucall_init(vm, UCALL_MMIO, NULL);
	382	+ ucall_init(vm, NULL);
341	383	#endif
342	384
343	385	/* Export the shared variables to the guest */
@@ -454,6 +496,9 @@ int main(int argc, char *argv[]) · 110 unmodified lines
454	496	vm_guest_mode_params_init(VM_MODE_P48V48_64K, true, true);
455	497	}
456	498	#endif
	499	+#ifdef __s390x__
	500	+ vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true);
	501	+#endif
457	502
458	503	while ((opt = getopt(argc, argv, "hi:I:p:m:")) != -1) {
459	504	switch (opt) {

tools/testing/selftests/kvm/include/kvm_util.h+1−7

@@ -165,12 +165,6 @@ int vm_create_device(struct kvm_vm vm, struct kvm_create_device cd); · 164 unmodified lines
165	165	memcpy(&(g), _p, sizeof(g)); \
166	166	})
167	167
168		−/* ucall implementation types */
169		−typedef enum {
170		− UCALL_PIO,
171		− UCALL_MMIO,
172		−} ucall_type_t;
173		−
174	168	/* Common ucalls */
175	169	enum {
176	170	UCALL_NONE,
@@ -186,7 +180,7 @@ struct ucall { · 9 unmodified lines
186	180	uint64_t args[UCALL_MAX_ARGS];
187	181	};
188	182
189		−void ucall_init(struct kvm_vm vm, ucall_type_t type, void arg);
	183	+void ucall_init(struct kvm_vm vm, void arg);
190	184	void ucall_uninit(struct kvm_vm *vm);
191	185	void ucall(uint64_t cmd, int nargs, ...);
192	186	uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc);

tools/testing/selftests/kvm/lib/aarch64/ucall.c+112−0

@@ -0,0 +1,112 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ucall support. A ucall is a "hypercall to userspace".
	4	+ *
	5	+ * Copyright (C) 2018, Red Hat, Inc.
	6	+ */
	7	+#include "kvm_util.h"
	8	+#include "../kvm_util_internal.h"
	9	+
	10	+static vm_vaddr_t *ucall_exit_mmio_addr;
	11	+
	12	+static bool ucall_mmio_init(struct kvm_vm *vm, vm_paddr_t gpa)
	13	+{
	14	+ if (kvm_userspace_memory_region_find(vm, gpa, gpa + 1))
	15	+ return false;
	16	+
	17	+ virt_pg_map(vm, gpa, gpa, 0);
	18	+
	19	+ ucall_exit_mmio_addr = (vm_vaddr_t *)gpa;
	20	+ sync_global_to_guest(vm, ucall_exit_mmio_addr);
	21	+
	22	+ return true;
	23	+}
	24	+
	25	+void ucall_init(struct kvm_vm vm, void arg)
	26	+{
	27	+ vm_paddr_t gpa, start, end, step, offset;
	28	+ unsigned int bits;
	29	+ bool ret;
	30	+
	31	+ if (arg) {
	32	+ gpa = (vm_paddr_t)arg;
	33	+ ret = ucall_mmio_init(vm, gpa);
	34	+ TEST_ASSERT(ret, "Can't set ucall mmio address to %lx", gpa);
	35	+ return;
	36	+ }
	37	+
	38	+ /*
	39	+ * Find an address within the allowed physical and virtual address
	40	+ * spaces, that does _not_ have a KVM memory region associated with
	41	+ * it. Identity mapping an address like this allows the guest to
	42	+ * access it, but as KVM doesn't know what to do with it, it
	43	+ * will assume it's something userspace handles and exit with
	44	+ * KVM_EXIT_MMIO. Well, at least that's how it works for AArch64.
	45	+ * Here we start with a guess that the addresses around 5/8th
	46	+ * of the allowed space are unmapped and then work both down and
	47	+ * up from there in 1/16th allowed space sized steps.
	48	+ *
	49	+ * Note, we need to use VA-bits - 1 when calculating the allowed
	50	+ * virtual address space for an identity mapping because the upper
	51	+ * half of the virtual address space is the two's complement of the
	52	+ * lower and won't match physical addresses.
	53	+ */
	54	+ bits = vm->va_bits - 1;
	55	+ bits = vm->pa_bits < bits ? vm->pa_bits : bits;
	56	+ end = 1ul << bits;
	57	+ start = end * 5 / 8;
	58	+ step = end / 16;
	59	+ for (offset = 0; offset < end - start; offset += step) {
	60	+ if (ucall_mmio_init(vm, start - offset))
	61	+ return;
	62	+ if (ucall_mmio_init(vm, start + offset))
	63	+ return;
	64	+ }
	65	+ TEST_ASSERT(false, "Can't find a ucall mmio address");
	66	+}
	67	+
	68	+void ucall_uninit(struct kvm_vm *vm)
	69	+{
	70	+ ucall_exit_mmio_addr = 0;
	71	+ sync_global_to_guest(vm, ucall_exit_mmio_addr);
	72	+}
	73	+
	74	+void ucall(uint64_t cmd, int nargs, ...)
	75	+{
	76	+ struct ucall uc = {
	77	+ .cmd = cmd,
	78	+ };
	79	+ va_list va;
	80	+ int i;
	81	+
	82	+ nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	83	+
	84	+ va_start(va, nargs);
	85	+ for (i = 0; i < nargs; ++i)
	86	+ uc.args[i] = va_arg(va, uint64_t);
	87	+ va_end(va);
	88	+
	89	+ *ucall_exit_mmio_addr = (vm_vaddr_t)&uc;
	90	+}
	91	+
	92	+uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	93	+{
	94	+ struct kvm_run *run = vcpu_state(vm, vcpu_id);
	95	+ struct ucall ucall = {};
	96	+
	97	+ if (run->exit_reason == KVM_EXIT_MMIO &&
	98	+ run->mmio.phys_addr == (uint64_t)ucall_exit_mmio_addr) {
	99	+ vm_vaddr_t gva;
	100	+
	101	+ TEST_ASSERT(run->mmio.is_write && run->mmio.len == 8,
	102	+ "Unexpected ucall exit mmio address access");
	103	+ memcpy(&gva, run->mmio.data, sizeof(gva));
	104	+ memcpy(&ucall, addr_gva2hva(vm, gva), sizeof(ucall));
	105	+
	106	+ vcpu_run_complete_io(vm, vcpu_id);
	107	+ if (uc)
	108	+ memcpy(uc, &ucall, sizeof(ucall));
	109	+ }
	110	+
	111	+ return ucall.cmd;
	112	+}

tools/testing/selftests/kvm/lib/s390x/ucall.c+56−0

@@ -0,0 +1,56 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ucall support. A ucall is a "hypercall to userspace".
	4	+ *
	5	+ * Copyright (C) 2019 Red Hat, Inc.
	6	+ */
	7	+#include "kvm_util.h"
	8	+
	9	+void ucall_init(struct kvm_vm vm, void arg)
	10	+{
	11	+}
	12	+
	13	+void ucall_uninit(struct kvm_vm *vm)
	14	+{
	15	+}
	16	+
	17	+void ucall(uint64_t cmd, int nargs, ...)
	18	+{
	19	+ struct ucall uc = {
	20	+ .cmd = cmd,
	21	+ };
	22	+ va_list va;
	23	+ int i;
	24	+
	25	+ nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	26	+
	27	+ va_start(va, nargs);
	28	+ for (i = 0; i < nargs; ++i)
	29	+ uc.args[i] = va_arg(va, uint64_t);
	30	+ va_end(va);
	31	+
	32	+ /* Exit via DIAGNOSE 0x501 (normally used for breakpoints) */
	33	+ asm volatile ("diag 0,%0,0x501" : : "a"(&uc) : "memory");
	34	+}
	35	+
	36	+uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	37	+{
	38	+ struct kvm_run *run = vcpu_state(vm, vcpu_id);
	39	+ struct ucall ucall = {};
	40	+
	41	+ if (run->exit_reason == KVM_EXIT_S390_SIEIC &&
	42	+ run->s390_sieic.icptcode == 4 &&
	43	+ (run->s390_sieic.ipa >> 8) == 0x83 && /* 0x83 means DIAGNOSE */
	44	+ (run->s390_sieic.ipb >> 16) == 0x501) {
	45	+ int reg = run->s390_sieic.ipa & 0xf;
	46	+
	47	+ memcpy(&ucall, addr_gva2hva(vm, run->s.regs.gprs[reg]),
	48	+ sizeof(ucall));
	49	+
	50	+ vcpu_run_complete_io(vm, vcpu_id);
	51	+ if (uc)
	52	+ memcpy(uc, &ucall, sizeof(ucall));
	53	+ }
	54	+
	55	+ return ucall.cmd;
	56	+}

tools/testing/selftests/kvm/lib/ucall.c+0−157

@@ -1,157 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−/*
3		− * ucall support. A ucall is a "hypercall to userspace".
4		− *
5		− * Copyright (C) 2018, Red Hat, Inc.
6		− */
7		−#include "kvm_util.h"
8		−#include "kvm_util_internal.h"
9		−
10		−#define UCALL_PIO_PORT ((uint16_t)0x1000)
11		−
12		−static ucall_type_t ucall_type;
13		−static vm_vaddr_t *ucall_exit_mmio_addr;
14		−
15		−static bool ucall_mmio_init(struct kvm_vm *vm, vm_paddr_t gpa)
16		−{
17		− if (kvm_userspace_memory_region_find(vm, gpa, gpa + 1))
18		− return false;
19		−
20		− virt_pg_map(vm, gpa, gpa, 0);
21		−
22		− ucall_exit_mmio_addr = (vm_vaddr_t *)gpa;
23		− sync_global_to_guest(vm, ucall_exit_mmio_addr);
24		−
25		− return true;
26		−}
27		−
28		−void ucall_init(struct kvm_vm vm, ucall_type_t type, void arg)
29		−{
30		− ucall_type = type;
31		− sync_global_to_guest(vm, ucall_type);
32		−
33		− if (type == UCALL_PIO)
34		− return;
35		−
36		− if (type == UCALL_MMIO) {
37		− vm_paddr_t gpa, start, end, step, offset;
38		− unsigned bits;
39		− bool ret;
40		−
41		− if (arg) {
42		− gpa = (vm_paddr_t)arg;
43		− ret = ucall_mmio_init(vm, gpa);
44		− TEST_ASSERT(ret, "Can't set ucall mmio address to %lx", gpa);
45		− return;
46		− }
47		−
48		− /*
49		− * Find an address within the allowed physical and virtual address
50		− * spaces, that does _not_ have a KVM memory region associated with
51		− * it. Identity mapping an address like this allows the guest to
52		− * access it, but as KVM doesn't know what to do with it, it
53		− * will assume it's something userspace handles and exit with
54		− * KVM_EXIT_MMIO. Well, at least that's how it works for AArch64.
55		− * Here we start with a guess that the addresses around 5/8th
56		− * of the allowed space are unmapped and then work both down and
57		− * up from there in 1/16th allowed space sized steps.
58		− *
59		− * Note, we need to use VA-bits - 1 when calculating the allowed
60		− * virtual address space for an identity mapping because the upper
61		− * half of the virtual address space is the two's complement of the
62		− * lower and won't match physical addresses.
63		− */
64		− bits = vm->va_bits - 1;
65		− bits = vm->pa_bits < bits ? vm->pa_bits : bits;
66		− end = 1ul << bits;
67		− start = end * 5 / 8;
68		− step = end / 16;
69		− for (offset = 0; offset < end - start; offset += step) {
70		− if (ucall_mmio_init(vm, start - offset))
71		− return;
72		− if (ucall_mmio_init(vm, start + offset))
73		− return;
74		− }
75		− TEST_ASSERT(false, "Can't find a ucall mmio address");
76		− }
77		−}
78		−
79		−void ucall_uninit(struct kvm_vm *vm)
80		−{
81		− ucall_type = 0;
82		− sync_global_to_guest(vm, ucall_type);
83		− ucall_exit_mmio_addr = 0;
84		− sync_global_to_guest(vm, ucall_exit_mmio_addr);
85		−}
86		−
87		−static void ucall_pio_exit(struct ucall *uc)
88		−{
89		−#ifdef __x86_64__
90		− asm volatile("in %[port], %%al"
91		− : : [port] "d" (UCALL_PIO_PORT), "D" (uc) : "rax");
92		−#endif
93		−}
94		−
95		−static void ucall_mmio_exit(struct ucall *uc)
96		−{
97		− *ucall_exit_mmio_addr = (vm_vaddr_t)uc;
98		−}
99		−
100		−void ucall(uint64_t cmd, int nargs, ...)
101		−{
102		− struct ucall uc = {
103		− .cmd = cmd,
104		− };
105		− va_list va;
106		− int i;
107		−
108		− nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
109		−
110		− va_start(va, nargs);
111		− for (i = 0; i < nargs; ++i)
112		− uc.args[i] = va_arg(va, uint64_t);
113		− va_end(va);
114		−
115		− switch (ucall_type) {
116		− case UCALL_PIO:
117		− ucall_pio_exit(&uc);
118		− break;
119		− case UCALL_MMIO:
120		− ucall_mmio_exit(&uc);
121		− break;
122		− };
123		−}
124		−
125		−uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
126		−{
127		− struct kvm_run *run = vcpu_state(vm, vcpu_id);
128		− struct ucall ucall = {};
129		− bool got_ucall = false;
130		−
131		−#ifdef __x86_64__
132		− if (ucall_type == UCALL_PIO && run->exit_reason == KVM_EXIT_IO &&
133		− run->io.port == UCALL_PIO_PORT) {
134		− struct kvm_regs regs;
135		− vcpu_regs_get(vm, vcpu_id, &regs);
136		− memcpy(&ucall, addr_gva2hva(vm, (vm_vaddr_t)regs.rdi), sizeof(ucall));
137		− got_ucall = true;
138		− }
139		−#endif
140		− if (ucall_type == UCALL_MMIO && run->exit_reason == KVM_EXIT_MMIO &&
141		− run->mmio.phys_addr == (uint64_t)ucall_exit_mmio_addr) {
142		− vm_vaddr_t gva;
143		− TEST_ASSERT(run->mmio.is_write && run->mmio.len == 8,
144		− "Unexpected ucall exit mmio address access");
145		− memcpy(&gva, run->mmio.data, sizeof(gva));
146		− memcpy(&ucall, addr_gva2hva(vm, gva), sizeof(ucall));
147		− got_ucall = true;
148		− }
149		−
150		− if (got_ucall) {
151		− vcpu_run_complete_io(vm, vcpu_id);
152		− if (uc)
153		− memcpy(uc, &ucall, sizeof(ucall));
154		− }
155		−
156		− return ucall.cmd;
157		−}

tools/testing/selftests/kvm/lib/x86_64/ucall.c+56−0

@@ -0,0 +1,56 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * ucall support. A ucall is a "hypercall to userspace".
	4	+ *
	5	+ * Copyright (C) 2018, Red Hat, Inc.
	6	+ */
	7	+#include "kvm_util.h"
	8	+
	9	+#define UCALL_PIO_PORT ((uint16_t)0x1000)
	10	+
	11	+void ucall_init(struct kvm_vm vm, void arg)
	12	+{
	13	+}
	14	+
	15	+void ucall_uninit(struct kvm_vm *vm)
	16	+{
	17	+}
	18	+
	19	+void ucall(uint64_t cmd, int nargs, ...)
	20	+{
	21	+ struct ucall uc = {
	22	+ .cmd = cmd,
	23	+ };
	24	+ va_list va;
	25	+ int i;
	26	+
	27	+ nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	28	+
	29	+ va_start(va, nargs);
	30	+ for (i = 0; i < nargs; ++i)
	31	+ uc.args[i] = va_arg(va, uint64_t);
	32	+ va_end(va);
	33	+
	34	+ asm volatile("in %[port], %%al"
	35	+ : : [port] "d" (UCALL_PIO_PORT), "D" (&uc) : "rax");
	36	+}
	37	+
	38	+uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	39	+{
	40	+ struct kvm_run *run = vcpu_state(vm, vcpu_id);
	41	+ struct ucall ucall = {};
	42	+
	43	+ if (run->exit_reason == KVM_EXIT_IO && run->io.port == UCALL_PIO_PORT) {
	44	+ struct kvm_regs regs;
	45	+
	46	+ vcpu_regs_get(vm, vcpu_id, &regs);
	47	+ memcpy(&ucall, addr_gva2hva(vm, (vm_vaddr_t)regs.rdi),
	48	+ sizeof(ucall));
	49	+
	50	+ vcpu_run_complete_io(vm, vcpu_id);
	51	+ if (uc)
	52	+ memcpy(uc, &ucall, sizeof(ucall));
	53	+ }
	54	+
	55	+ return ucall.cmd;
	56	+}

tools/testing/selftests/kvm/s390x/memop.c+166−0

@@ -0,0 +1,166 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+/*
	3	+ * Test for s390x KVM_S390_MEM_OP
	4	+ *
	5	+ * Copyright (C) 2019, Red Hat, Inc.
	6	+ */
	7	+
	8	+#include <stdio.h>
	9	+#include <stdlib.h>
	10	+#include <string.h>
	11	+#include <sys/ioctl.h>
	12	+
	13	+#include "test_util.h"
	14	+#include "kvm_util.h"
	15	+
	16	+#define VCPU_ID 1
	17	+
	18	+static uint8_t mem1[65536];
	19	+static uint8_t mem2[65536];
	20	+
	21	+static void guest_code(void)
	22	+{
	23	+ int i;
	24	+
	25	+ for (;;) {
	26	+ for (i = 0; i < sizeof(mem2); i++)
	27	+ mem2[i] = mem1[i];
	28	+ GUEST_SYNC(0);
	29	+ }
	30	+}
	31	+
	32	+int main(int argc, char *argv[])
	33	+{
	34	+ struct kvm_vm *vm;
	35	+ struct kvm_run *run;
	36	+ struct kvm_s390_mem_op ksmo;
	37	+ int rv, i, maxsize;
	38	+
	39	+ setbuf(stdout, NULL); /* Tell stdout not to buffer its content */
	40	+
	41	+ maxsize = kvm_check_cap(KVM_CAP_S390_MEM_OP);
	42	+ if (!maxsize) {
	43	+ fprintf(stderr, "CAP_S390_MEM_OP not supported -> skip test\n");
	44	+ exit(KSFT_SKIP);
	45	+ }
	46	+ if (maxsize > sizeof(mem1))
	47	+ maxsize = sizeof(mem1);
	48	+
	49	+ /* Create VM */
	50	+ vm = vm_create_default(VCPU_ID, 0, guest_code);
	51	+ run = vcpu_state(vm, VCPU_ID);
	52	+
	53	+ for (i = 0; i < sizeof(mem1); i++)
	54	+ mem1[i] = i * i + i;
	55	+
	56	+ /* Set the first array */
	57	+ ksmo.gaddr = addr_gva2gpa(vm, (uintptr_t)mem1);
	58	+ ksmo.flags = 0;
	59	+ ksmo.size = maxsize;
	60	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	61	+ ksmo.buf = (uintptr_t)mem1;
	62	+ ksmo.ar = 0;
	63	+ vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	64	+
	65	+ /* Let the guest code copy the first array to the second */
	66	+ vcpu_run(vm, VCPU_ID);
	67	+ TEST_ASSERT(run->exit_reason == KVM_EXIT_S390_SIEIC,
	68	+ "Unexpected exit reason: %u (%s)\n",
	69	+ run->exit_reason,
	70	+ exit_reason_str(run->exit_reason));
	71	+
	72	+ memset(mem2, 0xaa, sizeof(mem2));
	73	+
	74	+ /* Get the second array */
	75	+ ksmo.gaddr = (uintptr_t)mem2;
	76	+ ksmo.flags = 0;
	77	+ ksmo.size = maxsize;
	78	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_READ;
	79	+ ksmo.buf = (uintptr_t)mem2;
	80	+ ksmo.ar = 0;
	81	+ vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	82	+
	83	+ TEST_ASSERT(!memcmp(mem1, mem2, maxsize),
	84	+ "Memory contents do not match!");
	85	+
	86	+ /* Check error conditions - first bad size: */
	87	+ ksmo.gaddr = (uintptr_t)mem1;
	88	+ ksmo.flags = 0;
	89	+ ksmo.size = -1;
	90	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	91	+ ksmo.buf = (uintptr_t)mem1;
	92	+ ksmo.ar = 0;
	93	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	94	+ TEST_ASSERT(rv == -1 && errno == E2BIG, "ioctl allows insane sizes");
	95	+
	96	+ /* Zero size: */
	97	+ ksmo.gaddr = (uintptr_t)mem1;
	98	+ ksmo.flags = 0;
	99	+ ksmo.size = 0;
	100	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	101	+ ksmo.buf = (uintptr_t)mem1;
	102	+ ksmo.ar = 0;
	103	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	104	+ TEST_ASSERT(rv == -1 && (errno == EINVAL \|\| errno == ENOMEM),
	105	+ "ioctl allows 0 as size");
	106	+
	107	+ /* Bad flags: */
	108	+ ksmo.gaddr = (uintptr_t)mem1;
	109	+ ksmo.flags = -1;
	110	+ ksmo.size = maxsize;
	111	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	112	+ ksmo.buf = (uintptr_t)mem1;
	113	+ ksmo.ar = 0;
	114	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	115	+ TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows all flags");
	116	+
	117	+ /* Bad operation: */
	118	+ ksmo.gaddr = (uintptr_t)mem1;
	119	+ ksmo.flags = 0;
	120	+ ksmo.size = maxsize;
	121	+ ksmo.op = -1;
	122	+ ksmo.buf = (uintptr_t)mem1;
	123	+ ksmo.ar = 0;
	124	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	125	+ TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows bad operations");
	126	+
	127	+ /* Bad guest address: */
	128	+ ksmo.gaddr = ~0xfffUL;
	129	+ ksmo.flags = KVM_S390_MEMOP_F_CHECK_ONLY;
	130	+ ksmo.size = maxsize;
	131	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	132	+ ksmo.buf = (uintptr_t)mem1;
	133	+ ksmo.ar = 0;
	134	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	135	+ TEST_ASSERT(rv > 0, "ioctl does not report bad guest memory access");
	136	+
	137	+ /* Bad host address: */
	138	+ ksmo.gaddr = (uintptr_t)mem1;
	139	+ ksmo.flags = 0;
	140	+ ksmo.size = maxsize;
	141	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	142	+ ksmo.buf = 0;
	143	+ ksmo.ar = 0;
	144	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	145	+ TEST_ASSERT(rv == -1 && errno == EFAULT,
	146	+ "ioctl does not report bad host memory address");
	147	+
	148	+ /* Bad access register: */
	149	+ run->psw_mask &= ~(3UL << (63 - 17));
	150	+ run->psw_mask \|= 1UL << (63 - 17); /* Enable AR mode */
	151	+ vcpu_run(vm, VCPU_ID); /* To sync new state to SIE block */
	152	+ ksmo.gaddr = (uintptr_t)mem1;
	153	+ ksmo.flags = 0;
	154	+ ksmo.size = maxsize;
	155	+ ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	156	+ ksmo.buf = (uintptr_t)mem1;
	157	+ ksmo.ar = 17;
	158	+ rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	159	+ TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows ARs > 15");
	160	+ run->psw_mask &= ~(3UL << (63 - 17)); /* Disable AR mode */
	161	+ vcpu_run(vm, VCPU_ID); /* Run to sync new state */
	162	+
	163	+ kvm_vm_free(vm);
	164	+
	165	+ return 0;
	166	+}

tools/testing/selftests/kvm/s390x/sync_regs_test.c+34−2

@@ -25,9 +25,11 @@ · 24 unmodified lines
25	25
26	26	static void guest_code(void)
27	27	{
	28	+ register u64 stage asm("11") = 0;
	29	+
28	30	for (;;) {
29		− asm volatile ("diag 0,0,0x501");
30		− asm volatile ("ahi 11,1");
	31	+ GUEST_SYNC(0);
	32	+ asm volatile ("ahi %0,1" : : "r"(stage));
31	33	}
32	34	}
33	35
@@ -83,6 +85,36 @@ int main(int argc, char *argv[]) · 49 unmodified lines
83	85
84	86	run = vcpu_state(vm, VCPU_ID);
85	87
	88	+ /* Request reading invalid register set from VCPU. */
	89	+ run->kvm_valid_regs = INVALID_SYNC_FIELD;
	90	+ rv = _vcpu_run(vm, VCPU_ID);
	91	+ TEST_ASSERT(rv < 0 && errno == EINVAL,
	92	+ "Invalid kvm_valid_regs did not cause expected KVM_RUN error: %d\n",
	93	+ rv);
	94	+ vcpu_state(vm, VCPU_ID)->kvm_valid_regs = 0;
	95	+
	96	+ run->kvm_valid_regs = INVALID_SYNC_FIELD \| TEST_SYNC_FIELDS;
	97	+ rv = _vcpu_run(vm, VCPU_ID);
	98	+ TEST_ASSERT(rv < 0 && errno == EINVAL,
	99	+ "Invalid kvm_valid_regs did not cause expected KVM_RUN error: %d\n",
	100	+ rv);
	101	+ vcpu_state(vm, VCPU_ID)->kvm_valid_regs = 0;
	102	+
	103	+ /* Request setting invalid register set into VCPU. */
	104	+ run->kvm_dirty_regs = INVALID_SYNC_FIELD;
	105	+ rv = _vcpu_run(vm, VCPU_ID);
	106	+ TEST_ASSERT(rv < 0 && errno == EINVAL,
	107	+ "Invalid kvm_dirty_regs did not cause expected KVM_RUN error: %d\n",
	108	+ rv);
	109	+ vcpu_state(vm, VCPU_ID)->kvm_dirty_regs = 0;
	110	+
	111	+ run->kvm_dirty_regs = INVALID_SYNC_FIELD \| TEST_SYNC_FIELDS;
	112	+ rv = _vcpu_run(vm, VCPU_ID);
	113	+ TEST_ASSERT(rv < 0 && errno == EINVAL,
	114	+ "Invalid kvm_dirty_regs did not cause expected KVM_RUN error: %d\n",
	115	+ rv);
	116	+ vcpu_state(vm, VCPU_ID)->kvm_dirty_regs = 0;
	117	+
86	118	/* Request and verify all valid register sets. */
87	119	run->kvm_valid_regs = TEST_SYNC_FIELDS;
88	120	rv = _vcpu_run(vm, VCPU_ID);

virt/kvm/arm/arm.c+2−0

@@ -196,6 +196,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 195 unmodified lines
196	196	case KVM_CAP_MP_STATE:
197	197	case KVM_CAP_IMMEDIATE_EXIT:
198	198	case KVM_CAP_VCPU_EVENTS:
	199	+ case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
199	200	r = 1;
200	201	break;
201	202	case KVM_CAP_ARM_SET_DEVICE_ADDR:
@@ -888,6 +889,7 @@ int kvm_vm_ioctl_irq_line(struct kvm kvm, struct kvm_irq_level irq_level, · 686 unmodified lines
888	889
889	890	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
890	891	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
	892	+ vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
891	893	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
892	894
893	895	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);

virt/kvm/arm/vgic/vgic-init.c+6−2

@@ -54,6 +54,7 @@ void kvm_vgic_early_init(struct kvm *kvm) · 53 unmodified lines
54	54	struct vgic_dist *dist = &kvm->arch.vgic;
55	55
56	56	INIT_LIST_HEAD(&dist->lpi_list_head);
	57	+ INIT_LIST_HEAD(&dist->lpi_translation_cache);
57	58	raw_spin_lock_init(&dist->lpi_list_lock);
58	59	}
59	60
@@ -199,7 +200,6 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) · 139 unmodified lines
199	200	int i;
200	201
201	202	vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
202		− vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
203	203
204	204	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
205	205	raw_spin_lock_init(&vgic_cpu->ap_list_lock);
@@ -304,6 +304,7 @@ int vgic_init(struct kvm *kvm) · 98 unmodified lines
304	304	}
305	305
306	306	if (vgic_has_its(kvm)) {
	307	+ vgic_lpi_translation_cache_init(kvm);
307	308	ret = vgic_v4_init(kvm);
308	309	if (ret)
309	310	goto out;
@@ -345,6 +346,9 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) · 35 unmodified lines
345	346	INIT_LIST_HEAD(&dist->rd_regions);
346	347	}
347	348
	349	+ if (vgic_has_its(kvm))
	350	+ vgic_lpi_translation_cache_destroy(kvm);
	351	+
348	352	if (vgic_supports_direct_msis(kvm))
349	353	vgic_v4_teardown(kvm);
350	354	}
@@ -515,7 +519,7 @@ int kvm_vgic_hyp_init(void) · 164 unmodified lines
515	519	break;
516	520	default:
517	521	ret = -ENODEV;
518		− };
	522	+ }
519	523
520	524	if (ret)
521	525	return ret;

virt/kvm/arm/vgic/vgic-irqfd.c+30−6

@@ -66,6 +66,15 @@ out: · 65 unmodified lines
66	66	return r;
67	67	}
68	68
	69	+static void kvm_populate_msi(struct kvm_kernel_irq_routing_entry *e,
	70	+ struct kvm_msi *msi)
	71	+{
	72	+ msi->address_lo = e->msi.address_lo;
	73	+ msi->address_hi = e->msi.address_hi;
	74	+ msi->data = e->msi.data;
	75	+ msi->flags = e->msi.flags;
	76	+ msi->devid = e->msi.devid;
	77	+}
69	78	/**
70	79	* kvm_set_msi: inject the MSI corresponding to the
71	80	* MSI routing entry
@@ -79,21 +88,36 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, · 7 unmodified lines
79	88	{
80	89	struct kvm_msi msi;
81	90
82		− msi.address_lo = e->msi.address_lo;
83		− msi.address_hi = e->msi.address_hi;
84		− msi.data = e->msi.data;
85		− msi.flags = e->msi.flags;
86		− msi.devid = e->msi.devid;
87		−
88	91	if (!vgic_has_its(kvm))
89	92	return -ENODEV;
90	93
91	94	if (!level)
92	95	return -1;
93	96
	97	+ kvm_populate_msi(e, &msi);
94	98	return vgic_its_inject_msi(kvm, &msi);
95	99	}
96	100
	101	+/**
	102	+ * kvm_arch_set_irq_inatomic: fast-path for irqfd injection
	103	+ *
	104	+ * Currently only direct MSI injection is supported.
	105	+ */
	106	+int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
	107	+ struct kvm *kvm, int irq_source_id, int level,
	108	+ bool line_status)
	109	+{
	110	+ if (e->type == KVM_IRQ_ROUTING_MSI && vgic_has_its(kvm) && level) {
	111	+ struct kvm_msi msi;
	112	+
	113	+ kvm_populate_msi(e, &msi);
	114	+ if (!vgic_its_inject_cached_translation(kvm, &msi))
	115	+ return 0;
	116	+ }
	117	+
	118	+ return -EWOULDBLOCK;
	119	+}
	120	+
97	121	int kvm_vgic_setup_default_irq_routing(struct kvm *kvm)
98	122	{
99	123	struct kvm_irq_routing_entry *entries;

virt/kvm/arm/vgic/vgic-its.c+207−0

@@ -138,6 +138,14 @@ struct its_ite { · 137 unmodified lines
138	138	u32 event_id;
139	139	};
140	140
	141	+struct vgic_translation_cache_entry {
	142	+ struct list_head entry;
	143	+ phys_addr_t db;
	144	+ u32 devid;
	145	+ u32 eventid;
	146	+ struct vgic_irq *irq;
	147	+};
	148	+
141	149	/**
142	150	* struct vgic_its_abi - ITS abi ops and settings
143	151	* @cte_esz: collection table entry size
@@ -527,6 +535,127 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm, · 383 unmodified lines
527	535	return 0;
528	536	}
529	537
	538	+static struct vgic_irq __vgic_its_check_cache(struct vgic_dist dist,
	539	+ phys_addr_t db,
	540	+ u32 devid, u32 eventid)
	541	+{
	542	+ struct vgic_translation_cache_entry *cte;
	543	+
	544	+ list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
	545	+ /*
	546	+ * If we hit a NULL entry, there is nothing after this
	547	+ * point.
	548	+ */
	549	+ if (!cte->irq)
	550	+ break;
	551	+
	552	+ if (cte->db != db \|\| cte->devid != devid \|\|
	553	+ cte->eventid != eventid)
	554	+ continue;
	555	+
	556	+ /*
	557	+ * Move this entry to the head, as it is the most
	558	+ * recently used.
	559	+ */
	560	+ if (!list_is_first(&cte->entry, &dist->lpi_translation_cache))
	561	+ list_move(&cte->entry, &dist->lpi_translation_cache);
	562	+
	563	+ return cte->irq;
	564	+ }
	565	+
	566	+ return NULL;
	567	+}
	568	+
	569	+static struct vgic_irq vgic_its_check_cache(struct kvm kvm, phys_addr_t db,
	570	+ u32 devid, u32 eventid)
	571	+{
	572	+ struct vgic_dist *dist = &kvm->arch.vgic;
	573	+ struct vgic_irq *irq;
	574	+ unsigned long flags;
	575	+
	576	+ raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
	577	+ irq = __vgic_its_check_cache(dist, db, devid, eventid);
	578	+ raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
	579	+
	580	+ return irq;
	581	+}
	582	+
	583	+static void vgic_its_cache_translation(struct kvm kvm, struct vgic_its its,
	584	+ u32 devid, u32 eventid,
	585	+ struct vgic_irq *irq)
	586	+{
	587	+ struct vgic_dist *dist = &kvm->arch.vgic;
	588	+ struct vgic_translation_cache_entry *cte;
	589	+ unsigned long flags;
	590	+ phys_addr_t db;
	591	+
	592	+ /* Do not cache a directly injected interrupt */
	593	+ if (irq->hw)
	594	+ return;
	595	+
	596	+ raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
	597	+
	598	+ if (unlikely(list_empty(&dist->lpi_translation_cache)))
	599	+ goto out;
	600	+
	601	+ /*
	602	+ * We could have raced with another CPU caching the same
	603	+ * translation behind our back, so let's check it is not in
	604	+ * already
	605	+ */
	606	+ db = its->vgic_its_base + GITS_TRANSLATER;
	607	+ if (__vgic_its_check_cache(dist, db, devid, eventid))
	608	+ goto out;
	609	+
	610	+ /* Always reuse the last entry (LRU policy) */
	611	+ cte = list_last_entry(&dist->lpi_translation_cache,
	612	+ typeof(*cte), entry);
	613	+
	614	+ /*
	615	+ * Caching the translation implies having an extra reference
	616	+ * to the interrupt, so drop the potential reference on what
	617	+ * was in the cache, and increment it on the new interrupt.
	618	+ */
	619	+ if (cte->irq)
	620	+ __vgic_put_lpi_locked(kvm, cte->irq);
	621	+
	622	+ vgic_get_irq_kref(irq);
	623	+
	624	+ cte->db = db;
	625	+ cte->devid = devid;
	626	+ cte->eventid = eventid;
	627	+ cte->irq = irq;
	628	+
	629	+ /* Move the new translation to the head of the list */
	630	+ list_move(&cte->entry, &dist->lpi_translation_cache);
	631	+
	632	+out:
	633	+ raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
	634	+}
	635	+
	636	+void vgic_its_invalidate_cache(struct kvm *kvm)
	637	+{
	638	+ struct vgic_dist *dist = &kvm->arch.vgic;
	639	+ struct vgic_translation_cache_entry *cte;
	640	+ unsigned long flags;
	641	+
	642	+ raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
	643	+
	644	+ list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
	645	+ /*
	646	+ * If we hit a NULL entry, there is nothing after this
	647	+ * point.
	648	+ */
	649	+ if (!cte->irq)
	650	+ break;
	651	+
	652	+ __vgic_put_lpi_locked(kvm, cte->irq);
	653	+ cte->irq = NULL;
	654	+ }
	655	+
	656	+ raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
	657	+}
	658	+
530	659	int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its,
531	660	u32 devid, u32 eventid, struct vgic_irq **irq)
532	661	{
@@ -547,6 +676,8 @@ int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its, · 14 unmodified lines
547	676	if (!vcpu->arch.vgic_cpu.lpis_enabled)
548	677	return -EBUSY;
549	678
	679	+ vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
	680	+
550	681	*irq = ite->irq;
551	682	return 0;
552	683	}
@@ -608,6 +739,25 @@ static int vgic_its_trigger_msi(struct kvm kvm, struct vgic_its its, · 55 unmodified lines
608	739	return 0;
609	740	}
610	741
	742	+int vgic_its_inject_cached_translation(struct kvm kvm, struct kvm_msi msi)
	743	+{
	744	+ struct vgic_irq *irq;
	745	+ unsigned long flags;
	746	+ phys_addr_t db;
	747	+
	748	+ db = (u64)msi->address_hi << 32 \| msi->address_lo;
	749	+ irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
	750	+
	751	+ if (!irq)
	752	+ return -1;
	753	+
	754	+ raw_spin_lock_irqsave(&irq->irq_lock, flags);
	755	+ irq->pending_latch = true;
	756	+ vgic_queue_irq_unlock(kvm, irq, flags);
	757	+
	758	+ return 0;
	759	+}
	760	+
611	761	/*
612	762	* Queries the KVM IO bus framework to get the ITS pointer from the given
613	763	* doorbell address.
@@ -619,6 +769,9 @@ int vgic_its_inject_msi(struct kvm kvm, struct kvm_msi msi) · 5 unmodified lines
619	769	struct vgic_its *its;
620	770	int ret;
621	771
	772	+ if (!vgic_its_inject_cached_translation(kvm, msi))
	773	+ return 1;
	774	+
622	775	its = vgic_msi_to_its(kvm, msi);
623	776	if (IS_ERR(its))
624	777	return PTR_ERR(its);
@@ -691,6 +844,8 @@ static int vgic_its_cmd_handle_discard(struct kvm kvm, struct vgic_its its, · 66 unmodified lines
691	844	* don't bother here since we clear the ITTE anyway and the
692	845	* pending state is a property of the ITTE struct.
693	846	*/
	847	+ vgic_its_invalidate_cache(kvm);
	848	+
694	849	its_free_ite(kvm, ite);
695	850	return 0;
696	851	}
@@ -726,6 +881,8 @@ static int vgic_its_cmd_handle_movi(struct kvm kvm, struct vgic_its its, · 29 unmodified lines
726	881	ite->collection = collection;
727	882	vcpu = kvm_get_vcpu(kvm, collection->target_addr);
728	883
	884	+ vgic_its_invalidate_cache(kvm);
	885	+
729	886	return update_affinity(ite->irq, vcpu);
730	887	}
731	888
@@ -954,6 +1111,8 @@ static void vgic_its_free_device(struct kvm kvm, struct its_device device) · 222 unmodified lines
954	1111	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
955	1112	its_free_ite(kvm, ite);
956	1113
	1114	+ vgic_its_invalidate_cache(kvm);
	1115	+
957	1116	list_del(&device->dev_list);
958	1117	kfree(device);
959	1118	}
@@ -1059,6 +1218,7 @@ static int vgic_its_cmd_handle_mapc(struct kvm kvm, struct vgic_its its, · 99 unmodified lines
1059	1218
1060	1219	if (!valid) {
1061	1220	vgic_its_free_collection(its, coll_id);
	1221	+ vgic_its_invalidate_cache(kvm);
1062	1222	} else {
1063	1223	collection = find_collection(its, coll_id);
1064	1224
@@ -1207,6 +1367,8 @@ static int vgic_its_cmd_handle_movall(struct kvm kvm, struct vgic_its its, · 142 unmodified lines
1207	1367	vgic_put_irq(kvm, irq);
1208	1368	}
1209	1369
	1370	+ vgic_its_invalidate_cache(kvm);
	1371	+
1210	1372	kfree(intids);
1211	1373	return 0;
1212	1374	}
@@ -1557,6 +1719,8 @@ static void vgic_mmio_write_its_ctlr(struct kvm kvm, struct vgic_its its, · 344 unmodified lines
1557	1719	goto out;
1558	1720
1559	1721	its->enabled = !!(val & GITS_CTLR_ENABLE);
	1722	+ if (!its->enabled)
	1723	+ vgic_its_invalidate_cache(kvm);
1560	1724
1561	1725	/*
1562	1726	* Try to process any pending commands. This function bails out early
@@ -1657,6 +1821,47 @@ out: · 94 unmodified lines
1657	1821	return ret;
1658	1822	}
1659	1823
	1824	+/* Default is 16 cached LPIs per vcpu */
	1825	+#define LPI_DEFAULT_PCPU_CACHE_SIZE 16
	1826	+
	1827	+void vgic_lpi_translation_cache_init(struct kvm *kvm)
	1828	+{
	1829	+ struct vgic_dist *dist = &kvm->arch.vgic;
	1830	+ unsigned int sz;
	1831	+ int i;
	1832	+
	1833	+ if (!list_empty(&dist->lpi_translation_cache))
	1834	+ return;
	1835	+
	1836	+ sz = atomic_read(&kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;
	1837	+
	1838	+ for (i = 0; i < sz; i++) {
	1839	+ struct vgic_translation_cache_entry *cte;
	1840	+
	1841	+ /* An allocation failure is not fatal */
	1842	+ cte = kzalloc(sizeof(*cte), GFP_KERNEL);
	1843	+ if (WARN_ON(!cte))
	1844	+ break;
	1845	+
	1846	+ INIT_LIST_HEAD(&cte->entry);
	1847	+ list_add(&cte->entry, &dist->lpi_translation_cache);
	1848	+ }
	1849	+}
	1850	+
	1851	+void vgic_lpi_translation_cache_destroy(struct kvm *kvm)
	1852	+{
	1853	+ struct vgic_dist *dist = &kvm->arch.vgic;
	1854	+ struct vgic_translation_cache_entry cte, tmp;
	1855	+
	1856	+ vgic_its_invalidate_cache(kvm);
	1857	+
	1858	+ list_for_each_entry_safe(cte, tmp,
	1859	+ &dist->lpi_translation_cache, entry) {
	1860	+ list_del(&cte->entry);
	1861	+ kfree(cte);
	1862	+ }
	1863	+}
	1864	+
1660	1865	#define INITIAL_BASER_VALUE \
1661	1866	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) \| \
1662	1867	GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) \| \
@@ -1685,6 +1890,8 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) · 22 unmodified lines
1685	1890	kfree(its);
1686	1891	return ret;
1687	1892	}
	1893	+
	1894	+ vgic_lpi_translation_cache_init(dev->kvm);
1688	1895	}
1689	1896
1690	1897	mutex_init(&its->its_lock);

virt/kvm/arm/vgic/vgic-mmio-v3.c+27−58

@@ -192,8 +192,10 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, · 191 unmodified lines
192	192
193	193	vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
194	194
195		− if (was_enabled && !vgic_cpu->lpis_enabled)
	195	+ if (was_enabled && !vgic_cpu->lpis_enabled) {
196	196	vgic_flush_pending_lpis(vcpu);
	197	+ vgic_its_invalidate_cache(vcpu->kvm);
	198	+ }
197	199
198	200	if (!was_enabled && vgic_cpu->lpis_enabled)
199	201	vgic_enable_lpis(vcpu);
@@ -515,7 +517,8 @@ static const struct vgic_register_region vgic_v3_dist_registers[] = { · 315 unmodified lines
515	517	VGIC_ACCESS_32bit),
516	518	};
517	519
518		−static const struct vgic_register_region vgic_v3_rdbase_registers[] = {
	520	+static const struct vgic_register_region vgic_v3_rd_registers[] = {
	521	+ /* RD_base registers */
519	522	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
520	523	vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
521	524	VGIC_ACCESS_32bit),
@@ -540,44 +543,42 @@ static const struct vgic_register_region vgic_v3_rdbase_registers[] = { · 18 unmodified lines
540	543	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
541	544	vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
542	545	VGIC_ACCESS_32bit),
543		−};
544		−
545		−static const struct vgic_register_region vgic_v3_sgibase_registers[] = {
546		− REGISTER_DESC_WITH_LENGTH(GICR_IGROUPR0,
	546	+ /* SGI_base registers */
	547	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
547	548	vgic_mmio_read_group, vgic_mmio_write_group, 4,
548	549	VGIC_ACCESS_32bit),
549		− REGISTER_DESC_WITH_LENGTH(GICR_ISENABLER0,
	550	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ISENABLER0,
550	551	vgic_mmio_read_enable, vgic_mmio_write_senable, 4,
551	552	VGIC_ACCESS_32bit),
552		− REGISTER_DESC_WITH_LENGTH(GICR_ICENABLER0,
	553	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICENABLER0,
553	554	vgic_mmio_read_enable, vgic_mmio_write_cenable, 4,
554	555	VGIC_ACCESS_32bit),
555		− REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISPENDR0,
	556	+ REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
556	557	vgic_mmio_read_pending, vgic_mmio_write_spending,
557	558	vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
558	559	VGIC_ACCESS_32bit),
559		− REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICPENDR0,
	560	+ REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
560	561	vgic_mmio_read_pending, vgic_mmio_write_cpending,
561	562	vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
562	563	VGIC_ACCESS_32bit),
563		− REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISACTIVER0,
	564	+ REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
564	565	vgic_mmio_read_active, vgic_mmio_write_sactive,
565	566	NULL, vgic_mmio_uaccess_write_sactive,
566	567	4, VGIC_ACCESS_32bit),
567		− REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICACTIVER0,
	568	+ REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
568	569	vgic_mmio_read_active, vgic_mmio_write_cactive,
569	570	NULL, vgic_mmio_uaccess_write_cactive,
570	571	4, VGIC_ACCESS_32bit),
571		− REGISTER_DESC_WITH_LENGTH(GICR_IPRIORITYR0,
	572	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
572	573	vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
573	574	VGIC_ACCESS_32bit \| VGIC_ACCESS_8bit),
574		− REGISTER_DESC_WITH_LENGTH(GICR_ICFGR0,
	575	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
575	576	vgic_mmio_read_config, vgic_mmio_write_config, 8,
576	577	VGIC_ACCESS_32bit),
577		− REGISTER_DESC_WITH_LENGTH(GICR_IGRPMODR0,
	578	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
578	579	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
579	580	VGIC_ACCESS_32bit),
580		− REGISTER_DESC_WITH_LENGTH(GICR_NSACR,
	581	+ REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
581	582	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
582	583	VGIC_ACCESS_32bit),
583	584	};
@@ -607,9 +608,8 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) · 23 unmodified lines
607	608	struct vgic_dist *vgic = &kvm->arch.vgic;
608	609	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
609	610	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
610		− struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
611	611	struct vgic_redist_region *rdreg;
612		− gpa_t rd_base, sgi_base;
	612	+ gpa_t rd_base;
613	613	int ret;
614	614
615	615	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
@@ -631,52 +631,31 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) · 15 unmodified lines
631	631	vgic_cpu->rdreg = rdreg;
632	632
633	633	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
634		− sgi_base = rd_base + SZ_64K;
635	634
636	635	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
637	636	rd_dev->base_addr = rd_base;
638	637	rd_dev->iodev_type = IODEV_REDIST;
639		− rd_dev->regions = vgic_v3_rdbase_registers;
640		− rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);
	638	+ rd_dev->regions = vgic_v3_rd_registers;
	639	+ rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
641	640	rd_dev->redist_vcpu = vcpu;
642	641
643	642	mutex_lock(&kvm->slots_lock);
644	643	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
645		− SZ_64K, &rd_dev->dev);
	644	+ 2 * SZ_64K, &rd_dev->dev);
646	645	mutex_unlock(&kvm->slots_lock);
647	646
648	647	if (ret)
649	648	return ret;
650	649
651		− kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops);
652		− sgi_dev->base_addr = sgi_base;
653		− sgi_dev->iodev_type = IODEV_REDIST;
654		− sgi_dev->regions = vgic_v3_sgibase_registers;
655		− sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers);
656		− sgi_dev->redist_vcpu = vcpu;
657		−
658		− mutex_lock(&kvm->slots_lock);
659		− ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, sgi_base,
660		− SZ_64K, &sgi_dev->dev);
661		− if (ret) {
662		− kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
663		− &rd_dev->dev);
664		− goto out;
665		− }
666		−
667	650	rdreg->free_index++;
668		−out:
669		− mutex_unlock(&kvm->slots_lock);
670		− return ret;
	651	+ return 0;
671	652	}
672	653
673	654	static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
674	655	{
675	656	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
676		− struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
677	657
678	658	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
679		− kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &sgi_dev->dev);
680	659	}
681	660
682	661	static int vgic_register_all_redist_iodevs(struct kvm *kvm)
@@ -826,8 +805,8 @@ int vgic_v3_has_attr_regs(struct kvm_device dev, struct kvm_device_attr attr) · 143 unmodified lines
826	805	iodev.base_addr = 0;
827	806	break;
828	807	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
829		− iodev.regions = vgic_v3_rdbase_registers;
830		− iodev.nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);
	808	+ iodev.regions = vgic_v3_rd_registers;
	809	+ iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
831	810	iodev.base_addr = 0;
832	811	break;
833	812	}
@@ -985,21 +964,11 @@ int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, · 151 unmodified lines
985	964	int offset, u32 *val)
986	965	{
987	966	struct vgic_io_device rd_dev = {
988		− .regions = vgic_v3_rdbase_registers,
989		− .nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers),
	967	+ .regions = vgic_v3_rd_registers,
	968	+ .nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
990	969	};
991	970
992		− struct vgic_io_device sgi_dev = {
993		− .regions = vgic_v3_sgibase_registers,
994		− .nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers),
995		− };
996		−
997		− /* SGI_base is the next 64K frame after RD_base */
998		− if (offset >= SZ_64K)
999		− return vgic_uaccess(vcpu, &sgi_dev, is_write, offset - SZ_64K,
1000		− val);
1001		− else
1002		− return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
	971	+ return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1003	972	}
1004	973
1005	974	int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,

virt/kvm/arm/vgic/vgic-v2.c+4−3

@@ -357,10 +357,11 @@ out: · 356 unmodified lines
357	357	DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap);
358	358
359	359	/**
360		− * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT
361		− * @node: pointer to the DT node
	360	+ * vgic_v2_probe - probe for a VGICv2 compatible interrupt controller
	361	+ * @info: pointer to the GIC description
362	362	*
363		− * Returns 0 if a GICv2 has been found, returns an error code otherwise
	363	+ * Returns 0 if the VGICv2 has been probed successfully, returns an error code
	364	+ * otherwise
364	365	*/
365	366	int vgic_v2_probe(const struct gic_kvm_info *info)
366	367	{

virt/kvm/arm/vgic/vgic-v3.c+4−3

@@ -573,10 +573,11 @@ static int __init early_gicv4_enable(char *buf) · 572 unmodified lines
573	573	early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable);
574	574
575	575	/**
576		− * vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT
577		− * @node: pointer to the DT node
	576	+ * vgic_v3_probe - probe for a VGICv3 compatible interrupt controller
	577	+ * @info: pointer to the GIC description
578	578	*
579		− * Returns 0 if a GICv3 has been found, returns an error code otherwise
	579	+ * Returns 0 if the VGICv3 has been probed successfully, returns an error code
	580	+ * otherwise
580	581	*/
581	582	int vgic_v3_probe(const struct gic_kvm_info *info)
582	583	{

virt/kvm/arm/vgic/vgic.c+17−9

@@ -119,6 +119,22 @@ static void vgic_irq_release(struct kref *ref) · 118 unmodified lines
119	119	{
120	120	}
121	121
	122	+/*
	123	+ * Drop the refcount on the LPI. Must be called with lpi_list_lock held.
	124	+ */
	125	+void __vgic_put_lpi_locked(struct kvm kvm, struct vgic_irq irq)
	126	+{
	127	+ struct vgic_dist *dist = &kvm->arch.vgic;
	128	+
	129	+ if (!kref_put(&irq->refcount, vgic_irq_release))
	130	+ return;
	131	+
	132	+ list_del(&irq->lpi_list);
	133	+ dist->lpi_list_count--;
	134	+
	135	+ kfree(irq);
	136	+}
	137	+
122	138	void vgic_put_irq(struct kvm kvm, struct vgic_irq irq)
123	139	{
124	140	struct vgic_dist *dist = &kvm->arch.vgic;
@@ -128,16 +144,8 @@ void vgic_put_irq(struct kvm kvm, struct vgic_irq irq) · 3 unmodified lines
128	144	return;
129	145
130	146	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
131		− if (!kref_put(&irq->refcount, vgic_irq_release)) {
132		− raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
133		− return;
134		− };
135		−
136		− list_del(&irq->lpi_list);
137		− dist->lpi_list_count--;
	147	+ __vgic_put_lpi_locked(kvm, irq);
138	148	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
139		−
140		− kfree(irq);
141	149	}
142	150
143	151	void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)

virt/kvm/arm/vgic/vgic.h+5−0

@@ -161,6 +161,7 @@ vgic_get_mmio_region(struct kvm_vcpu vcpu, struct vgic_io_device iodev, · 160 unmodified lines
161	161	gpa_t addr, int len);
162	162	struct vgic_irq vgic_get_irq(struct kvm kvm, struct kvm_vcpu *vcpu,
163	163	u32 intid);
	164	+void __vgic_put_lpi_locked(struct kvm kvm, struct vgic_irq irq);
164	165	void vgic_put_irq(struct kvm kvm, struct vgic_irq irq);
165	166	bool vgic_get_phys_line_level(struct vgic_irq *irq);
166	167	void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);
@@ -307,6 +308,10 @@ int vgic_copy_lpi_list(struct kvm kvm, struct kvm_vcpu vcpu, u32 **intid_ptr); · 140 unmodified lines
307	308	int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its,
308	309	u32 devid, u32 eventid, struct vgic_irq **irq);
309	310	struct vgic_its vgic_msi_to_its(struct kvm kvm, struct kvm_msi *msi);
	311	+int vgic_its_inject_cached_translation(struct kvm kvm, struct kvm_msi msi);
	312	+void vgic_lpi_translation_cache_init(struct kvm *kvm);
	313	+void vgic_lpi_translation_cache_destroy(struct kvm *kvm);
	314	+void vgic_its_invalidate_cache(struct kvm *kvm);
310	315
311	316	bool vgic_supports_direct_msis(struct kvm *kvm);
312	317	int vgic_v4_init(struct kvm *kvm);

virt/kvm/kvm_main.c+3−4

@@ -2321,6 +2321,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 2320 unmodified lines
2321	2321	bool waited = false;
2322	2322	u64 block_ns;
2323	2323
	2324	+ kvm_arch_vcpu_blocking(vcpu);
	2325	+
2324	2326	start = cur = ktime_get();
2325	2327	if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
2326	2328	ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
@@ -2341,8 +2343,6 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 14 unmodified lines
2341	2343	} while (single_task_running() && ktime_before(cur, stop));
2342	2344	}
2343	2345
2344		− kvm_arch_vcpu_blocking(vcpu);
2345		−
2346	2346	for (;;) {
2347	2347	prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2348	2348
@@ -2355,9 +2355,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 6 unmodified lines
2355	2355
2356	2356	finish_swait(&vcpu->wq, &wait);
2357	2357	cur = ktime_get();
2358		−
2359		− kvm_arch_vcpu_unblocking(vcpu);
2360	2358	out:
	2359	+ kvm_arch_vcpu_unblocking(vcpu);
2361	2360	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2362	2361
2363	2362	if (!vcpu_valid_wakeup(vcpu))

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code