lael-labs/linux

Documentation/virt/kvm/api.txt+24−9

 Architectures: x86
 Type: vcpu ioctl
 Parameters: struct kvm_msrs (in)
-Returns: 0 on success, -1 on error
+Returns: number of msrs successfully set (see below), -1 on error
 Writes model-specific registers to the vcpu.  See KVM_GET_MSRS for the
 data structures.
 size of the entries array), and the 'index' and 'data' members of each
 array entry.
+It tries to set the MSRs in array entries[] one by one. If setting an MSR
+fails, e.g., due to setting reserved bits, the MSR isn't supported/emulated
+by KVM, etc..., it stops processing the MSR list and returns the number of
+MSRs that have been set successfully.
 4.20 KVM_SET_CPUID
 use PPIs designated for specific cpus.  The irq field is interpreted
 like this:
-  bits:  | 31 ... 24 | 23  ... 16 | 15    ...    0 |
+  bits:  |  31 ... 28  | 27 ... 24 | 23  ... 16 | 15 ... 0 |
-  field: | irq_type  | vcpu_index |     irq_id     |
+  field: | vcpu2_index | irq_type  | vcpu_index |  irq_id  |
 The irq_type field has the following values:
 - irq_type[0]: out-of-kernel GIC: irq_id 0 is IRQ, irq_id 1 is FIQ
 In both cases, level is used to assert/deassert the line.
+When KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 is supported, the target vcpu is
+identified as (256 * vcpu2_index + vcpu_index). Otherwise, vcpu2_index
+must be zero.
+Note that on arm/arm64, the KVM_CAP_IRQCHIP capability only conditions
+injection of interrupts for the in-kernel irqchip. KVM_IRQ_LINE can always
+be used for a userspace interrupt controller.
 struct kvm_irq_level {
 	union {
 		__u32 irq;     /* GSI */
 flag KVM_S390_MEMOP_F_INJECT_EXCEPTION is set in the "flags" field.
 The start address of the memory region has to be specified in the "gaddr"
-field, and the length of the region in the "size" field. "buf" is the buffer
+field, and the length of the region in the "size" field (which must not
-supplied by the userspace application where the read data should be written
+be 0). The maximum value for "size" can be obtained by checking the
-to for KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written
+KVM_CAP_S390_MEM_OP capability. "buf" is the buffer supplied by the
-is stored for a KVM_S390_MEMOP_LOGICAL_WRITE. "buf" is unused and can be NULL
+userspace application where the read data should be written to for
-when KVM_S390_MEMOP_F_CHECK_ONLY is specified. "ar" designates the access
+KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written is
-register number to be used.
+stored for a KVM_S390_MEMOP_LOGICAL_WRITE. When KVM_S390_MEMOP_F_CHECK_ONLY
+is specified, "buf" is unused and can be NULL. "ar" designates the access
+register number to be used; the valid range is 0..15.
 The "reserved" field is meant for future extensions. It is not used by
 KVM with the currently defined set of flags.

Documentation/virt/kvm/mmu.txt+2−2

arch/arm/include/uapi/asm/kvm.h+3−1

 #define   KVM_DEV_ARM_ITS_CTRL_RESET		4
 /* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_VCPU2_SHIFT		28
+#define KVM_ARM_IRQ_VCPU2_MASK		0xf
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
-#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_TYPE_MASK		0xf
 #define KVM_ARM_IRQ_VCPU_SHIFT		16
 #define KVM_ARM_IRQ_VCPU_MASK		0xff
 #define KVM_ARM_IRQ_NUM_SHIFT		0

arch/arm64/include/asm/pgtable-prot.h+1−1

@@ -77,7 +77,7 @@ · 76 unmodified lines
77	})	77	})
78		78
79	#define PAGE_S2 __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(NORMAL) \| PTE_S2_RDONLY \| PAGE_S2_XN)	79	#define PAGE_S2 __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(NORMAL) \| PTE_S2_RDONLY \| PAGE_S2_XN)
80	#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(DEVICE_nGnRE) \| PTE_S2_RDONLY \| PAGE_S2_XN)	80	#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT \| PAGE_S2_MEMATTR(DEVICE_nGnRE) \| PTE_S2_RDONLY \| PTE_S2_XN)
81		81
82	#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) \| PTE_PROT_NONE \| PTE_RDONLY \| PTE_NG \| PTE_PXN \| PTE_UXN)	82	#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) \| PTE_PROT_NONE \| PTE_RDONLY \| PTE_NG \| PTE_PXN \| PTE_UXN)
83	#define PAGE_SHARED __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN \| PTE_WRITE)	83	#define PAGE_SHARED __pgprot(_PAGE_DEFAULT \| PTE_USER \| PTE_NG \| PTE_PXN \| PTE_UXN \| PTE_WRITE)

arch/arm64/include/uapi/asm/kvm.h+3−1

 #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
 /* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_VCPU2_SHIFT		28
+#define KVM_ARM_IRQ_VCPU2_MASK		0xf
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
-#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_TYPE_MASK		0xf
 #define KVM_ARM_IRQ_VCPU_SHIFT		16
 #define KVM_ARM_IRQ_VCPU_MASK		0xff
 #define KVM_ARM_IRQ_NUM_SHIFT		0

arch/arm64/kvm/hyp/tlb.c+13−1

@@ -193,6 +193,18 @@ void __hyp_text __kvm_flush_vm_context(void) · 192 unmodified lines
193	{	193	{
194	dsb(ishst);	194	dsb(ishst);
195	__tlbi(alle1is);	195	__tlbi(alle1is);
196	asm volatile("ic ialluis" : : );	196
		197	/*
		198	* VIPT and PIPT caches are not affected by VMID, so no maintenance
		199	* is necessary across a VMID rollover.
		200	*
		201	* VPIPT caches constrain lookup and maintenance to the active VMID,
		202	* so we need to invalidate lines with a stale VMID to avoid an ABA
		203	* race after multiple rollovers.
		204	*
		205	*/
		206	if (icache_is_vpipt())
		207	asm volatile("ic ialluis");
		208
197	dsb(ish);	209	dsb(ish);
198	}	210	}

arch/powerpc/include/asm/kvm_host.h+18−4

@@ -232,11 +232,25 @@ struct revmap_entry { · 231 unmodified lines
232	};	232	};
233		233
234	/*	234	/*
235	* We use the top bit of each memslot->arch.rmap entry as a lock bit,	235	* The rmap array of size number of guest pages is allocated for each memslot.
236	* and bit 32 as a present flag. The bottom 32 bits are the	236	* This array is used to store usage specific information about the guest page.
237	* index in the guest HPT of a HPTE that points to the page.	237	* Below are the encodings of the various possible usage types.
238	*/	238	*/
239	#define KVMPPC_RMAP_LOCK_BIT 63	239	/* Free bits which can be used to define a new usage */
		240	#define KVMPPC_RMAP_TYPE_MASK 0xff00000000000000
		241	#define KVMPPC_RMAP_NESTED 0xc000000000000000 /* Nested rmap array */
		242	#define KVMPPC_RMAP_HPT 0x0100000000000000 /* HPT guest */
		243
		244	/*
		245	* rmap usage definition for a hash page table (hpt) guest:
		246	* 0x0000080000000000 Lock bit
		247	* 0x0000018000000000 RC bits
		248	* 0x0000000100000000 Present bit
		249	* 0x00000000ffffffff HPT index bits
		250	* The bottom 32 bits are the index in the guest HPT of a HPTE that points to
		251	* the page.
		252	*/
		253	#define KVMPPC_RMAP_LOCK_BIT 43
240	#define KVMPPC_RMAP_RC_SHIFT 32	254	#define KVMPPC_RMAP_RC_SHIFT 32
241	#define KVMPPC_RMAP_REFERENCED (HPTE_R_R << KVMPPC_RMAP_RC_SHIFT)	255	#define KVMPPC_RMAP_REFERENCED (HPTE_R_R << KVMPPC_RMAP_RC_SHIFT)
242	#define KVMPPC_RMAP_PRESENT 0x100000000ul	256	#define KVMPPC_RMAP_PRESENT 0x100000000ul

arch/powerpc/include/asm/kvm_ppc.h+1−0

@@ -598,6 +598,7 @@ extern int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, · 597 unmodified lines
598	union kvmppc_one_reg *val);	598	union kvmppc_one_reg *val);
599	extern int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu,	599	extern int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu,
600	union kvmppc_one_reg *val);	600	union kvmppc_one_reg *val);
		601	extern bool kvmppc_xive_native_supported(void);
601		602
602	#else	603	#else
603	static inline int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,	604	static inline int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,

arch/powerpc/include/asm/xive.h+9−0

 	/* Setup/used by frontend */
 	int target;
+	/*
+	 * saved_p means that there is a queue entry for this interrupt
+	 * in some CPU's queue (not including guest vcpu queues), even
+	 * if P is not set in the source ESB.
+	 * stale_p means that there is no queue entry for this interrupt
+	 * in some CPU's queue, even if P is set in the source ESB.
+	 */
 	bool saved_p;
+	bool stale_p;
 };
 #define XIVE_IRQ_FLAG_STORE_EOI	0x01
 #define XIVE_IRQ_FLAG_LSI	0x02
 extern int xive_native_set_queue_state(u32 vp_id, uint32_t prio, u32 qtoggle,
 				       u32 qindex);
 extern int xive_native_get_vp_state(u32 vp_id, u64 *out_state);
+extern bool xive_native_has_queue_state_support(void);
 #else

arch/powerpc/kvm/book3s.c+5−3

 	if (xics_on_xive()) {
 		kvmppc_xive_init_module();
 		kvm_register_device_ops(&kvm_xive_ops, KVM_DEV_TYPE_XICS);
-		kvmppc_xive_native_init_module();
+		if (kvmppc_xive_native_supported()) {
-		kvm_register_device_ops(&kvm_xive_native_ops,
+			kvmppc_xive_native_init_module();
-					KVM_DEV_TYPE_XIVE);
+			kvm_register_device_ops(&kvm_xive_native_ops,
+						KVM_DEV_TYPE_XIVE);
+		}
 	} else
 #endif
 		kvm_register_device_ops(&kvm_xics_ops, KVM_DEV_TYPE_XICS);

arch/powerpc/kvm/book3s_hv.c+18−6

 		*val = get_reg_val(id, vcpu->arch.pspb);
 		break;
 	case KVM_REG_PPC_DPDES:
-		*val = get_reg_val(id, vcpu->arch.vcore->dpdes);
+		/*
+		 * On POWER9, where we are emulating msgsndp etc.,
+		 * we return 1 bit for each vcpu, which can come from
+		 * either vcore->dpdes or doorbell_request.
+		 * On POWER8, doorbell_request is 0.
+		 */
+		*val = get_reg_val(id, vcpu->arch.vcore->dpdes |
+				   vcpu->arch.doorbell_request);
 		break;
 	case KVM_REG_PPC_VTB:
 		*val = get_reg_val(id, vcpu->arch.vcore->vtb);
 		if (!spin_trylock(&pvc->lock))
 			continue;
 		prepare_threads(pvc);
-		if (!pvc->n_runnable) {
+		if (!pvc->n_runnable || !pvc->kvm->arch.mmu_ready) {
 			list_del_init(&pvc->preempt_list);
 			if (pvc->runner == NULL) {
 				pvc->vcore_state = VCORE_INACTIVE;
 	spin_unlock(&lp->lock);
 }
-static bool recheck_signals(struct core_info *cip)
+static bool recheck_signals_and_mmu(struct core_info *cip)
 {
 	int sub, i;
 	struct kvm_vcpu *vcpu;
+	struct kvmppc_vcore *vc;
-	for (sub = 0; sub < cip->n_subcores; ++sub)
+	for (sub = 0; sub < cip->n_subcores; ++sub) {
-		for_each_runnable_thread(i, vcpu, cip->vc[sub])
+		vc = cip->vc[sub];
+		if (!vc->kvm->arch.mmu_ready)
+			return true;
+		for_each_runnable_thread(i, vcpu, vc)
 			if (signal_pending(vcpu->arch.run_task))
 				return true;
+	}
 	return false;
 }
 	local_irq_disable();
 	hard_irq_disable();
 	if (lazy_irq_pending() || need_resched() ||
-	    recheck_signals(&core_info) || !vc->kvm->arch.mmu_ready) {
+	    recheck_signals_and_mmu(&core_info)) {
 		local_irq_enable();
 		vc->vcore_state = VCORE_INACTIVE;
 		/* Unlock all except the primary vcore */

arch/powerpc/kvm/book3s_hv_rm_mmu.c+1−1

@@ -99,7 +99,7 @@ void kvmppc_add_revmap_chain(struct kvm kvm, struct revmap_entry rev, · 98 unmodified lines
99	} else {	99	} else {
100	rev->forw = rev->back = pte_index;	100	rev->forw = rev->back = pte_index;
101	rmap = (rmap & ~KVMPPC_RMAP_INDEX) \|	101	rmap = (rmap & ~KVMPPC_RMAP_INDEX) \|
102	pte_index \| KVMPPC_RMAP_PRESENT;	102	pte_index \| KVMPPC_RMAP_PRESENT \| KVMPPC_RMAP_HPT;
103	}	103	}
104	unlock_rmap(rmap);	104	unlock_rmap(rmap);
105	}	105	}

arch/powerpc/kvm/book3s_hv_rmhandlers.S+25−13

 	ld	r11, VCPU_XIVE_SAVED_STATE(r4)
 	li	r9, TM_QW1_OS
 	lwz	r8, VCPU_XIVE_CAM_WORD(r4)
+	cmpwi	r8, 0
+	beq	no_xive
 	li	r7, TM_QW1_OS + TM_WORD2
 	mfmsr	r0
 	andi.	r0, r0, MSR_DR		/* in real mode? */
 kvm_cede_exit:
 	ld	r9, HSTATE_KVM_VCPU(r13)
 #ifdef CONFIG_KVM_XICS
-	/* Abort if we still have a pending escalation */
+	/* are we using XIVE with single escalation? */
+	ld	r10, VCPU_XIVE_ESC_VADDR(r9)
+	cmpdi	r10, 0
+	beq	3f
+	li	r6, XIVE_ESB_SET_PQ_00
+	/*
+	 * If we still have a pending escalation, abort the cede,
+	 * and we must set PQ to 10 rather than 00 so that we don't
+	 * potentially end up with two entries for the escalation
+	 * interrupt in the XIVE interrupt queue.  In that case
+	 * we also don't want to set xive_esc_on to 1 here in
+	 * case we race with xive_esc_irq().
+	 */
 	lbz	r5, VCPU_XIVE_ESC_ON(r9)
 	cmpwi	r5, 0
-	beq	1f
+	beq	4f
 	li	r0, 0
 	stb	r0, VCPU_CEDED(r9)
-1:	/* Enable XIVE escalation */
+	li	r6, XIVE_ESB_SET_PQ_10
-	li	r5, XIVE_ESB_SET_PQ_00
+	b	5f
+4:	li	r0, 1
+	stb	r0, VCPU_XIVE_ESC_ON(r9)
+	/* make sure store to xive_esc_on is seen before xive_esc_irq runs */
+	sync
+5:	/* Enable XIVE escalation */
 	mfmsr	r0
 	andi.	r0, r0, MSR_DR		/* in real mode? */
 	beq	1f
-	ld	r10, VCPU_XIVE_ESC_VADDR(r9)
+	ldx	r0, r10, r6
-	cmpdi	r10, 0
-	beq	3f
-	ldx	r0, r10, r5
 	b	2f
 1:	ld	r10, VCPU_XIVE_ESC_RADDR(r9)
-	cmpdi	r10, 0
+	ldcix	r0, r10, r6
-	beq	3f
-	ldcix	r0, r10, r5
 2:	sync
-	li	r0, 1
-	stb	r0, VCPU_XIVE_ESC_ON(r9)
 #endif /* CONFIG_KVM_XICS */
 3:	b	guest_exit_cont

arch/powerpc/kvm/book3s_xive.c+51−9

@@ -67,8 +67,14 @@ void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) · 66 unmodified lines
67	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;	67	void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
68	u64 pq;	68	u64 pq;
69		69
70	if (!tima)	70	/*
		71	* Nothing to do if the platform doesn't have a XIVE
		72	* or this vCPU doesn't have its own XIVE context
		73	* (e.g. because it's not using an in-kernel interrupt controller).
		74	*/
		75	if (!tima \|\| !vcpu->arch.xive_cam_word)
71	return;	76	return;
		77
72	eieio();	78	eieio();
73	__raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);	79	__raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
74	__raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);	80	__raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
@@ -160,6 +166,9 @@ static irqreturn_t xive_esc_irq(int irq, void *data) · 85 unmodified lines
160	*/	166	*/
161	vcpu->arch.xive_esc_on = false;	167	vcpu->arch.xive_esc_on = false;
162		168
		169	/* This orders xive_esc_on = false vs. subsequent stale_p = true */
		170	smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
		171
163	return IRQ_HANDLED;	172	return IRQ_HANDLED;
164	}	173	}
165		174
@@ -1113,6 +1122,31 @@ void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu) · 947 unmodified lines
1113	vcpu->arch.xive_esc_raddr = 0;	1122	vcpu->arch.xive_esc_raddr = 0;
1114	}	1123	}
1115		1124
		1125	/*
		1126	* In single escalation mode, the escalation interrupt is marked so
		1127	* that EOI doesn't re-enable it, but just sets the stale_p flag to
		1128	* indicate that the P bit has already been dealt with. However, the
		1129	* assembly code that enters the guest sets PQ to 00 without clearing
		1130	* stale_p (because it has no easy way to address it). Hence we have
		1131	* to adjust stale_p before shutting down the interrupt.
		1132	*/
		1133	void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
		1134	struct kvmppc_xive_vcpu *xc, int irq)
		1135	{
		1136	struct irq_data *d = irq_get_irq_data(irq);
		1137	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
		1138
		1139	/*
		1140	* This slightly odd sequence gives the right result
		1141	* (i.e. stale_p set if xive_esc_on is false) even if
		1142	* we race with xive_esc_irq() and xive_irq_eoi().
		1143	*/
		1144	xd->stale_p = false;
		1145	smp_mb(); /* paired with smb_wmb in xive_esc_irq */
		1146	if (!vcpu->arch.xive_esc_on)
		1147	xd->stale_p = true;
		1148	}
		1149
1116	void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)	1150	void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1117	{	1151	{
1118	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;	1152	struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
@@ -1134,20 +1168,28 @@ void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu) · 15 unmodified lines
1134	/* Mask the VP IPI */	1168	/* Mask the VP IPI */
1135	xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);	1169	xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1136		1170
1137	/* Disable the VP */	1171	/* Free escalations */
1138	xive_native_disable_vp(xc->vp_id);
1139
1140	/* Free the queues & associated interrupts */
1141	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {	1172	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1142	struct xive_q *q = &xc->queues[i];
1143
1144	/* Free the escalation irq */
1145	if (xc->esc_virq[i]) {	1173	if (xc->esc_virq[i]) {
		1174	if (xc->xive->single_escalation)
		1175	xive_cleanup_single_escalation(vcpu, xc,
		1176	xc->esc_virq[i]);
1146	free_irq(xc->esc_virq[i], vcpu);	1177	free_irq(xc->esc_virq[i], vcpu);
1147	irq_dispose_mapping(xc->esc_virq[i]);	1178	irq_dispose_mapping(xc->esc_virq[i]);
1148	kfree(xc->esc_virq_names[i]);	1179	kfree(xc->esc_virq_names[i]);
1149	}	1180	}
1150	/* Free the queue */	1181	}
		1182
		1183	/* Disable the VP */
		1184	xive_native_disable_vp(xc->vp_id);
		1185
		1186	/* Clear the cam word so guest entry won't try to push context */
		1187	vcpu->arch.xive_cam_word = 0;
		1188
		1189	/* Free the queues */
		1190	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
		1191	struct xive_q *q = &xc->queues[i];
		1192
1151	xive_native_disable_queue(xc->vp_id, q, i);	1193	xive_native_disable_queue(xc->vp_id, q, i);
1152	if (q->qpage) {	1194	if (q->qpage) {
1153	free_pages((unsigned long)q->qpage,	1195	free_pages((unsigned long)q->qpage,

arch/powerpc/kvm/book3s_xive.h+2−0

@@ -282,6 +282,8 @@ int kvmppc_xive_select_target(struct kvm kvm, u32 server, u8 prio); · 281 unmodified lines
282	int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,	282	int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
283	bool single_escalation);	283	bool single_escalation);
284	struct kvmppc_xive kvmppc_xive_get_device(struct kvm kvm, u32 type);	284	struct kvmppc_xive kvmppc_xive_get_device(struct kvm kvm, u32 type);
		285	void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
		286	struct kvmppc_xive_vcpu *xc, int irq);
285		287
286	#endif /* CONFIG_KVM_XICS */	288	#endif /* CONFIG_KVM_XICS */
287	#endif /* _KVM_PPC_BOOK3S_XICS_H */	289	#endif /* _KVM_PPC_BOOK3S_XICS_H */

arch/powerpc/kvm/book3s_xive_native.c+18−5

 	xc->valid = false;
 	kvmppc_xive_disable_vcpu_interrupts(vcpu);
-	/* Disable the VP */
+	/* Free escalations */
-	xive_native_disable_vp(xc->vp_id);
-	/* Free the queues & associated interrupts */
 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
 		/* Free the escalation irq */
 		if (xc->esc_virq[i]) {
+			if (xc->xive->single_escalation)
+				xive_cleanup_single_escalation(vcpu, xc,
+							xc->esc_virq[i]);
 			free_irq(xc->esc_virq[i], vcpu);
 			irq_dispose_mapping(xc->esc_virq[i]);
 			kfree(xc->esc_virq_names[i]);
 			xc->esc_virq[i] = 0;
 		}
+	}
+	/* Disable the VP */
+	xive_native_disable_vp(xc->vp_id);
+	/* Clear the cam word so guest entry won't try to push context */
+	vcpu->arch.xive_cam_word = 0;
-		/* Free the queue */
+	/* Free the queues */
+	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
 		kvmppc_xive_native_cleanup_queue(vcpu, i);
 	}
 	return 0;
 }
+bool kvmppc_xive_native_supported(void)
+{
+	return xive_native_has_queue_state_support();
+}
 static int xive_native_debug_show(struct seq_file *m, void *private)
 {
 	struct kvmppc_xive *xive = m->private;

arch/powerpc/kvm/e500.c+3−0

@@ -440,6 +440,9 @@ static struct kvm_vcpu kvmppc_core_vcpu_create_e500(struct kvm kvm, · 439 unmodified lines
440	struct kvm_vcpu *vcpu;	440	struct kvm_vcpu *vcpu;
441	int err;	441	int err;
442		442
		443	BUILD_BUG_ON_MSG(offsetof(struct kvmppc_vcpu_e500, vcpu) != 0,
		444	"struct kvm_vcpu must be at offset 0 for arch usercopy region");
		445
443	vcpu_e500 = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);	446	vcpu_e500 = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
444	if (!vcpu_e500) {	447	if (!vcpu_e500) {
445	err = -ENOMEM;	448	err = -ENOMEM;

arch/powerpc/kvm/emulate.c+1−0

@@ -271,6 +271,7 @@ int kvmppc_emulate_instruction(struct kvm_run run, struct kvm_vcpu vcpu) · 270 unmodified lines
271	*/	271	*/
272	if (inst == KVMPPC_INST_SW_BREAKPOINT) {	272	if (inst == KVMPPC_INST_SW_BREAKPOINT) {
273	run->exit_reason = KVM_EXIT_DEBUG;	273	run->exit_reason = KVM_EXIT_DEBUG;
		274	run->debug.arch.status = 0;
274	run->debug.arch.address = kvmppc_get_pc(vcpu);	275	run->debug.arch.address = kvmppc_get_pc(vcpu);
275	emulated = EMULATE_EXIT_USER;	276	emulated = EMULATE_EXIT_USER;
276	advance = 0;	277	advance = 0;

arch/powerpc/kvm/emulate_loadstore.c+0−6

 	rs = get_rs(inst);
 	rt = get_rt(inst);
-	/*
-	 * if mmio_vsx_tx_sx_enabled == 0, copy data between
-	 * VSR[0..31] and memory
-	 * if mmio_vsx_tx_sx_enabled == 1, copy data between
-	 * VSR[32..63] and memory
-	 */
 	vcpu->arch.mmio_vsx_copy_nums = 0;
 	vcpu->arch.mmio_vsx_offset = 0;
 	vcpu->arch.mmio_copy_type = KVMPPC_VSX_COPY_NONE;

arch/powerpc/kvm/powerpc.c+2−1

@@ -561,7 +561,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 560 unmodified lines
561	* a POWER9 processor) and the PowerNV platform, as	561	* a POWER9 processor) and the PowerNV platform, as
562	* nested is not yet supported.	562	* nested is not yet supported.
563	*/	563	*/
564	r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE);	564	r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE) &&
		565	kvmppc_xive_native_supported();
565	break;	566	break;
566	#endif	567	#endif
567		568

arch/powerpc/sysdev/xive/common.c+64−23

 static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek)
 {
 	u32 irq = 0;
-	u8 prio;
+	u8 prio = 0;
 	/* Find highest pending priority */
 	while (xc->pending_prio != 0) {
 		irq = xive_read_eq(&xc->queue[prio], just_peek);
 		/* Found something ? That's it */
-		if (irq)
+		if (irq) {
-			break;
+			if (just_peek || irq_to_desc(irq))
+				break;
+			/*
+			 * We should never get here; if we do then we must
+			 * have failed to synchronize the interrupt properly
+			 * when shutting it down.
+			 */
+			pr_crit("xive: got interrupt %d without descriptor, dropping\n",
+				irq);
+			WARN_ON(1);
+			continue;
+		}
 		/* Clear pending bits */
 		xc->pending_prio &= ~(1 << prio);
  */
 static void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd)
 {
+	xd->stale_p = false;
 	/* If the XIVE supports the new "store EOI facility, use it */
 	if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI)
 		xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);
 	}
 }
-/* irq_chip eoi callback */
+/* irq_chip eoi callback, called with irq descriptor lock held */
 static void xive_irq_eoi(struct irq_data *d)
 {
 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
 	if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) &&
 	    !(xd->flags & XIVE_IRQ_NO_EOI))
 		xive_do_source_eoi(irqd_to_hwirq(d), xd);
+	else
+		xd->stale_p = true;
 	/*
 	 * Clear saved_p to indicate that it's no longer occupying
 	 */
 	if (mask) {
 		val = xive_esb_read(xd, XIVE_ESB_SET_PQ_01);
-		xd->saved_p = !!(val & XIVE_ESB_VAL_P);
+		if (!xd->stale_p && !!(val & XIVE_ESB_VAL_P))
-	} else if (xd->saved_p)
+			xd->saved_p = true;
+		xd->stale_p = false;
+	} else if (xd->saved_p) {
 		xive_esb_read(xd, XIVE_ESB_SET_PQ_10);
-	else
+		xd->saved_p = false;
+	} else {
 		xive_esb_read(xd, XIVE_ESB_SET_PQ_00);
+		xd->stale_p = false;
+	}
 }
 /*
 	unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
 	int target, rc;
+	xd->saved_p = false;
+	xd->stale_p = false;
 	pr_devel("xive_irq_startup: irq %d [0x%x] data @%p\n",
 		 d->irq, hw_irq, d);
 	return 0;
 }
+/* called with irq descriptor lock held */
 static void xive_irq_shutdown(struct irq_data *d)
 {
 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
 	/* Mask the interrupt at the source */
 	xive_do_source_set_mask(xd, true);
-	/*
-	 * The above may have set saved_p. We clear it otherwise it
-	 * will prevent re-enabling later on. It is ok to forget the
-	 * fact that the interrupt might be in a queue because we are
-	 * accounting that already in xive_dec_target_count() and will
-	 * be re-routing it to a new queue with proper accounting when
-	 * it's started up again
-	 */
-	xd->saved_p = false;
 	/*
 	 * Mask the interrupt in HW in the IVT/EAS and set the number
 	 * to be the "bad" IRQ number
 	return 1;
 }
+/*
+ * Caller holds the irq descriptor lock, so this won't be called
+ * concurrently with xive_get_irqchip_state on the same interrupt.
+ */
 static int xive_irq_set_vcpu_affinity(struct irq_data *d, void *state)
 {
 	struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
 		/* Set it to PQ=10 state to prevent further sends */
 		pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_10);
+		if (!xd->stale_p) {
+			xd->saved_p = !!(pq & XIVE_ESB_VAL_P);
+			xd->stale_p = !xd->saved_p;
+		}
 		/* No target ? nothing to do */
 		if (xd->target == XIVE_INVALID_TARGET) {
 			 * An untargetted interrupt should have been
 			 * also masked at the source
 			 */
-			WARN_ON(pq & 2);
+			WARN_ON(xd->saved_p);
 			return 0;
 		}
 		 * This saved_p is cleared by the host EOI, when we know
 		 * for sure the queue slot is no longer in use.
 		 */
-		if (pq & 2) {
+		if (xd->saved_p) {
-			pq = xive_esb_read(xd, XIVE_ESB_SET_PQ_11);
+			xive_esb_read(xd, XIVE_ESB_SET_PQ_11);
-			xd->saved_p = true;
 			/*
 			 * Sync the XIVE source HW to ensure the interrupt
 			 */
 			if (xive_ops->sync_source)
 				xive_ops->sync_source(hw_irq);
-		} else
+		}
-			xd->saved_p = false;
 	} else {
 		irqd_clr_forwarded_to_vcpu(d);
 	return 0;
 }
+/* Called with irq descriptor lock held. */
+static int xive_get_irqchip_state(struct irq_data *data,
+				  enum irqchip_irq_state which, bool *state)
+{
+	struct xive_irq_data *xd = irq_data_get_irq_handler_data(data);
+	switch (which) {
+	case IRQCHIP_STATE_ACTIVE:
+		*state = !xd->stale_p &&
+			 (xd->saved_p ||
+			  !!(xive_esb_read(xd, XIVE_ESB_GET) & XIVE_ESB_VAL_P));
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
 static struct irq_chip xive_irq_chip = {
 	.name = "XIVE-IRQ",
 	.irq_startup = xive_irq_startup,
 	.irq_set_type = xive_irq_set_type,
 	.irq_retrigger = xive_irq_retrigger,
 	.irq_set_vcpu_affinity = xive_irq_set_vcpu_affinity,
+	.irq_get_irqchip_state = xive_get_irqchip_state,
 };
 bool is_xive_irq(struct irq_chip *chip)
 		raw_spin_lock(&desc->lock);
 		xd = irq_desc_get_handler_data(desc);
+		/*
+		 * Clear saved_p to indicate that it's no longer pending
+		 */
+		xd->saved_p = false;
 		/*
 		 * For LSIs, we EOI, this will cause a resend if it's
 		 * still asserted. Otherwise do an MSI retrigger.

arch/powerpc/sysdev/xive/native.c+7−0

@@ -800,6 +800,13 @@ int xive_native_set_queue_state(u32 vp_id, u32 prio, u32 qtoggle, u32 qindex) · 799 unmodified lines
800	}	800	}
801	EXPORT_SYMBOL_GPL(xive_native_set_queue_state);	801	EXPORT_SYMBOL_GPL(xive_native_set_queue_state);
802		802
		803	bool xive_native_has_queue_state_support(void)
		804	{
		805	return opal_check_token(OPAL_XIVE_GET_QUEUE_STATE) &&
		806	opal_check_token(OPAL_XIVE_SET_QUEUE_STATE);
		807	}
		808	EXPORT_SYMBOL_GPL(xive_native_has_queue_state_support);
		809
803	int xive_native_get_vp_state(u32 vp_id, u64 *out_state)	810	int xive_native_get_vp_state(u32 vp_id, u64 *out_state)
804	{	811	{
805	__be64 state;	812	__be64 state;

arch/s390/include/uapi/asm/kvm.h+6−0

 #define KVM_SYNC_GSCB   (1UL << 9)
 #define KVM_SYNC_BPBC   (1UL << 10)
 #define KVM_SYNC_ETOKEN (1UL << 11)
+#define KVM_SYNC_S390_VALID_FIELDS \
+	(KVM_SYNC_PREFIX | KVM_SYNC_GPRS | KVM_SYNC_ACRS | KVM_SYNC_CRS | \
+	 KVM_SYNC_ARCH0 | KVM_SYNC_PFAULT | KVM_SYNC_VRS | KVM_SYNC_RICCB | \
+	 KVM_SYNC_FPRS | KVM_SYNC_GSCB | KVM_SYNC_BPBC | KVM_SYNC_ETOKEN)
 /* length and alignment of the sdnx as a power of two */
 #define SDNXC 8
 #define SDNXL (1UL << SDNXC)

arch/s390/kvm/kvm-s390.c+5−1

 	if (kvm_run->immediate_exit)
 		return -EINTR;
+	if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
+	    kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
+		return -EINVAL;
 	vcpu_load(vcpu);
 	if (guestdbg_exit_pending(vcpu)) {
 	const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
 				    | KVM_S390_MEMOP_F_CHECK_ONLY;
-	if (mop->flags & ~supported_flags)
+	if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
 		return -EINVAL;
 	if (mop->size > MEM_OP_MAX_SIZE)

arch/x86/include/asm/kvm_emulate.h+2−1

@@ -229,7 +229,7 @@ struct x86_emulate_ops { · 228 unmodified lines
229	int (pre_leave_smm)(struct x86_emulate_ctxt ctxt,	229	int (pre_leave_smm)(struct x86_emulate_ctxt ctxt,
230	const char *smstate);	230	const char *smstate);
231	void (post_leave_smm)(struct x86_emulate_ctxt ctxt);	231	void (post_leave_smm)(struct x86_emulate_ctxt ctxt);
232		232	int (set_xcr)(struct x86_emulate_ctxt ctxt, u32 index, u64 xcr);
233	};	233	};
234		234
235	typedef u32 __attribute__((vector_size(16))) sse128_t;	235	typedef u32 __attribute__((vector_size(16))) sse128_t;
@@ -429,6 +429,7 @@ enum x86_intercept { · 193 unmodified lines
429	x86_intercept_ins,	429	x86_intercept_ins,
430	x86_intercept_out,	430	x86_intercept_out,
431	x86_intercept_outs,	431	x86_intercept_outs,
		432	x86_intercept_xsetbv,
432		433
433	nr_x86_intercepts	434	nr_x86_intercepts
434	};	435	};

arch/x86/include/asm/kvm_host.h+15−4

 	/* Cache MMIO info */
 	u64 mmio_gva;
-	unsigned access;
+	unsigned mmio_access;
 	gfn_t mmio_gfn;
 	u64 mmio_gen;
 	void (*run)(struct kvm_vcpu *vcpu);
 	int (*handle_exit)(struct kvm_vcpu *vcpu);
-	void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
+	int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu);
 	void (*set_interrupt_shadow)(struct kvm_vcpu *vcpu, int mask);
 	u32 (*get_interrupt_shadow)(struct kvm_vcpu *vcpu);
 	void (*patch_hypercall)(struct kvm_vcpu *vcpu,
 	uint16_t (*nested_get_evmcs_version)(struct kvm_vcpu *vcpu);
 	bool (*need_emulation_on_page_fault)(struct kvm_vcpu *vcpu);
+	bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu);
 };
 struct kvm_arch_async_pf {
 void kvm_enable_efer_bits(u64);
 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
-int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
+int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr);
+int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
+int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
 struct x86_emulate_ctxt;
 void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
 		     struct kvm_lapic_irq *irq);
+static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
+{
+	/* We can only post Fixed and LowPrio IRQs */
+	return (irq->delivery_mode == dest_Fixed ||
+		irq->delivery_mode == dest_LowestPrio);
+}
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 {
 	if (kvm_x86_ops->vcpu_blocking)

arch/x86/include/asm/vmx.h+14−0

@@ -562,6 +562,20 @@ enum vm_instruction_error_number { · 561 unmodified lines
562	VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,	562	VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,
563	};	563	};
564		564
		565	/*
		566	* VM-instruction errors that can be encountered on VM-Enter, used to trace
		567	* nested VM-Enter failures reported by hardware. Errors unique to VM-Enter
		568	* from a SMI Transfer Monitor are not included as things have gone seriously
		569	* sideways if we get one of those...
		570	*/
		571	#define VMX_VMENTER_INSTRUCTION_ERRORS \
		572	{ VMXERR_VMLAUNCH_NONCLEAR_VMCS, "VMLAUNCH_NONCLEAR_VMCS" }, \
		573	{ VMXERR_VMRESUME_NONLAUNCHED_VMCS, "VMRESUME_NONLAUNCHED_VMCS" }, \
		574	{ VMXERR_VMRESUME_AFTER_VMXOFF, "VMRESUME_AFTER_VMXOFF" }, \
		575	{ VMXERR_ENTRY_INVALID_CONTROL_FIELD, "VMENTRY_INVALID_CONTROL_FIELD" }, \
		576	{ VMXERR_ENTRY_INVALID_HOST_STATE_FIELD, "VMENTRY_INVALID_HOST_STATE_FIELD" }, \
		577	{ VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS, "VMENTRY_EVENTS_BLOCKED_BY_MOV_SS" }
		578
565	enum vmx_l1d_flush_state {	579	enum vmx_l1d_flush_state {
566	VMENTER_L1D_FLUSH_AUTO,	580	VMENTER_L1D_FLUSH_AUTO,
567	VMENTER_L1D_FLUSH_NEVER,	581	VMENTER_L1D_FLUSH_NEVER,

arch/x86/include/uapi/asm/vmx.h+2−0

 #define EXIT_REASON_EXCEPTION_NMI       0
 #define EXIT_REASON_EXTERNAL_INTERRUPT  1
 #define EXIT_REASON_TRIPLE_FAULT        2
+#define EXIT_REASON_INIT_SIGNAL			3
 #define EXIT_REASON_PENDING_INTERRUPT   7
 #define EXIT_REASON_NMI_WINDOW          8
 	{ EXIT_REASON_EXCEPTION_NMI,         "EXCEPTION_NMI" }, \
 	{ EXIT_REASON_EXTERNAL_INTERRUPT,    "EXTERNAL_INTERRUPT" }, \
 	{ EXIT_REASON_TRIPLE_FAULT,          "TRIPLE_FAULT" }, \
+	{ EXIT_REASON_INIT_SIGNAL,           "INIT_SIGNAL" }, \
 	{ EXIT_REASON_PENDING_INTERRUPT,     "PENDING_INTERRUPT" }, \
 	{ EXIT_REASON_NMI_WINDOW,            "NMI_WINDOW" }, \
 	{ EXIT_REASON_TASK_SWITCH,           "TASK_SWITCH" }, \

arch/x86/kernel/kvm.c+0−12

 	__send_ipi_mask(local_mask, vector);
 }
-static void kvm_send_ipi_allbutself(int vector)
-{
-	kvm_send_ipi_mask_allbutself(cpu_online_mask, vector);
-}
-static void kvm_send_ipi_all(int vector)
-{
-	__send_ipi_mask(cpu_online_mask, vector);
-}
 /*
  * Set the IPI entry points
  */
 {
 	apic->send_IPI_mask = kvm_send_ipi_mask;
 	apic->send_IPI_mask_allbutself = kvm_send_ipi_mask_allbutself;
-	apic->send_IPI_allbutself = kvm_send_ipi_allbutself;
-	apic->send_IPI_all = kvm_send_ipi_all;
 	pr_info("KVM setup pv IPIs\n");
 }

arch/x86/kvm/cpuid.c+19−8

 		entry->edx &= kvm_cpuid_7_0_edx_x86_features;
 		cpuid_mask(&entry->edx, CPUID_7_EDX);
+		if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
+			entry->edx |= F(SPEC_CTRL);
+		if (boot_cpu_has(X86_FEATURE_STIBP))
+			entry->edx |= F(INTEL_STIBP);
+		if (boot_cpu_has(X86_FEATURE_SSBD))
+			entry->edx |= F(SPEC_CTRL_SSBD);
 		/*
 		 * We emulate ARCH_CAPABILITIES in software even
 		 * if the host doesn't support it.
 			g_phys_as = phys_as;
 		entry->eax = g_phys_as | (virt_as << 8);
 		entry->edx = 0;
+		entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
+		cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
 		/*
-		 * IBRS, IBPB and VIRT_SSBD aren't necessarily present in
+		 * AMD has separate bits for each SPEC_CTRL bit.
-		 * hardware cpuid
+		 * arch/x86/kernel/cpu/bugs.c is kind enough to
+		 * record that in cpufeatures so use them.
 		 */
-		if (boot_cpu_has(X86_FEATURE_AMD_IBPB))
+		if (boot_cpu_has(X86_FEATURE_IBPB))
 			entry->ebx |= F(AMD_IBPB);
-		if (boot_cpu_has(X86_FEATURE_AMD_IBRS))
+		if (boot_cpu_has(X86_FEATURE_IBRS))
 			entry->ebx |= F(AMD_IBRS);
-		if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
+		if (boot_cpu_has(X86_FEATURE_STIBP))
-			entry->ebx |= F(VIRT_SSBD);
+			entry->ebx |= F(AMD_STIBP);
-		entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
+		if (boot_cpu_has(X86_FEATURE_SSBD))
-		cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
+			entry->ebx |= F(AMD_SSBD);
+		if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
+			entry->ebx |= F(AMD_SSB_NO);
 		/*
 		 * The preference is to use SPEC CTRL MSR instead of the
 		 * VIRT_SPEC MSR.

arch/x86/kvm/emulate.c+25−2

 	return rc;
 }
+static int em_xsetbv(struct x86_emulate_ctxt *ctxt)
+{
+	u32 eax, ecx, edx;
+	eax = reg_read(ctxt, VCPU_REGS_RAX);
+	edx = reg_read(ctxt, VCPU_REGS_RDX);
+	ecx = reg_read(ctxt, VCPU_REGS_RCX);
+	if (ctxt->ops->set_xcr(ctxt, ecx, ((u64)edx << 32) | eax))
+		return emulate_gp(ctxt, 0);
+	return X86EMUL_CONTINUE;
+}
 static bool valid_cr(int nr)
 {
 	switch (nr) {
 	N, N, N, N, N, N,
 };
+static const struct opcode group7_rm2[] = {
+	N,
+	II(ImplicitOps | Priv,			em_xsetbv,	xsetbv),
+	N, N, N, N, N, N,
+};
 static const struct opcode group7_rm3[] = {
 	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
 	II(SrcNone  | Prot | EmulateOnUD,	em_hypercall,	vmmcall),
 }, {
 	EXT(0, group7_rm0),
 	EXT(0, group7_rm1),
-	N, EXT(0, group7_rm3),
+	EXT(0, group7_rm2),
+	EXT(0, group7_rm3),
 	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
 	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
 	EXT(0, group7_rm7),
 	else {
 		rc = __do_insn_fetch_bytes(ctxt, 1);
 		if (rc != X86EMUL_CONTINUE)
-			return rc;
+			goto done;
 	}
 	switch (mode) {
 					ctxt->memopp->addr.mem.ea + ctxt->_eip);
 done:
+	if (rc == X86EMUL_PROPAGATE_FAULT)
+		ctxt->have_exception = true;
 	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
 }

arch/x86/kvm/lapic.c+11−9

 }
 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
-static void apic_send_ipi(struct kvm_lapic *apic)
+static void apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
 {
-	u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
-	u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
 	struct kvm_lapic_irq irq;
 	irq.vector = icr_low & APIC_VECTOR_MASK;
 	}
 	case APIC_ICR:
 		/* No delay here, so we always clear the pending bit */
-		kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
+		val &= ~(1 << 12);
-		apic_send_ipi(apic);
+		apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
+		kvm_lapic_set_reg(apic, APIC_ICR, val);
 		break;
 	case APIC_ICR2:
 		return;
 	/*
-	 * INITs are latched while in SMM.  Because an SMM CPU cannot
+	 * INITs are latched while CPU is in specific states
-	 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
+	 * (SMM, VMX non-root mode, SVM with GIF=0).
-	 * and delay processing of INIT until the next RSM.
+	 * Because a CPU cannot be in these states immediately
+	 * after it has processed an INIT signal (and thus in
+	 * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
+	 * and leave the INIT pending.
 	 */
-	if (is_smm(vcpu)) {
+	if (is_smm(vcpu) || kvm_x86_ops->apic_init_signal_blocked(vcpu)) {
 		WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
 		if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
 			clear_bit(KVM_APIC_SIPI, &apic->pending_events);

arch/x86/kvm/mmu.c+39−22

@@ -214,6 +214,7 @@ static u64 __read_mostly shadow_accessed_mask; · 213 unmodified lines
214	static u64 __read_mostly shadow_dirty_mask;	214	static u64 __read_mostly shadow_dirty_mask;
215	static u64 __read_mostly shadow_mmio_mask;	215	static u64 __read_mostly shadow_mmio_mask;
216	static u64 __read_mostly shadow_mmio_value;	216	static u64 __read_mostly shadow_mmio_value;
		217	static u64 __read_mostly shadow_mmio_access_mask;
217	static u64 __read_mostly shadow_present_mask;	218	static u64 __read_mostly shadow_present_mask;
218	static u64 __read_mostly shadow_me_mask;	219	static u64 __read_mostly shadow_me_mask;
219		220
@@ -299,14 +300,21 @@ static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, · 79 unmodified lines
299	kvm_flush_remote_tlbs_with_range(kvm, &range);	300	kvm_flush_remote_tlbs_with_range(kvm, &range);
300	}	301	}
301		302
302	void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value)	303	void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask)
303	{	304	{
		305	BUG_ON((u64)(unsigned)access_mask != access_mask);
304	BUG_ON((mmio_mask & mmio_value) != mmio_value);	306	BUG_ON((mmio_mask & mmio_value) != mmio_value);
305	shadow_mmio_value = mmio_value \| SPTE_SPECIAL_MASK;	307	shadow_mmio_value = mmio_value \| SPTE_SPECIAL_MASK;
306	shadow_mmio_mask = mmio_mask \| SPTE_SPECIAL_MASK;	308	shadow_mmio_mask = mmio_mask \| SPTE_SPECIAL_MASK;
		309	shadow_mmio_access_mask = access_mask;
307	}	310	}
308	EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);	311	EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask);
309		312
		313	static bool is_mmio_spte(u64 spte)
		314	{
		315	return (spte & shadow_mmio_mask) == shadow_mmio_value;
		316	}
		317
310	static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)	318	static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
311	{	319	{
312	return sp->role.ad_disabled;	320	return sp->role.ad_disabled;
@@ -314,19 +322,19 @@ static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) · 1 unmodified line
314		322
315	static inline bool spte_ad_enabled(u64 spte)	323	static inline bool spte_ad_enabled(u64 spte)
316	{	324	{
317	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);	325	MMU_WARN_ON(is_mmio_spte(spte));
318	return !(spte & shadow_acc_track_value);	326	return !(spte & shadow_acc_track_value);
319	}	327	}
320		328
321	static inline u64 spte_shadow_accessed_mask(u64 spte)	329	static inline u64 spte_shadow_accessed_mask(u64 spte)
322	{	330	{
323	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);	331	MMU_WARN_ON(is_mmio_spte(spte));
324	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;	332	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
325	}	333	}
326		334
327	static inline u64 spte_shadow_dirty_mask(u64 spte)	335	static inline u64 spte_shadow_dirty_mask(u64 spte)
328	{	336	{
329	MMU_WARN_ON((spte & shadow_mmio_mask) == shadow_mmio_value);	337	MMU_WARN_ON(is_mmio_spte(spte));
330	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;	338	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
331	}	339	}
332		340
@@ -389,7 +397,7 @@ static void mark_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, u64 gfn, · 56 unmodified lines
389	u64 mask = generation_mmio_spte_mask(gen);	397	u64 mask = generation_mmio_spte_mask(gen);
390	u64 gpa = gfn << PAGE_SHIFT;	398	u64 gpa = gfn << PAGE_SHIFT;
391		399
392	access &= ACC_WRITE_MASK \| ACC_USER_MASK;	400	access &= shadow_mmio_access_mask;
393	mask \|= shadow_mmio_value \| access;	401	mask \|= shadow_mmio_value \| access;
394	mask \|= gpa \| shadow_nonpresent_or_rsvd_mask;	402	mask \|= gpa \| shadow_nonpresent_or_rsvd_mask;
395	mask \|= (gpa & shadow_nonpresent_or_rsvd_mask)	403	mask \|= (gpa & shadow_nonpresent_or_rsvd_mask)
@@ -401,11 +409,6 @@ static void mark_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, u64 gfn, · 5 unmodified lines
401	mmu_spte_set(sptep, mask);	409	mmu_spte_set(sptep, mask);
402	}	410	}
403		411
404	static bool is_mmio_spte(u64 spte)
405	{
406	return (spte & shadow_mmio_mask) == shadow_mmio_value;
407	}
408
409	static gfn_t get_mmio_spte_gfn(u64 spte)	412	static gfn_t get_mmio_spte_gfn(u64 spte)
410	{	413	{
411	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;	414	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
@@ -418,8 +421,7 @@ static gfn_t get_mmio_spte_gfn(u64 spte) · 6 unmodified lines
418		421
419	static unsigned get_mmio_spte_access(u64 spte)	422	static unsigned get_mmio_spte_access(u64 spte)
420	{	423	{
421	u64 mask = generation_mmio_spte_mask(MMIO_SPTE_GEN_MASK) \| shadow_mmio_mask;	424	return spte & shadow_mmio_access_mask;
422	return (spte & ~mask) & ~PAGE_MASK;
423	}	425	}
424		426
425	static bool set_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, gfn_t gfn,	427	static bool set_mmio_spte(struct kvm_vcpu vcpu, u64 sptep, gfn_t gfn,
@@ -3302,7 +3304,8 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn, · 2876 unmodified lines
3302	}	3304	}
3303		3305
3304	if (unlikely(is_noslot_pfn(pfn)))	3306	if (unlikely(is_noslot_pfn(pfn)))
3305	vcpu_cache_mmio_info(vcpu, gva, gfn, access);	3307	vcpu_cache_mmio_info(vcpu, gva, gfn,
		3308	access & shadow_mmio_access_mask);
3306		3309
3307	return false;	3310	return false;
3308	}	3311	}
@@ -5611,13 +5614,13 @@ slot_handle_leaf(struct kvm kvm, struct kvm_memory_slot memslot, · 2302 unmodified lines
5611	PT_PAGE_TABLE_LEVEL, lock_flush_tlb);	5614	PT_PAGE_TABLE_LEVEL, lock_flush_tlb);
5612	}	5615	}
5613		5616
5614	static void free_mmu_pages(struct kvm_vcpu *vcpu)	5617	static void free_mmu_pages(struct kvm_mmu *mmu)
5615	{	5618	{
5616	free_page((unsigned long)vcpu->arch.mmu->pae_root);	5619	free_page((unsigned long)mmu->pae_root);
5617	free_page((unsigned long)vcpu->arch.mmu->lm_root);	5620	free_page((unsigned long)mmu->lm_root);
5618	}	5621	}
5619		5622
5620	static int alloc_mmu_pages(struct kvm_vcpu *vcpu)	5623	static int alloc_mmu_pages(struct kvm_vcpu vcpu, struct kvm_mmu mmu)
5621	{	5624	{
5622	struct page *page;	5625	struct page *page;
5623	int i;	5626	int i;
@@ -5638,9 +5641,9 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) · 14 unmodified lines
5638	if (!page)	5641	if (!page)
5639	return -ENOMEM;	5642	return -ENOMEM;
5640		5643
5641	vcpu->arch.mmu->pae_root = page_address(page);	5644	mmu->pae_root = page_address(page);
5642	for (i = 0; i < 4; ++i)	5645	for (i = 0; i < 4; ++i)
5643	vcpu->arch.mmu->pae_root[i] = INVALID_PAGE;	5646	mmu->pae_root[i] = INVALID_PAGE;
5644		5647
5645	return 0;	5648	return 0;
5646	}	5649	}
@@ -5648,6 +5651,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) · 1 unmodified line
5648	int kvm_mmu_create(struct kvm_vcpu *vcpu)	5651	int kvm_mmu_create(struct kvm_vcpu *vcpu)
5649	{	5652	{
5650	uint i;	5653	uint i;
		5654	int ret;
5651		5655
5652	vcpu->arch.mmu = &vcpu->arch.root_mmu;	5656	vcpu->arch.mmu = &vcpu->arch.root_mmu;
5653	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;	5657	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
@@ -5665,7 +5669,19 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) · 11 unmodified lines
5665	vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;	5669	vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5666		5670
5667	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;	5671	vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
5668	return alloc_mmu_pages(vcpu);	5672
		5673	ret = alloc_mmu_pages(vcpu, &vcpu->arch.guest_mmu);
		5674	if (ret)
		5675	return ret;
		5676
		5677	ret = alloc_mmu_pages(vcpu, &vcpu->arch.root_mmu);
		5678	if (ret)
		5679	goto fail_allocate_root;
		5680
		5681	return ret;
		5682	fail_allocate_root:
		5683	free_mmu_pages(&vcpu->arch.guest_mmu);
		5684	return ret;
5669	}	5685	}
5670		5686
5671		5687
@@ -6094,7 +6110,7 @@ static void kvm_set_mmio_spte_mask(void) · 422 unmodified lines
6094	if (IS_ENABLED(CONFIG_X86_64) && shadow_phys_bits == 52)	6110	if (IS_ENABLED(CONFIG_X86_64) && shadow_phys_bits == 52)
6095	mask &= ~1ull;	6111	mask &= ~1ull;
6096		6112
6097	kvm_mmu_set_mmio_spte_mask(mask, mask);	6113	kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK \| ACC_USER_MASK);
6098	}	6114	}
6099		6115
6100	int kvm_mmu_module_init(void)	6116	int kvm_mmu_module_init(void)
@@ -6168,7 +6184,8 @@ unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm) · 67 unmodified lines
6168	void kvm_mmu_destroy(struct kvm_vcpu *vcpu)	6184	void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
6169	{	6185	{
6170	kvm_mmu_unload(vcpu);	6186	kvm_mmu_unload(vcpu);
6171	free_mmu_pages(vcpu);	6187	free_mmu_pages(&vcpu->arch.root_mmu);
		6188	free_mmu_pages(&vcpu->arch.guest_mmu);
6172	mmu_free_memory_caches(vcpu);	6189	mmu_free_memory_caches(vcpu);
6173	}	6190	}
6174		6191

arch/x86/kvm/mmu.h+1−1

@@ -51,7 +51,7 @@ static inline u64 rsvd_bits(int s, int e) · 50 unmodified lines
51	return ((1ULL << (e - s + 1)) - 1) << s;	51	return ((1ULL << (e - s + 1)) - 1) << s;
52	}	52	}
53		53
54	void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value);	54	void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask);
55		55
56	void	56	void
57	reset_shadow_zero_bits_mask(struct kvm_vcpu vcpu, struct kvm_mmu context);	57	reset_shadow_zero_bits_mask(struct kvm_vcpu vcpu, struct kvm_mmu context);

arch/x86/kvm/svm.c+93−105

@@ -68,10 +68,8 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); · 67 unmodified lines
68	#define SEG_TYPE_LDT 2	68	#define SEG_TYPE_LDT 2
69	#define SEG_TYPE_BUSY_TSS16 3	69	#define SEG_TYPE_BUSY_TSS16 3
70		70
71	#define SVM_FEATURE_NPT (1 << 0)
72	#define SVM_FEATURE_LBRV (1 << 1)	71	#define SVM_FEATURE_LBRV (1 << 1)
73	#define SVM_FEATURE_SVML (1 << 2)	72	#define SVM_FEATURE_SVML (1 << 2)
74	#define SVM_FEATURE_NRIP (1 << 3)
75	#define SVM_FEATURE_TSC_RATE (1 << 4)	73	#define SVM_FEATURE_TSC_RATE (1 << 4)
76	#define SVM_FEATURE_VMCB_CLEAN (1 << 5)	74	#define SVM_FEATURE_VMCB_CLEAN (1 << 5)
77	#define SVM_FEATURE_FLUSH_ASID (1 << 6)	75	#define SVM_FEATURE_FLUSH_ASID (1 << 6)
@@ -770,7 +768,7 @@ static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) · 692 unmodified lines
770		768
771	}	769	}
772		770
773	static void skip_emulated_instruction(struct kvm_vcpu *vcpu)	771	static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
774	{	772	{
775	struct vcpu_svm *svm = to_svm(vcpu);	773	struct vcpu_svm *svm = to_svm(vcpu);
776		774
@@ -779,18 +777,17 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) · 2 unmodified lines
779	svm->next_rip = svm->vmcb->control.next_rip;	777	svm->next_rip = svm->vmcb->control.next_rip;
780	}	778	}
781		779
782	if (!svm->next_rip) {	780	if (!svm->next_rip)
783	if (kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) !=	781	return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP);
784	EMULATE_DONE)	782
785	printk(KERN_DEBUG "%s: NOP\n", __func__);
786	return;
787	}
788	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)	783	if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
789	printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",	784	printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
790	__func__, kvm_rip_read(vcpu), svm->next_rip);	785	__func__, kvm_rip_read(vcpu), svm->next_rip);
791		786
792	kvm_rip_write(vcpu, svm->next_rip);	787	kvm_rip_write(vcpu, svm->next_rip);
793	svm_set_interrupt_shadow(vcpu, 0);	788	svm_set_interrupt_shadow(vcpu, 0);
		789
		790	return EMULATE_DONE;
794	}	791	}
795		792
796	static void svm_queue_exception(struct kvm_vcpu *vcpu)	793	static void svm_queue_exception(struct kvm_vcpu *vcpu)
@@ -821,7 +818,7 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu) · 24 unmodified lines
821	* raises a fault that is not intercepted. Still better than	818	* raises a fault that is not intercepted. Still better than
822	* failing in all cases.	819	* failing in all cases.
823	*/	820	*/
824	skip_emulated_instruction(&svm->vcpu);	821	(void)skip_emulated_instruction(&svm->vcpu);
825	rip = kvm_rip_read(&svm->vcpu);	822	rip = kvm_rip_read(&svm->vcpu);
826	svm->int3_rip = rip + svm->vmcb->save.cs.base;	823	svm->int3_rip = rip + svm->vmcb->save.cs.base;
827	svm->int3_injected = rip - old_rip;	824	svm->int3_injected = rip - old_rip;
@@ -1269,11 +1266,11 @@ static void grow_ple_window(struct kvm_vcpu *vcpu) · 441 unmodified lines
1269	pause_filter_count_grow,	1266	pause_filter_count_grow,
1270	pause_filter_count_max);	1267	pause_filter_count_max);
1271		1268
1272	if (control->pause_filter_count != old)	1269	if (control->pause_filter_count != old) {
1273	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);	1270	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1274		1271	trace_kvm_ple_window_update(vcpu->vcpu_id,
1275	trace_kvm_ple_window_grow(vcpu->vcpu_id,	1272	control->pause_filter_count, old);
1276	control->pause_filter_count, old);	1273	}
1277	}	1274	}
1278		1275
1279	static void shrink_ple_window(struct kvm_vcpu *vcpu)	1276	static void shrink_ple_window(struct kvm_vcpu *vcpu)
@@ -1287,11 +1284,11 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu) · 7 unmodified lines
1287	pause_filter_count,	1284	pause_filter_count,
1288	pause_filter_count_shrink,	1285	pause_filter_count_shrink,
1289	pause_filter_count);	1286	pause_filter_count);
1290	if (control->pause_filter_count != old)	1287	if (control->pause_filter_count != old) {
1291	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);	1288	mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1292		1289	trace_kvm_ple_window_update(vcpu->vcpu_id,
1293	trace_kvm_ple_window_shrink(vcpu->vcpu_id,	1290	control->pause_filter_count, old);
1294	control->pause_filter_count, old);	1291	}
1295	}	1292	}
1296		1293
1297	static __init int svm_hardware_setup(void)	1294	static __init int svm_hardware_setup(void)
@@ -2136,6 +2133,9 @@ static struct kvm_vcpu svm_create_vcpu(struct kvm kvm, unsigned int id) · 838 unmodified lines
2136	struct page *nested_msrpm_pages;	2133	struct page *nested_msrpm_pages;
2137	int err;	2134	int err;
2138		2135
		2136	BUILD_BUG_ON_MSG(offsetof(struct vcpu_svm, vcpu) != 0,
		2137	"struct kvm_vcpu must be at offset 0 for arch usercopy region");
		2138
2139	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);	2139	svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
2140	if (!svm) {	2140	if (!svm) {
2141	err = -ENOMEM;	2141	err = -ENOMEM;
@@ -2903,13 +2903,11 @@ static int nop_on_interception(struct vcpu_svm *svm) · 761 unmodified lines
2903		2903
2904	static int halt_interception(struct vcpu_svm *svm)	2904	static int halt_interception(struct vcpu_svm *svm)
2905	{	2905	{
2906	svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2907	return kvm_emulate_halt(&svm->vcpu);	2906	return kvm_emulate_halt(&svm->vcpu);
2908	}	2907	}
2909		2908
2910	static int vmmcall_interception(struct vcpu_svm *svm)	2909	static int vmmcall_interception(struct vcpu_svm *svm)
2911	{	2910	{
2912	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2913	return kvm_emulate_hypercall(&svm->vcpu);	2911	return kvm_emulate_hypercall(&svm->vcpu);
2914	}	2912	}
2915		2913
@@ -3588,9 +3586,9 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, · 672 unmodified lines
3588	mark_all_dirty(svm->vmcb);	3586	mark_all_dirty(svm->vmcb);
3589	}	3587	}
3590		3588
3591	static bool nested_svm_vmrun(struct vcpu_svm *svm)	3589	static int nested_svm_vmrun(struct vcpu_svm *svm)
3592	{	3590	{
3593	int rc;	3591	int ret;
3594	struct vmcb *nested_vmcb;	3592	struct vmcb *nested_vmcb;
3595	struct vmcb *hsave = svm->nested.hsave;	3593	struct vmcb *hsave = svm->nested.hsave;
3596	struct vmcb *vmcb = svm->vmcb;	3594	struct vmcb *vmcb = svm->vmcb;
@@ -3599,13 +3597,16 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 2 unmodified lines
3599		3597
3600	vmcb_gpa = svm->vmcb->save.rax;	3598	vmcb_gpa = svm->vmcb->save.rax;
3601		3599
3602	rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);	3600	ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
3603	if (rc) {	3601	if (ret == -EINVAL) {
3604	if (rc == -EINVAL)	3602	kvm_inject_gp(&svm->vcpu, 0);
3605	kvm_inject_gp(&svm->vcpu, 0);	3603	return 1;
3606	return false;	3604	} else if (ret) {
		3605	return kvm_skip_emulated_instruction(&svm->vcpu);
3607	}	3606	}
3608		3607
		3608	ret = kvm_skip_emulated_instruction(&svm->vcpu);
		3609
3609	nested_vmcb = map.hva;	3610	nested_vmcb = map.hva;
3610		3611
3611	if (!nested_vmcb_checks(nested_vmcb)) {	3612	if (!nested_vmcb_checks(nested_vmcb)) {
@@ -3616,7 +3617,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 4 unmodified lines
3616		3617
3617	kvm_vcpu_unmap(&svm->vcpu, &map, true);	3618	kvm_vcpu_unmap(&svm->vcpu, &map, true);
3618		3619
3619	return false;	3620	return ret;
3620	}	3621	}
3621		3622
3622	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,	3623	trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
@@ -3660,7 +3661,16 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm) · 37 unmodified lines
3660		3661
3661	enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);	3662	enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
3662		3663
3663	return true;	3664	if (!nested_svm_vmrun_msrpm(svm)) {
		3665	svm->vmcb->control.exit_code = SVM_EXIT_ERR;
		3666	svm->vmcb->control.exit_code_hi = 0;
		3667	svm->vmcb->control.exit_info_1 = 0;
		3668	svm->vmcb->control.exit_info_2 = 0;
		3669
		3670	nested_svm_vmexit(svm);
		3671	}
		3672
		3673	return ret;
3664	}	3674	}
3665		3675
3666	static void nested_svm_vmloadsave(struct vmcb from_vmcb, struct vmcb to_vmcb)	3676	static void nested_svm_vmloadsave(struct vmcb from_vmcb, struct vmcb to_vmcb)
@@ -3697,7 +3707,6 @@ static int vmload_interception(struct vcpu_svm *svm) · 30 unmodified lines
3697		3707
3698	nested_vmcb = map.hva;	3708	nested_vmcb = map.hva;
3699		3709
3700	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3701	ret = kvm_skip_emulated_instruction(&svm->vcpu);	3710	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3702		3711
3703	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);	3712	nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
@@ -3724,7 +3733,6 @@ static int vmsave_interception(struct vcpu_svm *svm) · 20 unmodified lines
3724		3733
3725	nested_vmcb = map.hva;	3734	nested_vmcb = map.hva;
3726		3735
3727	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3728	ret = kvm_skip_emulated_instruction(&svm->vcpu);	3736	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3729		3737
3730	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);	3738	nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
@@ -3738,27 +3746,7 @@ static int vmrun_interception(struct vcpu_svm *svm) · 7 unmodified lines
3738	if (nested_svm_check_permissions(svm))	3746	if (nested_svm_check_permissions(svm))
3739	return 1;	3747	return 1;
3740		3748
3741	/* Save rip after vmrun instruction */	3749	return nested_svm_vmrun(svm);
3742	kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3743
3744	if (!nested_svm_vmrun(svm))
3745	return 1;
3746
3747	if (!nested_svm_vmrun_msrpm(svm))
3748	goto failed;
3749
3750	return 1;
3751
3752	failed:
3753
3754	svm->vmcb->control.exit_code = SVM_EXIT_ERR;
3755	svm->vmcb->control.exit_code_hi = 0;
3756	svm->vmcb->control.exit_info_1 = 0;
3757	svm->vmcb->control.exit_info_2 = 0;
3758
3759	nested_svm_vmexit(svm);
3760
3761	return 1;
3762	}	3750	}
3763		3751
3764	static int stgi_interception(struct vcpu_svm *svm)	3752	static int stgi_interception(struct vcpu_svm *svm)
@@ -3775,7 +3763,6 @@ static int stgi_interception(struct vcpu_svm *svm) · 10 unmodified lines
3775	if (vgif_enabled(svm))	3763	if (vgif_enabled(svm))
3776	clr_intercept(svm, INTERCEPT_STGI);	3764	clr_intercept(svm, INTERCEPT_STGI);
3777		3765
3778	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3779	ret = kvm_skip_emulated_instruction(&svm->vcpu);	3766	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3780	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);	3767	kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3781		3768
@@ -3791,7 +3778,6 @@ static int clgi_interception(struct vcpu_svm *svm) · 9 unmodified lines
3791	if (nested_svm_check_permissions(svm))	3778	if (nested_svm_check_permissions(svm))
3792	return 1;	3779	return 1;
3793		3780
3794	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3795	ret = kvm_skip_emulated_instruction(&svm->vcpu);	3781	ret = kvm_skip_emulated_instruction(&svm->vcpu);
3796		3782
3797	disable_gif(svm);	3783	disable_gif(svm);
@@ -3816,7 +3802,6 @@ static int invlpga_interception(struct vcpu_svm *svm) · 18 unmodified lines
3816	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */	3802	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3817	kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));	3803	kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
3818		3804
3819	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3820	return kvm_skip_emulated_instruction(&svm->vcpu);	3805	return kvm_skip_emulated_instruction(&svm->vcpu);
3821	}	3806	}
3822		3807
@@ -3839,7 +3824,6 @@ static int xsetbv_interception(struct vcpu_svm *svm) · 16 unmodified lines
3839	u32 index = kvm_rcx_read(&svm->vcpu);	3824	u32 index = kvm_rcx_read(&svm->vcpu);
3840		3825
3841	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {	3826	if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3842	svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3843	return kvm_skip_emulated_instruction(&svm->vcpu);	3827	return kvm_skip_emulated_instruction(&svm->vcpu);
3844	}	3828	}
3845		3829
@@ -3898,25 +3882,29 @@ static int task_switch_interception(struct vcpu_svm *svm) · 52 unmodified lines
3898	if (reason != TASK_SWITCH_GATE \|\|	3882	if (reason != TASK_SWITCH_GATE \|\|
3899	int_type == SVM_EXITINTINFO_TYPE_SOFT \|\|	3883	int_type == SVM_EXITINTINFO_TYPE_SOFT \|\|
3900	(int_type == SVM_EXITINTINFO_TYPE_EXEPT &&	3884	(int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3901	(int_vec == OF_VECTOR \|\| int_vec == BP_VECTOR)))	3885	(int_vec == OF_VECTOR \|\| int_vec == BP_VECTOR))) {
3902	skip_emulated_instruction(&svm->vcpu);	3886	if (skip_emulated_instruction(&svm->vcpu) != EMULATE_DONE)
		3887	goto fail;
		3888	}
3903		3889
3904	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)	3890	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3905	int_vec = -1;	3891	int_vec = -1;
3906		3892
3907	if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,	3893	if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3908	has_error_code, error_code) == EMULATE_FAIL) {	3894	has_error_code, error_code) == EMULATE_FAIL)
3909	svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;	3895	goto fail;
3910	svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;	3896
3911	svm->vcpu.run->internal.ndata = 0;
3912	return 0;
3913	}
3914	return 1;	3897	return 1;
		3898
		3899	fail:
		3900	svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		3901	svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		3902	svm->vcpu.run->internal.ndata = 0;
		3903	return 0;
3915	}	3904	}
3916		3905
3917	static int cpuid_interception(struct vcpu_svm *svm)	3906	static int cpuid_interception(struct vcpu_svm *svm)
3918	{	3907	{
3919	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3920	return kvm_emulate_cpuid(&svm->vcpu);	3908	return kvm_emulate_cpuid(&svm->vcpu);
3921	}	3909	}
3922		3910
@@ -4232,23 +4220,7 @@ static int svm_get_msr(struct kvm_vcpu vcpu, struct msr_data msr_info) · 309 unmodified lines
4232		4220
4233	static int rdmsr_interception(struct vcpu_svm *svm)	4221	static int rdmsr_interception(struct vcpu_svm *svm)
4234	{	4222	{
4235	u32 ecx = kvm_rcx_read(&svm->vcpu);	4223	return kvm_emulate_rdmsr(&svm->vcpu);
4236	struct msr_data msr_info;
4237
4238	msr_info.index = ecx;
4239	msr_info.host_initiated = false;
4240	if (svm_get_msr(&svm->vcpu, &msr_info)) {
4241	trace_kvm_msr_read_ex(ecx);
4242	kvm_inject_gp(&svm->vcpu, 0);
4243	return 1;
4244	} else {
4245	trace_kvm_msr_read(ecx, msr_info.data);
4246
4247	kvm_rax_write(&svm->vcpu, msr_info.data & 0xffffffff);
4248	kvm_rdx_write(&svm->vcpu, msr_info.data >> 32);
4249	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4250	return kvm_skip_emulated_instruction(&svm->vcpu);
4251	}
4252	}	4224	}
4253		4225
4254	static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)	4226	static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
@@ -4438,23 +4410,7 @@ static int svm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) · 183 unmodified lines
4438		4410
4439	static int wrmsr_interception(struct vcpu_svm *svm)	4411	static int wrmsr_interception(struct vcpu_svm *svm)
4440	{	4412	{
4441	struct msr_data msr;	4413	return kvm_emulate_wrmsr(&svm->vcpu);
4442	u32 ecx = kvm_rcx_read(&svm->vcpu);
4443	u64 data = kvm_read_edx_eax(&svm->vcpu);
4444
4445	msr.data = data;
4446	msr.index = ecx;
4447	msr.host_initiated = false;
4448
4449	svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
4450	if (kvm_set_msr(&svm->vcpu, &msr)) {
4451	trace_kvm_msr_write_ex(ecx, data);
4452	kvm_inject_gp(&svm->vcpu, 0);
4453	return 1;
4454	} else {
4455	trace_kvm_msr_write(ecx, data);
4456	return kvm_skip_emulated_instruction(&svm->vcpu);
4457	}
4458	}	4414	}
4459		4415
4460	static int msr_interception(struct vcpu_svm *svm)	4416	static int msr_interception(struct vcpu_svm *svm)
@@ -5025,9 +4981,14 @@ static int handle_exit(struct kvm_vcpu *vcpu) · 564 unmodified lines
5025		4981
5026	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)	4982	if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
5027	\|\| !svm_exit_handlers[exit_code]) {	4983	\|\| !svm_exit_handlers[exit_code]) {
5028	WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);	4984	vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
5029	kvm_queue_exception(vcpu, UD_VECTOR);	4985	dump_vmcb(vcpu);
5030	return 1;	4986	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		4987	vcpu->run->internal.suberror =
		4988	KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
		4989	vcpu->run->internal.ndata = 1;
		4990	vcpu->run->internal.data[0] = exit_code;
		4991	return 0;
5031	}	4992	}
5032		4993
5033	return svm_exit_handlers[exit_code](svm);	4994	return svm_exit_handlers[exit_code](svm);
@@ -5274,7 +5235,8 @@ get_pi_vcpu_info(struct kvm kvm, struct kvm_kernel_irq_routing_entry e, · 240 unmodified lines
5274		5235
5275	kvm_set_msi_irq(kvm, e, &irq);	5236	kvm_set_msi_irq(kvm, e, &irq);
5276		5237
5277	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {	5238	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) \|\|
		5239	!kvm_irq_is_postable(&irq)) {
5278	pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",	5240	pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5279	__func__, irq.vector);	5241	__func__, irq.vector);
5280	return -1;	5242	return -1;
@@ -5328,6 +5290,7 @@ static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, · 47 unmodified lines
5328	* 1. When cannot target interrupt to a specific vcpu.	5290	* 1. When cannot target interrupt to a specific vcpu.
5329	* 2. Unsetting posted interrupt.	5291	* 2. Unsetting posted interrupt.
5330	* 3. APIC virtialization is disabled for the vcpu.	5292	* 3. APIC virtialization is disabled for the vcpu.
		5293	* 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
5331	*/	5294	*/
5332	if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&	5295	if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5333	kvm_vcpu_apicv_active(&svm->vcpu)) {	5296	kvm_vcpu_apicv_active(&svm->vcpu)) {
@@ -5933,6 +5896,8 @@ static void svm_cpuid_update(struct kvm_vcpu *vcpu) · 599 unmodified lines
5933	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);	5896	guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5934	}	5897	}
5935		5898
		5899	#define F(x) bit(X86_FEATURE_##x)
		5900
5936	static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)	5901	static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5937	{	5902	{
5938	switch (func) {	5903	switch (func) {
@@ -5944,6 +5909,11 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) · 5 unmodified lines
5944	if (nested)	5909	if (nested)
5945	entry->ecx \|= (1 << 2); /* Set SVM bit */	5910	entry->ecx \|= (1 << 2); /* Set SVM bit */
5946	break;	5911	break;
		5912	case 0x80000008:
		5913	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) \|\|
		5914	boot_cpu_has(X86_FEATURE_AMD_SSBD))
		5915	entry->ebx \|= F(VIRT_SSBD);
		5916	break;
5947	case 0x8000000A:	5917	case 0x8000000A:
5948	entry->eax = 1; /* SVM revision 1 */	5918	entry->eax = 1; /* SVM revision 1 */
5949	entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper	5919	entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
@@ -5954,11 +5924,11 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry) · 4 unmodified lines
5954		5924
5955	/* Support next_rip if host supports it */	5925	/* Support next_rip if host supports it */
5956	if (boot_cpu_has(X86_FEATURE_NRIPS))	5926	if (boot_cpu_has(X86_FEATURE_NRIPS))
5957	entry->edx \|= SVM_FEATURE_NRIP;	5927	entry->edx \|= F(NRIPS);
5958		5928
5959	/* Support NPT for the guest if enabled */	5929	/* Support NPT for the guest if enabled */
5960	if (npt_enabled)	5930	if (npt_enabled)
5961	entry->edx \|= SVM_FEATURE_NPT;	5931	entry->edx \|= F(NPT);
5962		5932
5963	break;	5933	break;
5964	case 0x8000001F:	5934	case 0x8000001F:
@@ -6067,6 +6037,7 @@ static const struct __x86_intercept { · 102 unmodified lines
6067	[x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),	6037	[x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
6068	[x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),	6038	[x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
6069	[x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),	6039	[x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
		6040	[x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV),
6070	};	6041	};
6071		6042
6072	#undef PRE_EX	6043	#undef PRE_EX
@@ -7193,6 +7164,21 @@ static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) · 1120 unmodified lines
7193	return false;	7164	return false;
7194	}	7165	}
7195		7166
		7167	static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
		7168	{
		7169	struct vcpu_svm *svm = to_svm(vcpu);
		7170
		7171	/*
		7172	* TODO: Last condition latch INIT signals on vCPU when
		7173	* vCPU is in guest-mode and vmcb12 defines intercept on INIT.
		7174	* To properly emulate the INIT intercept, SVM should implement
		7175	* kvm_x86_ops->check_nested_events() and call nested_svm_vmexit()
		7176	* there if an INIT signal is pending.
		7177	*/
		7178	return !gif_set(svm) \|\|
		7179	(svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
		7180	}
		7181
7196	static struct kvm_x86_ops svm_x86_ops __ro_after_init = {	7182	static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7197	.cpu_has_kvm_support = has_svm,	7183	.cpu_has_kvm_support = has_svm,
7198	.disabled_by_bios = is_disabled,	7184	.disabled_by_bios = is_disabled,
@@ -7329,6 +7315,8 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { · 130 unmodified lines
7329	.nested_get_evmcs_version = NULL,	7315	.nested_get_evmcs_version = NULL,
7330		7316
7331	.need_emulation_on_page_fault = svm_need_emulation_on_page_fault,	7317	.need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
		7318
		7319	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
7332	};	7320	};
7333		7321
7334	static int __init svm_init(void)	7322	static int __init svm_init(void)

arch/x86/kvm/trace.h+54−20

 		__field(	u32,	        isa             )
 		__field(	u64,	        info1           )
 		__field(	u64,	        info2           )
+		__field(	unsigned int,	vcpu_id         )
 	),
 	TP_fast_assign(
 		__entry->exit_reason	= exit_reason;
 		__entry->guest_rip	= kvm_rip_read(vcpu);
 		__entry->isa            = isa;
+		__entry->vcpu_id        = vcpu->vcpu_id;
 		kvm_x86_ops->get_exit_info(vcpu, &__entry->info1,
 					   &__entry->info2);
 	),
-	TP_printk("reason %s rip 0x%lx info %llx %llx",
+	TP_printk("vcpu %u reason %s rip 0x%lx info %llx %llx",
+		  __entry->vcpu_id,
 		 (__entry->isa == KVM_ISA_VMX) ?
 		 __print_symbolic(__entry->exit_reason, VMX_EXIT_REASONS) :
 		 __print_symbolic(__entry->exit_reason, SVM_EXIT_REASONS),
 	TP_printk("vcpu %d: PML full", __entry->vcpu_id)
 );
-TRACE_EVENT(kvm_ple_window,
+TRACE_EVENT(kvm_ple_window_update,
-	TP_PROTO(bool grow, unsigned int vcpu_id, int new, int old),
+	TP_PROTO(unsigned int vcpu_id, unsigned int new, unsigned int old),
-	TP_ARGS(grow, vcpu_id, new, old),
+	TP_ARGS(vcpu_id, new, old),
 	TP_STRUCT__entry(
-		__field(                bool,      grow         )
 		__field(        unsigned int,   vcpu_id         )
-		__field(                 int,       new         )
+		__field(        unsigned int,       new         )
-		__field(                 int,       old         )
+		__field(        unsigned int,       old         )
 	),
 	TP_fast_assign(
-		__entry->grow           = grow;
 		__entry->vcpu_id        = vcpu_id;
 		__entry->new            = new;
 		__entry->old            = old;
 	),
-	TP_printk("vcpu %u: ple_window %d (%s %d)",
+	TP_printk("vcpu %u old %u new %u (%s)",
-	          __entry->vcpu_id,
+	          __entry->vcpu_id, __entry->old, __entry->new,
-	          __entry->new,
+		  __entry->old < __entry->new ? "growed" : "shrinked")
-	          __entry->grow ? "grow" : "shrink",
-	          __entry->old)
 );
-#define trace_kvm_ple_window_grow(vcpu_id, new, old) \
-	trace_kvm_ple_window(true, vcpu_id, new, old)
-#define trace_kvm_ple_window_shrink(vcpu_id, new, old) \
-	trace_kvm_ple_window(false, vcpu_id, new, old)
 TRACE_EVENT(kvm_pvclock_update,
 	TP_PROTO(unsigned int vcpu_id, struct pvclock_vcpu_time_info *pvclock),
 	TP_ARGS(vcpu_id, pvclock),
 		__entry->index = index;
 	),
-	TP_printk("vcpu=%u, icrh:icrl=%#010x:%08x, id=%u, index=%u\n",
+	TP_printk("vcpu=%u, icrh:icrl=%#010x:%08x, id=%u, index=%u",
 		  __entry->vcpu, __entry->icrh, __entry->icrl,
 		  __entry->id, __entry->index)
 );
 		__entry->vec = vec;
 	),
-	TP_printk("vcpu=%u, offset=%#x(%s), %s, %s, vec=%#x\n",
+	TP_printk("vcpu=%u, offset=%#x(%s), %s, %s, vec=%#x",
 		  __entry->vcpu,
 		  __entry->offset,
 		  __print_symbolic(__entry->offset, kvm_trace_symbol_apic),
 		  __entry->vector, __entry->format,
 		  __entry->valid_bank_mask)
 );
+TRACE_EVENT(kvm_pv_tlb_flush,
+	TP_PROTO(unsigned int vcpu_id, bool need_flush_tlb),
+	TP_ARGS(vcpu_id, need_flush_tlb),
+	TP_STRUCT__entry(
+		__field(	unsigned int,	vcpu_id		)
+		__field(	bool,	need_flush_tlb		)
+	),
+	TP_fast_assign(
+		__entry->vcpu_id	= vcpu_id;
+		__entry->need_flush_tlb = need_flush_tlb;
+	),
+	TP_printk("vcpu %u need_flush_tlb %s", __entry->vcpu_id,
+		__entry->need_flush_tlb ? "true" : "false")
+);
+/*
+ * Tracepoint for failed nested VMX VM-Enter.
+ */
+TRACE_EVENT(kvm_nested_vmenter_failed,
+	TP_PROTO(const char *msg, u32 err),
+	TP_ARGS(msg, err),
+	TP_STRUCT__entry(
+		__field(const char *, msg)
+		__field(u32, err)
+	),
+	TP_fast_assign(
+		__entry->msg = msg;
+		__entry->err = err;
+	),
+	TP_printk("%s%s", __entry->msg, !__entry->err ? "" :
+		__print_symbolic(__entry->err, VMX_VMENTER_INSTRUCTION_ERRORS))
+);
 #endif /* _TRACE_KVM_H */
 #undef TRACE_INCLUDE_PATH

arch/x86/kvm/vmx/nested.c+162−143

@@ -19,6 +19,14 @@ module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); · 18 unmodified lines
19	static bool __read_mostly nested_early_check = 0;	19	static bool __read_mostly nested_early_check = 0;
20	module_param(nested_early_check, bool, S_IRUGO);	20	module_param(nested_early_check, bool, S_IRUGO);
21		21
		22	#define CC(consistency_check) \
		23	({ \
		24	bool failed = (consistency_check); \
		25	if (failed) \
		26	trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
		27	failed; \
		28	})
		29
22	/*	30	/*
23	* Hyper-V requires all of these, so mark them as supported even though	31	* Hyper-V requires all of these, so mark them as supported even though
24	* they are just treated the same as all-context.	32	* they are just treated the same as all-context.
@@ -430,8 +438,8 @@ static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, · 405 unmodified lines
430	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))	438	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
431	return 0;	439	return 0;
432		440
433	if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) \|\|	441	if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) \|\|
434	!page_address_valid(vcpu, vmcs12->io_bitmap_b))	442	CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
435	return -EINVAL;	443	return -EINVAL;
436		444
437	return 0;	445	return 0;
@@ -443,7 +451,7 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
443	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))	451	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
444	return 0;	452	return 0;
445		453
446	if (!page_address_valid(vcpu, vmcs12->msr_bitmap))	454	if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
447	return -EINVAL;	455	return -EINVAL;
448		456
449	return 0;	457	return 0;
@@ -455,7 +463,7 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
455	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))	463	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
456	return 0;	464	return 0;
457		465
458	if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))	466	if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
459	return -EINVAL;	467	return -EINVAL;
460		468
461	return 0;	469	return 0;
@@ -688,7 +696,7 @@ static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, · 226 unmodified lines
688	struct vmcs12 *vmcs12)	696	struct vmcs12 *vmcs12)
689	{	697	{
690	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&	698	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
691	!page_address_valid(vcpu, vmcs12->apic_access_addr))	699	CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
692	return -EINVAL;	700	return -EINVAL;
693	else	701	else
694	return 0;	702	return 0;
@@ -707,16 +715,15 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 12 unmodified lines
707	* If virtualize x2apic mode is enabled,	715	* If virtualize x2apic mode is enabled,
708	* virtualize apic access must be disabled.	716	* virtualize apic access must be disabled.
709	*/	717	*/
710	if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&	718	if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
711	nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))	719	nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
712	return -EINVAL;	720	return -EINVAL;
713		721
714	/*	722	/*
715	* If virtual interrupt delivery is enabled,	723	* If virtual interrupt delivery is enabled,
716	* we must exit on external interrupts.	724	* we must exit on external interrupts.
717	*/	725	*/
718	if (nested_cpu_has_vid(vmcs12) &&	726	if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
719	!nested_exit_on_intr(vcpu))
720	return -EINVAL;	727	return -EINVAL;
721		728
722	/*	729	/*
@@ -727,15 +734,15 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
727	* bits 5:0 of posted_intr_desc_addr should be zero.	734	* bits 5:0 of posted_intr_desc_addr should be zero.
728	*/	735	*/
729	if (nested_cpu_has_posted_intr(vmcs12) &&	736	if (nested_cpu_has_posted_intr(vmcs12) &&
730	(!nested_cpu_has_vid(vmcs12) \|\|	737	(CC(!nested_cpu_has_vid(vmcs12)) \|\|
731	!nested_exit_intr_ack_set(vcpu) \|\|	738	CC(!nested_exit_intr_ack_set(vcpu)) \|\|
732	(vmcs12->posted_intr_nv & 0xff00) \|\|	739	CC((vmcs12->posted_intr_nv & 0xff00)) \|\|
733	(vmcs12->posted_intr_desc_addr & 0x3f) \|\|	740	CC((vmcs12->posted_intr_desc_addr & 0x3f)) \|\|
734	(vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu))))	741	CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
735	return -EINVAL;	742	return -EINVAL;
736		743
737	/* tpr shadow is needed by all apicv features. */	744	/* tpr shadow is needed by all apicv features. */
738	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))	745	if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
739	return -EINVAL;	746	return -EINVAL;
740		747
741	return 0;	748	return 0;
@@ -759,10 +766,12 @@ static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, · 17 unmodified lines
759	static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,	766	static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
760	struct vmcs12 *vmcs12)	767	struct vmcs12 *vmcs12)
761	{	768	{
762	if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count,	769	if (CC(nested_vmx_check_msr_switch(vcpu,
763	vmcs12->vm_exit_msr_load_addr) \|\|	770	vmcs12->vm_exit_msr_load_count,
764	nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count,	771	vmcs12->vm_exit_msr_load_addr)) \|\|
765	vmcs12->vm_exit_msr_store_addr))	772	CC(nested_vmx_check_msr_switch(vcpu,
		773	vmcs12->vm_exit_msr_store_count,
		774	vmcs12->vm_exit_msr_store_addr)))
766	return -EINVAL;	775	return -EINVAL;
767		776
768	return 0;	777	return 0;
@@ -771,8 +780,9 @@ static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, · 2 unmodified lines
771	static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,	780	static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
772	struct vmcs12 *vmcs12)	781	struct vmcs12 *vmcs12)
773	{	782	{
774	if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count,	783	if (CC(nested_vmx_check_msr_switch(vcpu,
775	vmcs12->vm_entry_msr_load_addr))	784	vmcs12->vm_entry_msr_load_count,
		785	vmcs12->vm_entry_msr_load_addr)))
776	return -EINVAL;	786	return -EINVAL;
777		787
778	return 0;	788	return 0;
@@ -784,8 +794,8 @@ static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, · 5 unmodified lines
784	if (!nested_cpu_has_pml(vmcs12))	794	if (!nested_cpu_has_pml(vmcs12))
785	return 0;	795	return 0;
786		796
787	if (!nested_cpu_has_ept(vmcs12) \|\|	797	if (CC(!nested_cpu_has_ept(vmcs12)) \|\|
788	!page_address_valid(vcpu, vmcs12->pml_address))	798	CC(!page_address_valid(vcpu, vmcs12->pml_address)))
789	return -EINVAL;	799	return -EINVAL;
790		800
791	return 0;	801	return 0;
@@ -794,8 +804,8 @@ static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, · 2 unmodified lines
794	static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,	804	static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
795	struct vmcs12 *vmcs12)	805	struct vmcs12 *vmcs12)
796	{	806	{
797	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&	807	if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
798	!nested_cpu_has_ept(vmcs12))	808	!nested_cpu_has_ept(vmcs12)))
799	return -EINVAL;	809	return -EINVAL;
800	return 0;	810	return 0;
801	}	811	}
@@ -803,8 +813,8 @@ static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
803	static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,	813	static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
804	struct vmcs12 *vmcs12)	814	struct vmcs12 *vmcs12)
805	{	815	{
806	if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&	816	if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
807	!nested_cpu_has_ept(vmcs12))	817	!nested_cpu_has_ept(vmcs12)))
808	return -EINVAL;	818	return -EINVAL;
809	return 0;	819	return 0;
810	}	820	}
@@ -815,8 +825,8 @@ static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
815	if (!nested_cpu_has_shadow_vmcs(vmcs12))	825	if (!nested_cpu_has_shadow_vmcs(vmcs12))
816	return 0;	826	return 0;
817		827
818	if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) \|\|	828	if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) \|\|
819	!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))	829	CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
820	return -EINVAL;	830	return -EINVAL;
821		831
822	return 0;	832	return 0;
@@ -826,12 +836,12 @@ static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, · 3 unmodified lines
826	struct vmx_msr_entry *e)	836	struct vmx_msr_entry *e)
827	{	837	{
828	/* x2APIC MSR accesses are not allowed */	838	/* x2APIC MSR accesses are not allowed */
829	if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)	839	if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
830	return -EINVAL;	840	return -EINVAL;
831	if (e->index == MSR_IA32_UCODE_WRITE \|\| /* SDM Table 35-2 */	841	if (CC(e->index == MSR_IA32_UCODE_WRITE) \|\| /* SDM Table 35-2 */
832	e->index == MSR_IA32_UCODE_REV)	842	CC(e->index == MSR_IA32_UCODE_REV))
833	return -EINVAL;	843	return -EINVAL;
834	if (e->reserved != 0)	844	if (CC(e->reserved != 0))
835	return -EINVAL;	845	return -EINVAL;
836	return 0;	846	return 0;
837	}	847	}
@@ -839,9 +849,9 @@ static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, · 1 unmodified line
839	static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,	849	static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
840	struct vmx_msr_entry *e)	850	struct vmx_msr_entry *e)
841	{	851	{
842	if (e->index == MSR_FS_BASE \|\|	852	if (CC(e->index == MSR_FS_BASE) \|\|
843	e->index == MSR_GS_BASE \|\|	853	CC(e->index == MSR_GS_BASE) \|\|
844	e->index == MSR_IA32_SMM_MONITOR_CTL \|\| /* SMM is not supported */	854	CC(e->index == MSR_IA32_SMM_MONITOR_CTL) \|\| /* SMM is not supported */
845	nested_vmx_msr_check_common(vcpu, e))	855	nested_vmx_msr_check_common(vcpu, e))
846	return -EINVAL;	856	return -EINVAL;
847	return 0;	857	return 0;
@@ -850,7 +860,7 @@ static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, · 2 unmodified lines
850	static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,	860	static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
851	struct vmx_msr_entry *e)	861	struct vmx_msr_entry *e)
852	{	862	{
853	if (e->index == MSR_IA32_SMBASE \|\| /* SMM is not supported */	863	if (CC(e->index == MSR_IA32_SMBASE) \|\| /* SMM is not supported */
854	nested_vmx_msr_check_common(vcpu, e))	864	nested_vmx_msr_check_common(vcpu, e))
855	return -EINVAL;	865	return -EINVAL;
856	return 0;	866	return 0;
@@ -864,9 +874,7 @@ static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 7 unmodified lines
864	{	874	{
865	u32 i;	875	u32 i;
866	struct vmx_msr_entry e;	876	struct vmx_msr_entry e;
867	struct msr_data msr;
868		877
869	msr.host_initiated = false;
870	for (i = 0; i < count; i++) {	878	for (i = 0; i < count; i++) {
871	if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),	879	if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
872	&e, sizeof(e))) {	880	&e, sizeof(e))) {
@@ -881,9 +889,7 @@ static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 8 unmodified lines
881	__func__, i, e.index, e.reserved);	889	__func__, i, e.index, e.reserved);
882	goto fail;	890	goto fail;
883	}	891	}
884	msr.index = e.index;	892	if (kvm_set_msr(vcpu, e.index, e.value)) {
885	msr.data = e.value;
886	if (kvm_set_msr(vcpu, &msr)) {
887	pr_debug_ratelimited(	893	pr_debug_ratelimited(
888	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",	894	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
889	__func__, i, e.index, e.value);	895	__func__, i, e.index, e.value);
@@ -897,11 +903,11 @@ fail: · 7 unmodified lines
897		903
898	static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)	904	static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
899	{	905	{
		906	u64 data;
900	u32 i;	907	u32 i;
901	struct vmx_msr_entry e;	908	struct vmx_msr_entry e;
902		909
903	for (i = 0; i < count; i++) {	910	for (i = 0; i < count; i++) {
904	struct msr_data msr_info;
905	if (kvm_vcpu_read_guest(vcpu,	911	if (kvm_vcpu_read_guest(vcpu,
906	gpa + i * sizeof(e),	912	gpa + i * sizeof(e),
907	&e, 2 * sizeof(u32))) {	913	&e, 2 * sizeof(u32))) {
@@ -916,9 +922,7 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 8 unmodified lines
916	__func__, i, e.index, e.reserved);	922	__func__, i, e.index, e.reserved);
917	return -EINVAL;	923	return -EINVAL;
918	}	924	}
919	msr_info.host_initiated = false;	925	if (kvm_get_msr(vcpu, e.index, &data)) {
920	msr_info.index = e.index;
921	if (kvm_get_msr(vcpu, &msr_info)) {
922	pr_debug_ratelimited(	926	pr_debug_ratelimited(
923	"%s cannot read MSR (%u, 0x%x)\n",	927	"%s cannot read MSR (%u, 0x%x)\n",
924	__func__, i, e.index);	928	__func__, i, e.index);
@@ -927,10 +931,10 @@ static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) · 2 unmodified lines
927	if (kvm_vcpu_write_guest(vcpu,	931	if (kvm_vcpu_write_guest(vcpu,
928	gpa + i * sizeof(e) +	932	gpa + i * sizeof(e) +
929	offsetof(struct vmx_msr_entry, value),	933	offsetof(struct vmx_msr_entry, value),
930	&msr_info.data, sizeof(msr_info.data))) {	934	&data, sizeof(data))) {
931	pr_debug_ratelimited(	935	pr_debug_ratelimited(
932	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",	936	"%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
933	__func__, i, e.index, msr_info.data);	937	__func__, i, e.index, data);
934	return -EINVAL;	938	return -EINVAL;
935	}	939	}
936	}	940	}
@@ -955,7 +959,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne · 18 unmodified lines
955	u32 *entry_failure_code)	959	u32 *entry_failure_code)
956	{	960	{
957	if (cr3 != kvm_read_cr3(vcpu) \|\| (!nested_ept && pdptrs_changed(vcpu))) {	961	if (cr3 != kvm_read_cr3(vcpu) \|\| (!nested_ept && pdptrs_changed(vcpu))) {
958	if (!nested_cr3_valid(vcpu, cr3)) {	962	if (CC(!nested_cr3_valid(vcpu, cr3))) {
959	*entry_failure_code = ENTRY_FAIL_DEFAULT;	963	*entry_failure_code = ENTRY_FAIL_DEFAULT;
960	return -EINVAL;	964	return -EINVAL;
961	}	965	}
@@ -965,7 +969,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne · 3 unmodified lines
965	* must not be dereferenced.	969	* must not be dereferenced.
966	*/	970	*/
967	if (is_pae_paging(vcpu) && !nested_ept) {	971	if (is_pae_paging(vcpu) && !nested_ept) {
968	if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {	972	if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) {
969	*entry_failure_code = ENTRY_FAIL_PDPTE;	973	*entry_failure_code = ENTRY_FAIL_PDPTE;
970	return -EINVAL;	974	return -EINVAL;
971	}	975	}
@@ -2411,12 +2415,12 @@ static int prepare_vmcs02(struct kvm_vcpu vcpu, struct vmcs12 vmcs12, · 1439 unmodified lines
2411		2415
2412	static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)	2416	static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2413	{	2417	{
2414	if (!nested_cpu_has_nmi_exiting(vmcs12) &&	2418	if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
2415	nested_cpu_has_virtual_nmis(vmcs12))	2419	nested_cpu_has_virtual_nmis(vmcs12)))
2416	return -EINVAL;	2420	return -EINVAL;
2417		2421
2418	if (!nested_cpu_has_virtual_nmis(vmcs12) &&	2422	if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
2419	nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING))	2423	nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)))
2420	return -EINVAL;	2424	return -EINVAL;
2421		2425
2422	return 0;	2426	return 0;
@@ -2430,11 +2434,11 @@ static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) · 7 unmodified lines
2430	/* Check for memory type validity */	2434	/* Check for memory type validity */
2431	switch (address & VMX_EPTP_MT_MASK) {	2435	switch (address & VMX_EPTP_MT_MASK) {
2432	case VMX_EPTP_MT_UC:	2436	case VMX_EPTP_MT_UC:
2433	if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))	2437	if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2434	return false;	2438	return false;
2435	break;	2439	break;
2436	case VMX_EPTP_MT_WB:	2440	case VMX_EPTP_MT_WB:
2437	if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))	2441	if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2438	return false;	2442	return false;
2439	break;	2443	break;
2440	default:	2444	default:
@@ -2442,16 +2446,16 @@ static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) · 1 unmodified line
2442	}	2446	}
2443		2447
2444	/* only 4 levels page-walk length are valid */	2448	/* only 4 levels page-walk length are valid */
2445	if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)	2449	if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4))
2446	return false;	2450	return false;
2447		2451
2448	/* Reserved bits should not be set */	2452	/* Reserved bits should not be set */
2449	if (address >> maxphyaddr \|\| ((address >> 7) & 0x1f))	2453	if (CC(address >> maxphyaddr \|\| ((address >> 7) & 0x1f)))
2450	return false;	2454	return false;
2451		2455
2452	/* AD, if set, should be supported */	2456	/* AD, if set, should be supported */
2453	if (address & VMX_EPTP_AD_ENABLE_BIT) {	2457	if (address & VMX_EPTP_AD_ENABLE_BIT) {
2454	if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))	2458	if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2455	return false;	2459	return false;
2456	}	2460	}
2457		2461
@@ -2466,21 +2470,21 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 8 unmodified lines
2466	{	2470	{
2467	struct vcpu_vmx *vmx = to_vmx(vcpu);	2471	struct vcpu_vmx *vmx = to_vmx(vcpu);
2468		2472
2469	if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control,	2473	if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2470	vmx->nested.msrs.pinbased_ctls_low,	2474	vmx->nested.msrs.pinbased_ctls_low,
2471	vmx->nested.msrs.pinbased_ctls_high) \|\|	2475	vmx->nested.msrs.pinbased_ctls_high)) \|\|
2472	!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,	2476	CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2473	vmx->nested.msrs.procbased_ctls_low,	2477	vmx->nested.msrs.procbased_ctls_low,
2474	vmx->nested.msrs.procbased_ctls_high))	2478	vmx->nested.msrs.procbased_ctls_high)))
2475	return -EINVAL;	2479	return -EINVAL;
2476		2480
2477	if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&	2481	if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2478	!vmx_control_verify(vmcs12->secondary_vm_exec_control,	2482	CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
2479	vmx->nested.msrs.secondary_ctls_low,	2483	vmx->nested.msrs.secondary_ctls_low,
2480	vmx->nested.msrs.secondary_ctls_high))	2484	vmx->nested.msrs.secondary_ctls_high)))
2481	return -EINVAL;	2485	return -EINVAL;
2482		2486
2483	if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) \|\|	2487	if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) \|\|
2484	nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) \|\|	2488	nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) \|\|
2485	nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) \|\|	2489	nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) \|\|
2486	nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) \|\|	2490	nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) \|\|
@@ -2491,7 +2495,7 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 4 unmodified lines
2491	nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) \|\|	2495	nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) \|\|
2492	nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) \|\|	2496	nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) \|\|
2493	nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) \|\|	2497	nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) \|\|
2494	(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))	2498	CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2495	return -EINVAL;	2499	return -EINVAL;
2496		2500
2497	if (!nested_cpu_has_preemption_timer(vmcs12) &&	2501	if (!nested_cpu_has_preemption_timer(vmcs12) &&
@@ -2499,17 +2503,17 @@ static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
2499	return -EINVAL;	2503	return -EINVAL;
2500		2504
2501	if (nested_cpu_has_ept(vmcs12) &&	2505	if (nested_cpu_has_ept(vmcs12) &&
2502	!valid_ept_address(vcpu, vmcs12->ept_pointer))	2506	CC(!valid_ept_address(vcpu, vmcs12->ept_pointer)))
2503	return -EINVAL;	2507	return -EINVAL;
2504		2508
2505	if (nested_cpu_has_vmfunc(vmcs12)) {	2509	if (nested_cpu_has_vmfunc(vmcs12)) {
2506	if (vmcs12->vm_function_control &	2510	if (CC(vmcs12->vm_function_control &
2507	~vmx->nested.msrs.vmfunc_controls)	2511	~vmx->nested.msrs.vmfunc_controls))
2508	return -EINVAL;	2512	return -EINVAL;
2509		2513
2510	if (nested_cpu_has_eptp_switching(vmcs12)) {	2514	if (nested_cpu_has_eptp_switching(vmcs12)) {
2511	if (!nested_cpu_has_ept(vmcs12) \|\|	2515	if (CC(!nested_cpu_has_ept(vmcs12)) \|\|
2512	!page_address_valid(vcpu, vmcs12->eptp_list_address))	2516	CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
2513	return -EINVAL;	2517	return -EINVAL;
2514	}	2518	}
2515	}	2519	}
@@ -2525,10 +2529,10 @@ static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, · 9 unmodified lines
2525	{	2529	{
2526	struct vcpu_vmx *vmx = to_vmx(vcpu);	2530	struct vcpu_vmx *vmx = to_vmx(vcpu);
2527		2531
2528	if (!vmx_control_verify(vmcs12->vm_exit_controls,	2532	if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
2529	vmx->nested.msrs.exit_ctls_low,	2533	vmx->nested.msrs.exit_ctls_low,
2530	vmx->nested.msrs.exit_ctls_high) \|\|	2534	vmx->nested.msrs.exit_ctls_high)) \|\|
2531	nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))	2535	CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
2532	return -EINVAL;	2536	return -EINVAL;
2533		2537
2534	return 0;	2538	return 0;
@@ -2542,9 +2546,9 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 7 unmodified lines
2542	{	2546	{
2543	struct vcpu_vmx *vmx = to_vmx(vcpu);	2547	struct vcpu_vmx *vmx = to_vmx(vcpu);
2544		2548
2545	if (!vmx_control_verify(vmcs12->vm_entry_controls,	2549	if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
2546	vmx->nested.msrs.entry_ctls_low,	2550	vmx->nested.msrs.entry_ctls_low,
2547	vmx->nested.msrs.entry_ctls_high))	2551	vmx->nested.msrs.entry_ctls_high)))
2548	return -EINVAL;	2552	return -EINVAL;
2549		2553
2550	/*	2554	/*
@@ -2564,31 +2568,31 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 13 unmodified lines
2564	bool prot_mode = !urg \|\| vmcs12->guest_cr0 & X86_CR0_PE;	2568	bool prot_mode = !urg \|\| vmcs12->guest_cr0 & X86_CR0_PE;
2565		2569
2566	/* VM-entry interruption-info field: interruption type */	2570	/* VM-entry interruption-info field: interruption type */
2567	if (intr_type == INTR_TYPE_RESERVED \|\|	2571	if (CC(intr_type == INTR_TYPE_RESERVED) \|\|
2568	(intr_type == INTR_TYPE_OTHER_EVENT &&	2572	CC(intr_type == INTR_TYPE_OTHER_EVENT &&
2569	!nested_cpu_supports_monitor_trap_flag(vcpu)))	2573	!nested_cpu_supports_monitor_trap_flag(vcpu)))
2570	return -EINVAL;	2574	return -EINVAL;
2571		2575
2572	/* VM-entry interruption-info field: vector */	2576	/* VM-entry interruption-info field: vector */
2573	if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) \|\|	2577	if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) \|\|
2574	(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) \|\|	2578	CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) \|\|
2575	(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))	2579	CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2576	return -EINVAL;	2580	return -EINVAL;
2577		2581
2578	/* VM-entry interruption-info field: deliver error code */	2582	/* VM-entry interruption-info field: deliver error code */
2579	should_have_error_code =	2583	should_have_error_code =
2580	intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&	2584	intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2581	x86_exception_has_error_code(vector);	2585	x86_exception_has_error_code(vector);
2582	if (has_error_code != should_have_error_code)	2586	if (CC(has_error_code != should_have_error_code))
2583	return -EINVAL;	2587	return -EINVAL;
2584		2588
2585	/* VM-entry exception error code */	2589	/* VM-entry exception error code */
2586	if (has_error_code &&	2590	if (CC(has_error_code &&
2587	vmcs12->vm_entry_exception_error_code & GENMASK(31, 15))	2591	vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)))
2588	return -EINVAL;	2592	return -EINVAL;
2589		2593
2590	/* VM-entry interruption-info field: reserved bits */	2594	/* VM-entry interruption-info field: reserved bits */
2591	if (intr_info & INTR_INFO_RESVD_BITS_MASK)	2595	if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
2592	return -EINVAL;	2596	return -EINVAL;
2593		2597
2594	/* VM-entry instruction length */	2598	/* VM-entry instruction length */
@@ -2596,9 +2600,9 @@ static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, · 1 unmodified line
2596	case INTR_TYPE_SOFT_EXCEPTION:	2600	case INTR_TYPE_SOFT_EXCEPTION:
2597	case INTR_TYPE_SOFT_INTR:	2601	case INTR_TYPE_SOFT_INTR:
2598	case INTR_TYPE_PRIV_SW_EXCEPTION:	2602	case INTR_TYPE_PRIV_SW_EXCEPTION:
2599	if ((vmcs12->vm_entry_instruction_len > 15) \|\|	2603	if (CC(vmcs12->vm_entry_instruction_len > 15) \|\|
2600	(vmcs12->vm_entry_instruction_len == 0 &&	2604	CC(vmcs12->vm_entry_instruction_len == 0 &&
2601	!nested_cpu_has_zero_length_injection(vcpu)))	2605	CC(!nested_cpu_has_zero_length_injection(vcpu))))
2602	return -EINVAL;	2606	return -EINVAL;
2603	}	2607	}
2604	}	2608	}
@@ -2625,40 +2629,40 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 20 unmodified lines
2625	{	2629	{
2626	bool ia32e;	2630	bool ia32e;
2627		2631
2628	if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) \|\|	2632	if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) \|\|
2629	!nested_host_cr4_valid(vcpu, vmcs12->host_cr4) \|\|	2633	CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) \|\|
2630	!nested_cr3_valid(vcpu, vmcs12->host_cr3))	2634	CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
2631	return -EINVAL;	2635	return -EINVAL;
2632		2636
2633	if (is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu) \|\|	2637	if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) \|\|
2634	is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu))	2638	CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
2635	return -EINVAL;	2639	return -EINVAL;
2636		2640
2637	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&	2641	if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
2638	!kvm_pat_valid(vmcs12->host_ia32_pat))	2642	CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
2639	return -EINVAL;	2643	return -EINVAL;
2640		2644
2641	ia32e = (vmcs12->vm_exit_controls &	2645	ia32e = (vmcs12->vm_exit_controls &
2642	VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;	2646	VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
2643		2647
2644	if (vmcs12->host_cs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2648	if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2645	vmcs12->host_ss_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2649	CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2646	vmcs12->host_ds_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2650	CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2647	vmcs12->host_es_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2651	CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2648	vmcs12->host_fs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2652	CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2649	vmcs12->host_gs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2653	CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2650	vmcs12->host_tr_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK) \|\|	2654	CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK \| SEGMENT_TI_MASK)) \|\|
2651	vmcs12->host_cs_selector == 0 \|\|	2655	CC(vmcs12->host_cs_selector == 0) \|\|
2652	vmcs12->host_tr_selector == 0 \|\|	2656	CC(vmcs12->host_tr_selector == 0) \|\|
2653	(vmcs12->host_ss_selector == 0 && !ia32e))	2657	CC(vmcs12->host_ss_selector == 0 && !ia32e))
2654	return -EINVAL;	2658	return -EINVAL;
2655		2659
2656	#ifdef CONFIG_X86_64	2660	#ifdef CONFIG_X86_64
2657	if (is_noncanonical_address(vmcs12->host_fs_base, vcpu) \|\|	2661	if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) \|\|
2658	is_noncanonical_address(vmcs12->host_gs_base, vcpu) \|\|	2662	CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) \|\|
2659	is_noncanonical_address(vmcs12->host_gdtr_base, vcpu) \|\|	2663	CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) \|\|
2660	is_noncanonical_address(vmcs12->host_idtr_base, vcpu) \|\|	2664	CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) \|\|
2661	is_noncanonical_address(vmcs12->host_tr_base, vcpu))	2665	CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)))
2662	return -EINVAL;	2666	return -EINVAL;
2663	#endif	2667	#endif
2664		2668
@@ -2669,9 +2673,9 @@ static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, · 4 unmodified lines
2669	* the host address-space size VM-exit control.	2673	* the host address-space size VM-exit control.
2670	*/	2674	*/
2671	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {	2675	if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
2672	if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) \|\|	2676	if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) \|\|
2673	ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) \|\|	2677	CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) \|\|
2674	ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))	2678	CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
2675	return -EINVAL;	2679	return -EINVAL;
2676	}	2680	}
2677		2681
@@ -2688,16 +2692,16 @@ static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, · 10 unmodified lines
2688	if (vmcs12->vmcs_link_pointer == -1ull)	2692	if (vmcs12->vmcs_link_pointer == -1ull)
2689	return 0;	2693	return 0;
2690		2694
2691	if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))	2695	if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
2692	return -EINVAL;	2696	return -EINVAL;
2693		2697
2694	if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))	2698	if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)))
2695	return -EINVAL;	2699	return -EINVAL;
2696		2700
2697	shadow = map.hva;	2701	shadow = map.hva;
2698		2702
2699	if (shadow->hdr.revision_id != VMCS12_REVISION \|\|	2703	if (CC(shadow->hdr.revision_id != VMCS12_REVISION) \|\|
2700	shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))	2704	CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
2701	r = -EINVAL;	2705	r = -EINVAL;
2702		2706
2703	kvm_vcpu_unmap(vcpu, &map, false);	2707	kvm_vcpu_unmap(vcpu, &map, false);
@@ -2709,8 +2713,8 @@ static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, · 5 unmodified lines
2709	*/	2713	*/
2710	static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)	2714	static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
2711	{	2715	{
2712	if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&	2716	if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
2713	vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)	2717	vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT))
2714	return -EINVAL;	2718	return -EINVAL;
2715		2719
2716	return 0;	2720	return 0;
@@ -2724,12 +2728,12 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, · 7 unmodified lines
2724		2728
2725	*exit_qual = ENTRY_FAIL_DEFAULT;	2729	*exit_qual = ENTRY_FAIL_DEFAULT;
2726		2730
2727	if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) \|\|	2731	if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) \|\|
2728	!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))	2732	CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
2729	return -EINVAL;	2733	return -EINVAL;
2730		2734
2731	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&	2735	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
2732	!kvm_pat_valid(vmcs12->guest_ia32_pat))	2736	CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
2733	return -EINVAL;	2737	return -EINVAL;
2734		2738
2735	if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {	2739	if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
@@ -2749,16 +2753,16 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, · 13 unmodified lines
2749	if (to_vmx(vcpu)->nested.nested_run_pending &&	2753	if (to_vmx(vcpu)->nested.nested_run_pending &&
2750	(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {	2754	(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
2751	ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;	2755	ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
2752	if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) \|\|	2756	if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) \|\|
2753	ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) \|\|	2757	CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) \|\|
2754	((vmcs12->guest_cr0 & X86_CR0_PG) &&	2758	CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
2755	ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))	2759	ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
2756	return -EINVAL;	2760	return -EINVAL;
2757	}	2761	}
2758		2762
2759	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&	2763	if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
2760	(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) \|\|	2764	(CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) \|\|
2761	(vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))	2765	CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
2762	return -EINVAL;	2766	return -EINVAL;
2763		2767
2764	if (nested_check_guest_non_reg_state(vmcs12))	2768	if (nested_check_guest_non_reg_state(vmcs12))
@@ -2841,9 +2845,13 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) · 76 unmodified lines
2841	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);	2845	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2842		2846
2843	if (vm_fail) {	2847	if (vm_fail) {
		2848	u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
		2849
2844	preempt_enable();	2850	preempt_enable();
2845	WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=	2851
2846	VMXERR_ENTRY_INVALID_CONTROL_FIELD);	2852	trace_kvm_nested_vmenter_failed(
		2853	"early hardware check VM-instruction error: ", error);
		2854	WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
2847	return 1;	2855	return 1;
2848	}	2856	}
2849		2857
@@ -3401,6 +3409,15 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) · 551 unmodified lines
3401	unsigned long exit_qual;	3409	unsigned long exit_qual;
3402	bool block_nested_events =	3410	bool block_nested_events =
3403	vmx->nested.nested_run_pending \|\| kvm_event_needs_reinjection(vcpu);	3411	vmx->nested.nested_run_pending \|\| kvm_event_needs_reinjection(vcpu);
		3412	struct kvm_lapic *apic = vcpu->arch.apic;
		3413
		3414	if (lapic_in_kernel(vcpu) &&
		3415	test_bit(KVM_APIC_INIT, &apic->pending_events)) {
		3416	if (block_nested_events)
		3417	return -EBUSY;
		3418	nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
		3419	return 0;
		3420	}
3404		3421
3405	if (vcpu->arch.exception.pending &&	3422	if (vcpu->arch.exception.pending &&
3406	nested_vmx_check_exception(vcpu, &exit_qual)) {	3423	nested_vmx_check_exception(vcpu, &exit_qual)) {
@@ -3889,7 +3906,6 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 482 unmodified lines
3889	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);	3906	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3890	struct vcpu_vmx *vmx = to_vmx(vcpu);	3907	struct vcpu_vmx *vmx = to_vmx(vcpu);
3891	struct vmx_msr_entry g, h;	3908	struct vmx_msr_entry g, h;
3892	struct msr_data msr;
3893	gpa_t gpa;	3909	gpa_t gpa;
3894	u32 i, j;	3910	u32 i, j;
3895		3911
@@ -3949,7 +3965,6 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 53 unmodified lines
3949	* from the guest value. The intent is to stuff host state as	3965	* from the guest value. The intent is to stuff host state as
3950	* silently as possible, not to fully process the exit load list.	3966	* silently as possible, not to fully process the exit load list.
3951	*/	3967	*/
3952	msr.host_initiated = false;
3953	for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {	3968	for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
3954	gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));	3969	gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
3955	if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {	3970	if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
@@ -3979,9 +3994,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) · 23 unmodified lines
3979	goto vmabort;	3994	goto vmabort;
3980	}	3995	}
3981		3996
3982	msr.index = h.index;	3997	if (kvm_set_msr(vcpu, h.index, h.value)) {
3983	msr.data = h.value;
3984	if (kvm_set_msr(vcpu, &msr)) {
3985	pr_debug_ratelimited(	3998	pr_debug_ratelimited(
3986	"%s WRMSR failed (%u, 0x%x, 0x%llx)\n",	3999	"%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
3987	__func__, j, h.index, h.value);	4000	__func__, j, h.index, h.value);
@@ -4466,7 +4479,12 @@ static int handle_vmoff(struct kvm_vcpu *vcpu) · 478 unmodified lines
4466	{	4479	{
4467	if (!nested_vmx_check_permission(vcpu))	4480	if (!nested_vmx_check_permission(vcpu))
4468	return 1;	4481	return 1;
		4482
4469	free_nested(vcpu);	4483	free_nested(vcpu);
		4484
		4485	/* Process a latched INIT during time CPU was in VMX operation */
		4486	kvm_make_request(KVM_REQ_EVENT, vcpu);
		4487
4470	return nested_vmx_succeed(vcpu);	4488	return nested_vmx_succeed(vcpu);
4471	}	4489	}
4472		4490
@@ -5261,8 +5279,9 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) · 788 unmodified lines
5261	return false;	5279	return false;
5262		5280
5263	if (unlikely(vmx->fail)) {	5281	if (unlikely(vmx->fail)) {
5264	pr_info_ratelimited("%s failed vm entry %x\n", __func__,	5282	trace_kvm_nested_vmenter_failed(
5265	vmcs_read32(VM_INSTRUCTION_ERROR));	5283	"hardware VM-instruction error: ",
		5284	vmcs_read32(VM_INSTRUCTION_ERROR));
5266	return true;	5285	return true;
5267	}	5286	}
5268		5287

arch/x86/kvm/vmx/vmenter.S+2−2

@@ -94,7 +94,7 @@ ENDPROC(vmx_vmexit) · 93 unmodified lines
94		94
95	/**	95	/**
96	* __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode	96	* __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
97	* @vmx: struct vcpu_vmx *	97	* @vmx: struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
98	* @regs: unsigned long * (to guest registers)	98	* @regs: unsigned long * (to guest registers)
99	* @launched: %true if the VMCS has been launched	99	* @launched: %true if the VMCS has been launched
100	*	100	*
@@ -151,7 +151,7 @@ ENTRY(__vmx_vcpu_run) · 50 unmodified lines
151	mov VCPU_R14(%_ASM_AX), %r14	151	mov VCPU_R14(%_ASM_AX), %r14
152	mov VCPU_R15(%_ASM_AX), %r15	152	mov VCPU_R15(%_ASM_AX), %r15
153	#endif	153	#endif
154	/* Load guest RAX. This kills the vmx_vcpu pointer! */	154	/* Load guest RAX. This kills the @regs pointer! */
155	mov VCPU_RAX(%_ASM_AX), %_ASM_AX	155	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
156		156
157	/* Enter guest mode */	157	/* Enter guest mode */

arch/x86/kvm/vmx/vmx.c+48−46

@@ -1472,8 +1472,11 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) · 1471 unmodified lines
1472	return 0;	1472	return 0;
1473	}	1473	}
1474		1474
1475		1475	/*
1476	static void skip_emulated_instruction(struct kvm_vcpu *vcpu)	1476	* Returns an int to be compatible with SVM implementation (which can fail).
		1477	* Do not use directly, use skip_emulated_instruction() instead.
		1478	*/
		1479	static int __skip_emulated_instruction(struct kvm_vcpu *vcpu)
1477	{	1480	{
1478	unsigned long rip;	1481	unsigned long rip;
1479		1482
@@ -1483,6 +1486,13 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) · 3 unmodified lines
1483		1486
1484	/* skipping an emulated instruction also counts */	1487	/* skipping an emulated instruction also counts */
1485	vmx_set_interrupt_shadow(vcpu, 0);	1488	vmx_set_interrupt_shadow(vcpu, 0);
		1489
		1490	return EMULATE_DONE;
		1491	}
		1492
		1493	static inline void skip_emulated_instruction(struct kvm_vcpu *vcpu)
		1494	{
		1495	(void)__skip_emulated_instruction(vcpu);
1486	}	1496	}
1487		1497
1488	static void vmx_clear_hlt(struct kvm_vcpu *vcpu)	1498	static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
@@ -4026,7 +4036,7 @@ static void ept_set_mmio_spte_mask(void) · 2537 unmodified lines
4026	* of an EPT paging-structure entry is 110b (write/execute).	4036	* of an EPT paging-structure entry is 110b (write/execute).
4027	*/	4037	*/
4028	kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,	4038	kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
4029	VMX_EPT_MISCONFIG_WX_VALUE);	4039	VMX_EPT_MISCONFIG_WX_VALUE, 0);
4030	}	4040	}
4031		4041
4032	#define VMX_XSS_EXIT_BITMAP 0	4042	#define VMX_XSS_EXIT_BITMAP 0
@@ -4152,6 +4162,7 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) · 119 unmodified lines
4152		4162
4153	vcpu->arch.microcode_version = 0x100000000ULL;	4163	vcpu->arch.microcode_version = 0x100000000ULL;
4154	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();	4164	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
		4165	vmx->hv_deadline_tsc = -1;
4155	kvm_set_cr8(vcpu, 0);	4166	kvm_set_cr8(vcpu, 0);
4156		4167
4157	if (!init_event) {	4168	if (!init_event) {
@@ -4856,41 +4867,12 @@ static int handle_cpuid(struct kvm_vcpu *vcpu) · 698 unmodified lines
4856		4867
4857	static int handle_rdmsr(struct kvm_vcpu *vcpu)	4868	static int handle_rdmsr(struct kvm_vcpu *vcpu)
4858	{	4869	{
4859	u32 ecx = kvm_rcx_read(vcpu);	4870	return kvm_emulate_rdmsr(vcpu);
4860	struct msr_data msr_info;
4861
4862	msr_info.index = ecx;
4863	msr_info.host_initiated = false;
4864	if (vmx_get_msr(vcpu, &msr_info)) {
4865	trace_kvm_msr_read_ex(ecx);
4866	kvm_inject_gp(vcpu, 0);
4867	return 1;
4868	}
4869
4870	trace_kvm_msr_read(ecx, msr_info.data);
4871
4872	kvm_rax_write(vcpu, msr_info.data & -1u);
4873	kvm_rdx_write(vcpu, (msr_info.data >> 32) & -1u);
4874	return kvm_skip_emulated_instruction(vcpu);
4875	}	4871	}
4876		4872
4877	static int handle_wrmsr(struct kvm_vcpu *vcpu)	4873	static int handle_wrmsr(struct kvm_vcpu *vcpu)
4878	{	4874	{
4879	struct msr_data msr;	4875	return kvm_emulate_wrmsr(vcpu);
4880	u32 ecx = kvm_rcx_read(vcpu);
4881	u64 data = kvm_read_edx_eax(vcpu);
4882
4883	msr.data = data;
4884	msr.index = ecx;
4885	msr.host_initiated = false;
4886	if (kvm_set_msr(vcpu, &msr) != 0) {
4887	trace_kvm_msr_write_ex(ecx, data);
4888	kvm_inject_gp(vcpu, 0);
4889	return 1;
4890	}
4891
4892	trace_kvm_msr_write(ecx, data);
4893	return kvm_skip_emulated_instruction(vcpu);
4894	}	4876	}
4895		4877
4896	static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)	4878	static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
@@ -5227,31 +5209,33 @@ emulation_error: · 330 unmodified lines
5227	static void grow_ple_window(struct kvm_vcpu *vcpu)	5209	static void grow_ple_window(struct kvm_vcpu *vcpu)
5228	{	5210	{
5229	struct vcpu_vmx *vmx = to_vmx(vcpu);	5211	struct vcpu_vmx *vmx = to_vmx(vcpu);
5230	int old = vmx->ple_window;	5212	unsigned int old = vmx->ple_window;
5231		5213
5232	vmx->ple_window = __grow_ple_window(old, ple_window,	5214	vmx->ple_window = __grow_ple_window(old, ple_window,
5233	ple_window_grow,	5215	ple_window_grow,
5234	ple_window_max);	5216	ple_window_max);
5235		5217
5236	if (vmx->ple_window != old)	5218	if (vmx->ple_window != old) {
5237	vmx->ple_window_dirty = true;	5219	vmx->ple_window_dirty = true;
5238		5220	trace_kvm_ple_window_update(vcpu->vcpu_id,
5239	trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);	5221	vmx->ple_window, old);
		5222	}
5240	}	5223	}
5241		5224
5242	static void shrink_ple_window(struct kvm_vcpu *vcpu)	5225	static void shrink_ple_window(struct kvm_vcpu *vcpu)
5243	{	5226	{
5244	struct vcpu_vmx *vmx = to_vmx(vcpu);	5227	struct vcpu_vmx *vmx = to_vmx(vcpu);
5245	int old = vmx->ple_window;	5228	unsigned int old = vmx->ple_window;
5246		5229
5247	vmx->ple_window = __shrink_ple_window(old, ple_window,	5230	vmx->ple_window = __shrink_ple_window(old, ple_window,
5248	ple_window_shrink,	5231	ple_window_shrink,
5249	ple_window);	5232	ple_window);
5250		5233
5251	if (vmx->ple_window != old)	5234	if (vmx->ple_window != old) {
5252	vmx->ple_window_dirty = true;	5235	vmx->ple_window_dirty = true;
5253		5236	trace_kvm_ple_window_update(vcpu->vcpu_id,
5254	trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);	5237	vmx->ple_window, old);
		5238	}
5255	}	5239	}
5256		5240
5257	/*	5241	/*
@@ -5887,8 +5871,13 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu) · 629 unmodified lines
5887	else {	5871	else {
5888	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",	5872	vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
5889	exit_reason);	5873	exit_reason);
5890	kvm_queue_exception(vcpu, UD_VECTOR);	5874	dump_vmcs();
5891	return 1;	5875	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		5876	vcpu->run->internal.suberror =
		5877	KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
		5878	vcpu->run->internal.ndata = 1;
		5879	vcpu->run->internal.data[0] = exit_reason;
		5880	return 0;
5892	}	5881	}
5893	}	5882	}
5894		5883
@@ -6615,6 +6604,9 @@ static struct kvm_vcpu vmx_create_vcpu(struct kvm kvm, unsigned int id) · 720 unmodified lines
6615	unsigned long *msr_bitmap;	6604	unsigned long *msr_bitmap;
6616	int cpu;	6605	int cpu;
6617		6606
		6607	BUILD_BUG_ON_MSG(offsetof(struct vcpu_vmx, vcpu) != 0,
		6608	"struct kvm_vcpu must be at offset 0 for arch usercopy region");
		6609
6618	vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);	6610	vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
6619	if (!vmx)	6611	if (!vmx)
6620	return ERR_PTR(-ENOMEM);	6612	return ERR_PTR(-ENOMEM);
@@ -7369,10 +7361,14 @@ static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq, · 748 unmodified lines
7369	* irqbalance to make the interrupts single-CPU.	7361	* irqbalance to make the interrupts single-CPU.
7370	*	7362	*
7371	* We will support full lowest-priority interrupt later.	7363	* We will support full lowest-priority interrupt later.
		7364	*
		7365	* In addition, we can only inject generic interrupts using
		7366	* the PI mechanism, refuse to route others through it.
7372	*/	7367	*/
7373		7368
7374	kvm_set_msi_irq(kvm, e, &irq);	7369	kvm_set_msi_irq(kvm, e, &irq);
7375	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {	7370	if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) \|\|
		7371	!kvm_irq_is_postable(&irq)) {
7376	/*	7372	/*
7377	* Make sure the IRTE is in remapped mode if	7373	* Make sure the IRTE is in remapped mode if
7378	* we don't handle it in posted mode.	7374	* we don't handle it in posted mode.
@@ -7474,6 +7470,11 @@ static bool vmx_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) · 95 unmodified lines
7474	return false;	7470	return false;
7475	}	7471	}
7476		7472
		7473	static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
		7474	{
		7475	return to_vmx(vcpu)->nested.vmxon;
		7476	}
		7477
7477	static __init int hardware_setup(void)	7478	static __init int hardware_setup(void)
7478	{	7479	{
7479	unsigned long host_bndcfgs;	7480	unsigned long host_bndcfgs;
@@ -7705,7 +7706,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { · 225 unmodified lines
7705		7706
7706	.run = vmx_vcpu_run,	7707	.run = vmx_vcpu_run,
7707	.handle_exit = vmx_handle_exit,	7708	.handle_exit = vmx_handle_exit,
7708	.skip_emulated_instruction = skip_emulated_instruction,	7709	.skip_emulated_instruction = __skip_emulated_instruction,
7709	.set_interrupt_shadow = vmx_set_interrupt_shadow,	7710	.set_interrupt_shadow = vmx_set_interrupt_shadow,
7710	.get_interrupt_shadow = vmx_get_interrupt_shadow,	7711	.get_interrupt_shadow = vmx_get_interrupt_shadow,
7711	.patch_hypercall = vmx_patch_hypercall,	7712	.patch_hypercall = vmx_patch_hypercall,
@@ -7799,6 +7800,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { · 87 unmodified lines
7799	.nested_enable_evmcs = NULL,	7800	.nested_enable_evmcs = NULL,
7800	.nested_get_evmcs_version = NULL,	7801	.nested_get_evmcs_version = NULL,
7801	.need_emulation_on_page_fault = vmx_need_emulation_on_page_fault,	7802	.need_emulation_on_page_fault = vmx_need_emulation_on_page_fault,
		7803	.apic_init_signal_blocked = vmx_apic_init_signal_blocked,
7802	};	7804	};
7803		7805
7804	static void vmx_cleanup_l1d_flush(void)	7806	static void vmx_cleanup_l1d_flush(void)

arch/x86/kvm/vmx/vmx.h+1−1

@@ -253,7 +253,7 @@ struct vcpu_vmx { · 252 unmodified lines
253	struct nested_vmx nested;	253	struct nested_vmx nested;
254		254
255	/* Dynamic PLE window. */	255	/* Dynamic PLE window. */
256	int ple_window;	256	unsigned int ple_window;
257	bool ple_window_dirty;	257	bool ple_window_dirty;
258		258
259	bool req_immediate_exit;	259	bool req_immediate_exit;

arch/x86/kvm/x86.c+128−69

@@ -674,8 +674,14 @@ static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, · 673 unmodified lines
674	data, offset, len, access);	674	data, offset, len, access);
675	}	675	}
676		676
		677	static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
		678	{
		679	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63) \| rsvd_bits(5, 8) \|
		680	rsvd_bits(1, 2);
		681	}
		682
677	/*	683	/*
678	* Load the pae pdptrs. Return true is they are all valid.	684	* Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise.
679	*/	685	*/
680	int load_pdptrs(struct kvm_vcpu vcpu, struct kvm_mmu mmu, unsigned long cr3)	686	int load_pdptrs(struct kvm_vcpu vcpu, struct kvm_mmu mmu, unsigned long cr3)
681	{	687	{
@@ -694,8 +700,7 @@ int load_pdptrs(struct kvm_vcpu vcpu, struct kvm_mmu mmu, unsigned long cr3) · 12 unmodified lines
694	}	700	}
695	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {	701	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
696	if ((pdpte[i] & PT_PRESENT_MASK) &&	702	if ((pdpte[i] & PT_PRESENT_MASK) &&
697	(pdpte[i] &	703	(pdpte[i] & pdptr_rsvd_bits(vcpu))) {
698	vcpu->arch.mmu->guest_rsvd_check.rsvd_bits_mask[0][2])) {
699	ret = 0;	704	ret = 0;
700	goto out;	705	goto out;
701	}	706	}
@@ -1254,6 +1259,13 @@ static u64 kvm_get_arch_capabilities(void) · 552 unmodified lines
1254	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)	1259	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1255	data \|= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;	1260	data \|= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1256		1261
		1262	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
		1263	data \|= ARCH_CAP_RDCL_NO;
		1264	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
		1265	data \|= ARCH_CAP_SSB_NO;
		1266	if (!boot_cpu_has_bug(X86_BUG_MDS))
		1267	data \|= ARCH_CAP_MDS_NO;
		1268
1257	return data;	1269	return data;
1258	}	1270	}
1259		1271
@@ -1351,19 +1363,23 @@ void kvm_enable_efer_bits(u64 mask) · 91 unmodified lines
1351	EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);	1363	EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1352		1364
1353	/*	1365	/*
1354	* Writes msr value into into the appropriate "register".	1366	* Write @data into the MSR specified by @index. Select MSR specific fault
		1367	* checks are bypassed if @host_initiated is %true.
1355	* Returns 0 on success, non-0 otherwise.	1368	* Returns 0 on success, non-0 otherwise.
1356	* Assumes vcpu_load() was already called.	1369	* Assumes vcpu_load() was already called.
1357	*/	1370	*/
1358	int kvm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr)	1371	static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
		1372	bool host_initiated)
1359	{	1373	{
1360	switch (msr->index) {	1374	struct msr_data msr;
		1375
		1376	switch (index) {
1361	case MSR_FS_BASE:	1377	case MSR_FS_BASE:
1362	case MSR_GS_BASE:	1378	case MSR_GS_BASE:
1363	case MSR_KERNEL_GS_BASE:	1379	case MSR_KERNEL_GS_BASE:
1364	case MSR_CSTAR:	1380	case MSR_CSTAR:
1365	case MSR_LSTAR:	1381	case MSR_LSTAR:
1366	if (is_noncanonical_address(msr->data, vcpu))	1382	if (is_noncanonical_address(data, vcpu))
1367	return 1;	1383	return 1;
1368	break;	1384	break;
1369	case MSR_IA32_SYSENTER_EIP:	1385	case MSR_IA32_SYSENTER_EIP:
@@ -1380,38 +1396,95 @@ int kvm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) · 10 unmodified lines
1380	* value, and that something deterministic happens if the guest	1396	* value, and that something deterministic happens if the guest
1381	* invokes 64-bit SYSENTER.	1397	* invokes 64-bit SYSENTER.
1382	*/	1398	*/
1383	msr->data = get_canonical(msr->data, vcpu_virt_addr_bits(vcpu));	1399	data = get_canonical(data, vcpu_virt_addr_bits(vcpu));
1384	}	1400	}
1385	return kvm_x86_ops->set_msr(vcpu, msr);	1401
		1402	msr.data = data;
		1403	msr.index = index;
		1404	msr.host_initiated = host_initiated;
		1405
		1406	return kvm_x86_ops->set_msr(vcpu, &msr);
1386	}	1407	}
1387	EXPORT_SYMBOL_GPL(kvm_set_msr);
1388		1408
1389	/*	1409	/*
1390	* Adapt set_msr() to msr_io()'s calling convention	1410	* Read the MSR specified by @index into @data. Select MSR specific fault
		1411	* checks are bypassed if @host_initiated is %true.
		1412	* Returns 0 on success, non-0 otherwise.
		1413	* Assumes vcpu_load() was already called.
1391	*/	1414	*/
1392	static int do_get_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)	1415	static int __kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data,
		1416	bool host_initiated)
1393	{	1417	{
1394	struct msr_data msr;	1418	struct msr_data msr;
1395	int r;	1419	int ret;
1396		1420
1397	msr.index = index;	1421	msr.index = index;
1398	msr.host_initiated = true;	1422	msr.host_initiated = host_initiated;
1399	r = kvm_get_msr(vcpu, &msr);
1400	if (r)
1401	return r;
1402		1423
1403	*data = msr.data;	1424	ret = kvm_x86_ops->get_msr(vcpu, &msr);
1404	return 0;	1425	if (!ret)
		1426	*data = msr.data;
		1427	return ret;
1405	}	1428	}
1406		1429
1407	static int do_set_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)	1430	int kvm_get_msr(struct kvm_vcpu vcpu, u32 index, u64 data)
1408	{	1431	{
1409	struct msr_data msr;	1432	return __kvm_get_msr(vcpu, index, data, false);
		1433	}
		1434	EXPORT_SYMBOL_GPL(kvm_get_msr);
1410		1435
1411	msr.data = *data;	1436	int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1412	msr.index = index;	1437	{
1413	msr.host_initiated = true;	1438	return __kvm_set_msr(vcpu, index, data, false);
1414	return kvm_set_msr(vcpu, &msr);	1439	}
		1440	EXPORT_SYMBOL_GPL(kvm_set_msr);
		1441
		1442	int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
		1443	{
		1444	u32 ecx = kvm_rcx_read(vcpu);
		1445	u64 data;
		1446
		1447	if (kvm_get_msr(vcpu, ecx, &data)) {
		1448	trace_kvm_msr_read_ex(ecx);
		1449	kvm_inject_gp(vcpu, 0);
		1450	return 1;
		1451	}
		1452
		1453	trace_kvm_msr_read(ecx, data);
		1454
		1455	kvm_rax_write(vcpu, data & -1u);
		1456	kvm_rdx_write(vcpu, (data >> 32) & -1u);
		1457	return kvm_skip_emulated_instruction(vcpu);
		1458	}
		1459	EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
		1460
		1461	int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
		1462	{
		1463	u32 ecx = kvm_rcx_read(vcpu);
		1464	u64 data = kvm_read_edx_eax(vcpu);
		1465
		1466	if (kvm_set_msr(vcpu, ecx, data)) {
		1467	trace_kvm_msr_write_ex(ecx, data);
		1468	kvm_inject_gp(vcpu, 0);
		1469	return 1;
		1470	}
		1471
		1472	trace_kvm_msr_write(ecx, data);
		1473	return kvm_skip_emulated_instruction(vcpu);
		1474	}
		1475	EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
		1476
		1477	/*
		1478	* Adapt set_msr() to msr_io()'s calling convention
		1479	*/
		1480	static int do_get_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
		1481	{
		1482	return __kvm_get_msr(vcpu, index, data, true);
		1483	}
		1484
		1485	static int do_set_msr(struct kvm_vcpu vcpu, unsigned index, u64 data)
		1486	{
		1487	return __kvm_set_msr(vcpu, index, *data, true);
1415	}	1488	}
1416		1489
1417	#ifdef CONFIG_X86_64	1490	#ifdef CONFIG_X86_64
@@ -2452,6 +2525,8 @@ static void record_steal_time(struct kvm_vcpu *vcpu) · 1034 unmodified lines
2452	* Doing a TLB flush here, on the guest's behalf, can avoid	2525	* Doing a TLB flush here, on the guest's behalf, can avoid
2453	* expensive IPIs.	2526	* expensive IPIs.
2454	*/	2527	*/
		2528	trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
		2529	vcpu->arch.st.steal.preempted & KVM_VCPU_FLUSH_TLB);
2455	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)	2530	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
2456	kvm_vcpu_flush_tlb(vcpu, false);	2531	kvm_vcpu_flush_tlb(vcpu, false);
2457		2532
@@ -2748,18 +2823,6 @@ int kvm_set_msr_common(struct kvm_vcpu vcpu, struct msr_data msr_info) · 290 unmodified lines
2748	}	2823	}
2749	EXPORT_SYMBOL_GPL(kvm_set_msr_common);	2824	EXPORT_SYMBOL_GPL(kvm_set_msr_common);
2750		2825
2751
2752	/*
2753	* Reads an msr value (of 'msr_index') into 'pdata'.
2754	* Returns 0 on success, non-0 otherwise.
2755	* Assumes vcpu_load() was already called.
2756	*/
2757	int kvm_get_msr(struct kvm_vcpu vcpu, struct msr_data msr)
2758	{
2759	return kvm_x86_ops->get_msr(vcpu, msr);
2760	}
2761	EXPORT_SYMBOL_GPL(kvm_get_msr);
2762
2763	static int get_msr_mce(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)	2826	static int get_msr_mce(struct kvm_vcpu vcpu, u32 msr, u64 pdata, bool host)
2764	{	2827	{
2765	u64 data;	2828	u64 data;
@@ -3506,8 +3569,7 @@ static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, · 740 unmodified lines
3506	for (bank = 0; bank < bank_num; bank++)	3569	for (bank = 0; bank < bank_num; bank++)
3507	vcpu->arch.mce_banks[bank*4] = ~(u64)0;	3570	vcpu->arch.mce_banks[bank*4] = ~(u64)0;
3508		3571
3509	if (kvm_x86_ops->setup_mce)	3572	kvm_x86_ops->setup_mce(vcpu);
3510	kvm_x86_ops->setup_mce(vcpu);
3511	out:	3573	out:
3512	return r;	3574	return r;
3513	}	3575	}
@@ -5377,7 +5439,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, · 1863 unmodified lines
5377	*/	5439	*/
5378	if (vcpu_match_mmio_gva(vcpu, gva)	5440	if (vcpu_match_mmio_gva(vcpu, gva)
5379	&& !permission_fault(vcpu, vcpu->arch.walk_mmu,	5441	&& !permission_fault(vcpu, vcpu->arch.walk_mmu,
5380	vcpu->arch.access, 0, access)) {	5442	vcpu->arch.mmio_access, 0, access)) {
5381	*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT \|	5443	*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT \|
5382	(gva & (PAGE_SIZE - 1));	5444	(gva & (PAGE_SIZE - 1));
5383	trace_vcpu_match_mmio(gva, *gpa, write, false);	5445	trace_vcpu_match_mmio(gva, *gpa, write, false);
@@ -5971,28 +6033,13 @@ static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, · 587 unmodified lines
5971	static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,	6033	static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
5972	u32 msr_index, u64 *pdata)	6034	u32 msr_index, u64 *pdata)
5973	{	6035	{
5974	struct msr_data msr;	6036	return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
5975	int r;
5976
5977	msr.index = msr_index;
5978	msr.host_initiated = false;
5979	r = kvm_get_msr(emul_to_vcpu(ctxt), &msr);
5980	if (r)
5981	return r;
5982
5983	*pdata = msr.data;
5984	return 0;
5985	}	6037	}
5986		6038
5987	static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,	6039	static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
5988	u32 msr_index, u64 data)	6040	u32 msr_index, u64 data)
5989	{	6041	{
5990	struct msr_data msr;	6042	return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
5991
5992	msr.data = data;
5993	msr.index = msr_index;
5994	msr.host_initiated = false;
5995	return kvm_set_msr(emul_to_vcpu(ctxt), &msr);
5996	}	6043	}
5997		6044
5998	static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)	6045	static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
@@ -6075,6 +6122,11 @@ static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt) · 76 unmodified lines
6075	kvm_smm_changed(emul_to_vcpu(ctxt));	6122	kvm_smm_changed(emul_to_vcpu(ctxt));
6076	}	6123	}
6077		6124
		6125	static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
		6126	{
		6127	return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
		6128	}
		6129
6078	static const struct x86_emulate_ops emulate_ops = {	6130	static const struct x86_emulate_ops emulate_ops = {
6079	.read_gpr = emulator_read_gpr,	6131	.read_gpr = emulator_read_gpr,
6080	.write_gpr = emulator_write_gpr,	6132	.write_gpr = emulator_write_gpr,
@@ -6116,6 +6168,7 @@ static const struct x86_emulate_ops emulate_ops = { · 35 unmodified lines
6116	.set_hflags = emulator_set_hflags,	6168	.set_hflags = emulator_set_hflags,
6117	.pre_leave_smm = emulator_pre_leave_smm,	6169	.pre_leave_smm = emulator_pre_leave_smm,
6118	.post_leave_smm = emulator_post_leave_smm,	6170	.post_leave_smm = emulator_post_leave_smm,
		6171	.set_xcr = emulator_set_xcr,
6119	};	6172	};
6120		6173
6121	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)	6174	static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
@@ -6390,9 +6443,11 @@ static void kvm_vcpu_do_singlestep(struct kvm_vcpu vcpu, int r) · 268 unmodified lines
6390	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)	6443	int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
6391	{	6444	{
6392	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);	6445	unsigned long rflags = kvm_x86_ops->get_rflags(vcpu);
6393	int r = EMULATE_DONE;	6446	int r;
6394		6447
6395	kvm_x86_ops->skip_emulated_instruction(vcpu);	6448	r = kvm_x86_ops->skip_emulated_instruction(vcpu);
		6449	if (unlikely(r != EMULATE_DONE))
		6450	return 0;
6396		6451
6397	/*	6452	/*
6398	* rflags is the old, "raw" value of the flags. The new value has	6453	* rflags is the old, "raw" value of the flags. The new value has
@@ -6528,8 +6583,16 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, · 129 unmodified lines
6528	if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,	6583	if (reexecute_instruction(vcpu, cr2, write_fault_to_spt,
6529	emulation_type))	6584	emulation_type))
6530	return EMULATE_DONE;	6585	return EMULATE_DONE;
6531	if (ctxt->have_exception && inject_emulated_exception(vcpu))	6586	if (ctxt->have_exception) {
		6587	/*
		6588	* #UD should result in just EMULATION_FAILED, and trap-like
		6589	* exception should not be encountered during decode.
		6590	*/
		6591	WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR \|\|
		6592	exception_type(ctxt->exception.vector) == EXCPT_TRAP);
		6593	inject_emulated_exception(vcpu);
6532	return EMULATE_DONE;	6594	return EMULATE_DONE;
		6595	}
6533	if (emulation_type & EMULTYPE_SKIP)	6596	if (emulation_type & EMULTYPE_SKIP)
6534	return EMULATE_FAIL;	6597	return EMULATE_FAIL;
6535	return handle_emulation_failure(vcpu, emulation_type);	6598	return handle_emulation_failure(vcpu, emulation_type);
@@ -6544,6 +6607,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, · 8 unmodified lines
6544	kvm_rip_write(vcpu, ctxt->_eip);	6607	kvm_rip_write(vcpu, ctxt->_eip);
6545	if (ctxt->eflags & X86_EFLAGS_RF)	6608	if (ctxt->eflags & X86_EFLAGS_RF)
6546	kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);	6609	kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
		6610	kvm_x86_ops->set_interrupt_shadow(vcpu, 0);
6547	return EMULATE_DONE;	6611	return EMULATE_DONE;
6548	}	6612	}
6549		6613
@@ -9322,10 +9386,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 2772 unmodified lines
9322	kvm_page_track_init(kvm);	9386	kvm_page_track_init(kvm);
9323	kvm_mmu_init_vm(kvm);	9387	kvm_mmu_init_vm(kvm);
9324		9388
9325	if (kvm_x86_ops->vm_init)	9389	return kvm_x86_ops->vm_init(kvm);
9326	return kvm_x86_ops->vm_init(kvm);
9327
9328	return 0;
9329	}	9390	}
9330		9391
9331	static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)	9392	static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
@@ -10017,7 +10078,7 @@ EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma); · 685 unmodified lines
10017		10078
10018	bool kvm_arch_has_irq_bypass(void)	10079	bool kvm_arch_has_irq_bypass(void)
10019	{	10080	{
10020	return kvm_x86_ops->update_pi_irte != NULL;	10081	return true;
10021	}	10082	}
10022		10083
10023	int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,	10084	int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
@@ -10057,9 +10118,6 @@ void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, · 33 unmodified lines
10057	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,	10118	int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
10058	uint32_t guest_irq, bool set)	10119	uint32_t guest_irq, bool set)
10059	{	10120	{
10060	if (!kvm_x86_ops->update_pi_irte)
10061	return -EINVAL;
10062
10063	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);	10121	return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set);
10064	}	10122	}
10065		10123
@@ -10086,11 +10144,12 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); · 20 unmodified lines
10086	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);	10144	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
10087	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);	10145	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
10088	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);	10146	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
		10147	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
10089	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);	10148	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
10090	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);	10149	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
10091	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);	10150	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
10092	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);	10151	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
10093	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window);	10152	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
10094	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);	10153	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
10095	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);	10154	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
10096	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);	10155	EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);

arch/x86/kvm/x86.h+1−1

@@ -196,7 +196,7 @@ static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, · 195 unmodified lines
196	* actually a nGPA.	196	* actually a nGPA.
197	*/	197	*/
198	vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;	198	vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;
199	vcpu->arch.access = access;	199	vcpu->arch.mmio_access = access;
200	vcpu->arch.mmio_gfn = gfn;	200	vcpu->arch.mmio_gfn = gfn;
201	vcpu->arch.mmio_gen = gen;	201	vcpu->arch.mmio_gen = gen;
202	}	202	}

include/kvm/arm_vgic.h+3−1

 	struct list_head	lpi_list_head;
 	int			lpi_list_count;
+	/* LPI translation cache */
+	struct list_head	lpi_translation_cache;
 	/* used by vgic-debug */
 	struct vgic_state_iter *iter;
 	 * parts of the redistributor.
 	 */
 	struct vgic_io_device	rd_iodev;
-	struct vgic_io_device	sgi_iodev;
 	struct vgic_redist_region *rdreg;
 	/* Contains the attributes and gpa of the LPI pending tables. */

include/uapi/linux/kvm.h+3−0

 #define KVM_INTERNAL_ERROR_SIMUL_EX	2
 /* Encounter unexpected vm-exit due to delivery event. */
 #define KVM_INTERNAL_ERROR_DELIVERY_EV	3
+/* Encounter unexpected vm-exit reason */
+#define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON	4
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
 #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
 #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
 #define KVM_CAP_PMU_EVENT_FILTER 173
+#define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
 #ifdef KVM_CAP_IRQ_ROUTING

tools/testing/selftests/kvm/Makefile+6−4

 KSFT_KHDR_INSTALL := 1
 UNAME_M := $(shell uname -m)
-LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/ucall.c lib/sparsebit.c
+LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/sparsebit.c
-LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c
+LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/ucall.c
-LIBKVM_aarch64 = lib/aarch64/processor.c
+LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c
-LIBKVM_s390x = lib/s390x/processor.c
+LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c
 TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test
 TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test
 TEST_GEN_PROGS_aarch64 += dirty_log_test
 TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus
+TEST_GEN_PROGS_s390x = s390x/memop
 TEST_GEN_PROGS_s390x += s390x/sync_regs_test
+TEST_GEN_PROGS_s390x += dirty_log_test
 TEST_GEN_PROGS_s390x += kvm_create_max_vcpus
 TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M))

tools/testing/selftests/kvm/dirty_log_test.c+53−8

 /* The memory slot index to track dirty pages */
 #define TEST_MEM_SLOT_INDEX		1
-/* Default guest test memory offset, 1G */
+/* Default guest test virtual memory offset */
-#define DEFAULT_GUEST_TEST_MEM		0x40000000
+#define DEFAULT_GUEST_TEST_MEM		0xc0000000
 /* How many pages to dirty for each guest loop */
 #define TEST_PAGES_PER_LOOP		1024
 /* Interval for each host loop (ms) */
 #define TEST_HOST_LOOP_INTERVAL		10UL
+/* Dirty bitmaps are always little endian, so we need to swap on big endian */
+#if defined(__s390x__)
+# define BITOP_LE_SWIZZLE	((BITS_PER_LONG-1) & ~0x7)
+# define test_bit_le(nr, addr) \
+	test_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
+# define set_bit_le(nr, addr) \
+	set_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
+# define clear_bit_le(nr, addr) \
+	clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
+# define test_and_set_bit_le(nr, addr) \
+	test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
+# define test_and_clear_bit_le(nr, addr) \
+	test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr)
+#else
+# define test_bit_le		test_bit
+# define set_bit_le		set_bit
+# define clear_bit_le		clear_bit
+# define test_and_set_bit_le	test_and_set_bit
+# define test_and_clear_bit_le	test_and_clear_bit
+#endif
 /*
  * Guest/Host shared variables. Ensure addr_gva2hva() and/or
  * sync_global_to/from_guest() are used when accessing from
  */
 static void guest_code(void)
 {
+	uint64_t addr;
 	int i;
+	/*
+	 * On s390x, all pages of a 1M segment are initially marked as dirty
+	 * when a page of the segment is written to for the very first time.
+	 * To compensate this specialty in this test, we need to touch all
+	 * pages during the first iteration.
+	 */
+	for (i = 0; i < guest_num_pages; i++) {
+		addr = guest_test_virt_mem + i * guest_page_size;
+		*(uint64_t *)addr = READ_ONCE(iteration);
+	}
 	while (true) {
 		for (i = 0; i < TEST_PAGES_PER_LOOP; i++) {
-			uint64_t addr = guest_test_virt_mem;
+			addr = guest_test_virt_mem;
 			addr += (READ_ONCE(random_array[i]) % guest_num_pages)
 				* guest_page_size;
 			addr &= ~(host_page_size - 1);
 		value_ptr = host_test_mem + page * host_page_size;
 		/* If this is a special page that we were tracking... */
-		if (test_and_clear_bit(page, host_bmap_track)) {
+		if (test_and_clear_bit_le(page, host_bmap_track)) {
 			host_track_next_count++;
-			TEST_ASSERT(test_bit(page, bmap),
+			TEST_ASSERT(test_bit_le(page, bmap),
 				    "Page %"PRIu64" should have its dirty bit "
 				    "set in this iteration but it is missing",
 				    page);
 		}
-		if (test_bit(page, bmap)) {
+		if (test_bit_le(page, bmap)) {
 			host_dirty_count++;
 			/*
 			 * If the bit is set, the value written onto
 				 * should report its dirtyness in the
 				 * next run
 				 */
-				set_bit(page, host_bmap_track);
+				set_bit_le(page, host_bmap_track);
 			}
 		}
 	}
 	 * case where the size is not aligned to 64 pages.
 	 */
 	guest_num_pages = (1ul << (30 - guest_page_shift)) + 16;
+#ifdef __s390x__
+	/* Round up to multiple of 1M (segment size) */
+	guest_num_pages = (guest_num_pages + 0xff) & ~0xffUL;
+#endif
 	host_page_size = getpagesize();
 	host_num_pages = (guest_num_pages * guest_page_size) / host_page_size +
 			 !!((guest_num_pages * guest_page_size) % host_page_size);
 		guest_test_phys_mem = phys_offset;
 	}
+#ifdef __s390x__
+	/* Align to 1M (segment size) */
+	guest_test_phys_mem &= ~((1 << 20) - 1);
+#endif
 	DEBUG("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem);
 	bmap = bitmap_alloc(host_num_pages);
 	vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
 #endif
 #ifdef __aarch64__
-	ucall_init(vm, UCALL_MMIO, NULL);
+	ucall_init(vm, NULL);
 #endif
 	/* Export the shared variables to the guest */
 		vm_guest_mode_params_init(VM_MODE_P48V48_64K, true, true);
 	}
 #endif
+#ifdef __s390x__
+	vm_guest_mode_params_init(VM_MODE_P40V48_4K, true, true);
+#endif
 	while ((opt = getopt(argc, argv, "hi:I:p:m:")) != -1) {
 		switch (opt) {

tools/testing/selftests/kvm/include/kvm_util.h+1−7

 	memcpy(&(g), _p, sizeof(g));				\
 })
-/* ucall implementation types */
-typedef enum {
-	UCALL_PIO,
-	UCALL_MMIO,
-} ucall_type_t;
 /* Common ucalls */
 enum {
 	UCALL_NONE,
 	uint64_t args[UCALL_MAX_ARGS];
 };
-void ucall_init(struct kvm_vm *vm, ucall_type_t type, void *arg);
+void ucall_init(struct kvm_vm *vm, void *arg);
 void ucall_uninit(struct kvm_vm *vm);
 void ucall(uint64_t cmd, int nargs, ...);
 uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc);

tools/testing/selftests/kvm/lib/aarch64/ucall.c+112−0

@@ -0,0 +1,112 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* ucall support. A ucall is a "hypercall to userspace".
	4	*
	5	* Copyright (C) 2018, Red Hat, Inc.
	6	*/
	7	#include "kvm_util.h"
	8	#include "../kvm_util_internal.h"
	9
	10	static vm_vaddr_t *ucall_exit_mmio_addr;
	11
	12	static bool ucall_mmio_init(struct kvm_vm *vm, vm_paddr_t gpa)
	13	{
	14	if (kvm_userspace_memory_region_find(vm, gpa, gpa + 1))
	15	return false;
	16
	17	virt_pg_map(vm, gpa, gpa, 0);
	18
	19	ucall_exit_mmio_addr = (vm_vaddr_t *)gpa;
	20	sync_global_to_guest(vm, ucall_exit_mmio_addr);
	21
	22	return true;
	23	}
	24
	25	void ucall_init(struct kvm_vm vm, void arg)
	26	{
	27	vm_paddr_t gpa, start, end, step, offset;
	28	unsigned int bits;
	29	bool ret;
	30
	31	if (arg) {
	32	gpa = (vm_paddr_t)arg;
	33	ret = ucall_mmio_init(vm, gpa);
	34	TEST_ASSERT(ret, "Can't set ucall mmio address to %lx", gpa);
	35	return;
	36	}
	37
	38	/*
	39	* Find an address within the allowed physical and virtual address
	40	* spaces, that does _not_ have a KVM memory region associated with
	41	* it. Identity mapping an address like this allows the guest to
	42	* access it, but as KVM doesn't know what to do with it, it
	43	* will assume it's something userspace handles and exit with
	44	* KVM_EXIT_MMIO. Well, at least that's how it works for AArch64.
	45	* Here we start with a guess that the addresses around 5/8th
	46	* of the allowed space are unmapped and then work both down and
	47	* up from there in 1/16th allowed space sized steps.
	48	*
	49	* Note, we need to use VA-bits - 1 when calculating the allowed
	50	* virtual address space for an identity mapping because the upper
	51	* half of the virtual address space is the two's complement of the
	52	* lower and won't match physical addresses.
	53	*/
	54	bits = vm->va_bits - 1;
	55	bits = vm->pa_bits < bits ? vm->pa_bits : bits;
	56	end = 1ul << bits;
	57	start = end * 5 / 8;
	58	step = end / 16;
	59	for (offset = 0; offset < end - start; offset += step) {
	60	if (ucall_mmio_init(vm, start - offset))
	61	return;
	62	if (ucall_mmio_init(vm, start + offset))
	63	return;
	64	}
	65	TEST_ASSERT(false, "Can't find a ucall mmio address");
	66	}
	67
	68	void ucall_uninit(struct kvm_vm *vm)
	69	{
	70	ucall_exit_mmio_addr = 0;
	71	sync_global_to_guest(vm, ucall_exit_mmio_addr);
	72	}
	73
	74	void ucall(uint64_t cmd, int nargs, ...)
	75	{
	76	struct ucall uc = {
	77	.cmd = cmd,
	78	};
	79	va_list va;
	80	int i;
	81
	82	nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	83
	84	va_start(va, nargs);
	85	for (i = 0; i < nargs; ++i)
	86	uc.args[i] = va_arg(va, uint64_t);
	87	va_end(va);
	88
	89	*ucall_exit_mmio_addr = (vm_vaddr_t)&uc;
	90	}
	91
	92	uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	93	{
	94	struct kvm_run *run = vcpu_state(vm, vcpu_id);
	95	struct ucall ucall = {};
	96
	97	if (run->exit_reason == KVM_EXIT_MMIO &&
	98	run->mmio.phys_addr == (uint64_t)ucall_exit_mmio_addr) {
	99	vm_vaddr_t gva;
	100
	101	TEST_ASSERT(run->mmio.is_write && run->mmio.len == 8,
	102	"Unexpected ucall exit mmio address access");
	103	memcpy(&gva, run->mmio.data, sizeof(gva));
	104	memcpy(&ucall, addr_gva2hva(vm, gva), sizeof(ucall));
	105
	106	vcpu_run_complete_io(vm, vcpu_id);
	107	if (uc)
	108	memcpy(uc, &ucall, sizeof(ucall));
	109	}
	110
	111	return ucall.cmd;
	112	}

tools/testing/selftests/kvm/lib/s390x/ucall.c+56−0

@@ -0,0 +1,56 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* ucall support. A ucall is a "hypercall to userspace".
	4	*
	5	* Copyright (C) 2019 Red Hat, Inc.
	6	*/
	7	#include "kvm_util.h"
	8
	9	void ucall_init(struct kvm_vm vm, void arg)
	10	{
	11	}
	12
	13	void ucall_uninit(struct kvm_vm *vm)
	14	{
	15	}
	16
	17	void ucall(uint64_t cmd, int nargs, ...)
	18	{
	19	struct ucall uc = {
	20	.cmd = cmd,
	21	};
	22	va_list va;
	23	int i;
	24
	25	nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	26
	27	va_start(va, nargs);
	28	for (i = 0; i < nargs; ++i)
	29	uc.args[i] = va_arg(va, uint64_t);
	30	va_end(va);
	31
	32	/* Exit via DIAGNOSE 0x501 (normally used for breakpoints) */
	33	asm volatile ("diag 0,%0,0x501" : : "a"(&uc) : "memory");
	34	}
	35
	36	uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	37	{
	38	struct kvm_run *run = vcpu_state(vm, vcpu_id);
	39	struct ucall ucall = {};
	40
	41	if (run->exit_reason == KVM_EXIT_S390_SIEIC &&
	42	run->s390_sieic.icptcode == 4 &&
	43	(run->s390_sieic.ipa >> 8) == 0x83 && /* 0x83 means DIAGNOSE */
	44	(run->s390_sieic.ipb >> 16) == 0x501) {
	45	int reg = run->s390_sieic.ipa & 0xf;
	46
	47	memcpy(&ucall, addr_gva2hva(vm, run->s.regs.gprs[reg]),
	48	sizeof(ucall));
	49
	50	vcpu_run_complete_io(vm, vcpu_id);
	51	if (uc)
	52	memcpy(uc, &ucall, sizeof(ucall));
	53	}
	54
	55	return ucall.cmd;
	56	}

tools/testing/selftests/kvm/lib/ucall.c+0−157

@@ -1,157 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* ucall support. A ucall is a "hypercall to userspace".
4	*
5	* Copyright (C) 2018, Red Hat, Inc.
6	*/
7	#include "kvm_util.h"
8	#include "kvm_util_internal.h"
9
10	#define UCALL_PIO_PORT ((uint16_t)0x1000)
11
12	static ucall_type_t ucall_type;
13	static vm_vaddr_t *ucall_exit_mmio_addr;
14
15	static bool ucall_mmio_init(struct kvm_vm *vm, vm_paddr_t gpa)
16	{
17	if (kvm_userspace_memory_region_find(vm, gpa, gpa + 1))
18	return false;
19
20	virt_pg_map(vm, gpa, gpa, 0);
21
22	ucall_exit_mmio_addr = (vm_vaddr_t *)gpa;
23	sync_global_to_guest(vm, ucall_exit_mmio_addr);
24
25	return true;
26	}
27
28	void ucall_init(struct kvm_vm vm, ucall_type_t type, void arg)
29	{
30	ucall_type = type;
31	sync_global_to_guest(vm, ucall_type);
32
33	if (type == UCALL_PIO)
34	return;
35
36	if (type == UCALL_MMIO) {
37	vm_paddr_t gpa, start, end, step, offset;
38	unsigned bits;
39	bool ret;
40
41	if (arg) {
42	gpa = (vm_paddr_t)arg;
43	ret = ucall_mmio_init(vm, gpa);
44	TEST_ASSERT(ret, "Can't set ucall mmio address to %lx", gpa);
45	return;
46	}
47
48	/*
49	* Find an address within the allowed physical and virtual address
50	* spaces, that does _not_ have a KVM memory region associated with
51	* it. Identity mapping an address like this allows the guest to
52	* access it, but as KVM doesn't know what to do with it, it
53	* will assume it's something userspace handles and exit with
54	* KVM_EXIT_MMIO. Well, at least that's how it works for AArch64.
55	* Here we start with a guess that the addresses around 5/8th
56	* of the allowed space are unmapped and then work both down and
57	* up from there in 1/16th allowed space sized steps.
58	*
59	* Note, we need to use VA-bits - 1 when calculating the allowed
60	* virtual address space for an identity mapping because the upper
61	* half of the virtual address space is the two's complement of the
62	* lower and won't match physical addresses.
63	*/
64	bits = vm->va_bits - 1;
65	bits = vm->pa_bits < bits ? vm->pa_bits : bits;
66	end = 1ul << bits;
67	start = end * 5 / 8;
68	step = end / 16;
69	for (offset = 0; offset < end - start; offset += step) {
70	if (ucall_mmio_init(vm, start - offset))
71	return;
72	if (ucall_mmio_init(vm, start + offset))
73	return;
74	}
75	TEST_ASSERT(false, "Can't find a ucall mmio address");
76	}
77	}
78
79	void ucall_uninit(struct kvm_vm *vm)
80	{
81	ucall_type = 0;
82	sync_global_to_guest(vm, ucall_type);
83	ucall_exit_mmio_addr = 0;
84	sync_global_to_guest(vm, ucall_exit_mmio_addr);
85	}
86
87	static void ucall_pio_exit(struct ucall *uc)
88	{
89	#ifdef __x86_64__
90	asm volatile("in %[port], %%al"
91	: : [port] "d" (UCALL_PIO_PORT), "D" (uc) : "rax");
92	#endif
93	}
94
95	static void ucall_mmio_exit(struct ucall *uc)
96	{
97	*ucall_exit_mmio_addr = (vm_vaddr_t)uc;
98	}
99
100	void ucall(uint64_t cmd, int nargs, ...)
101	{
102	struct ucall uc = {
103	.cmd = cmd,
104	};
105	va_list va;
106	int i;
107
108	nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
109
110	va_start(va, nargs);
111	for (i = 0; i < nargs; ++i)
112	uc.args[i] = va_arg(va, uint64_t);
113	va_end(va);
114
115	switch (ucall_type) {
116	case UCALL_PIO:
117	ucall_pio_exit(&uc);
118	break;
119	case UCALL_MMIO:
120	ucall_mmio_exit(&uc);
121	break;
122	};
123	}
124
125	uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
126	{
127	struct kvm_run *run = vcpu_state(vm, vcpu_id);
128	struct ucall ucall = {};
129	bool got_ucall = false;
130
131	#ifdef __x86_64__
132	if (ucall_type == UCALL_PIO && run->exit_reason == KVM_EXIT_IO &&
133	run->io.port == UCALL_PIO_PORT) {
134	struct kvm_regs regs;
135	vcpu_regs_get(vm, vcpu_id, &regs);
136	memcpy(&ucall, addr_gva2hva(vm, (vm_vaddr_t)regs.rdi), sizeof(ucall));
137	got_ucall = true;
138	}
139	#endif
140	if (ucall_type == UCALL_MMIO && run->exit_reason == KVM_EXIT_MMIO &&
141	run->mmio.phys_addr == (uint64_t)ucall_exit_mmio_addr) {
142	vm_vaddr_t gva;
143	TEST_ASSERT(run->mmio.is_write && run->mmio.len == 8,
144	"Unexpected ucall exit mmio address access");
145	memcpy(&gva, run->mmio.data, sizeof(gva));
146	memcpy(&ucall, addr_gva2hva(vm, gva), sizeof(ucall));
147	got_ucall = true;
148	}
149
150	if (got_ucall) {
151	vcpu_run_complete_io(vm, vcpu_id);
152	if (uc)
153	memcpy(uc, &ucall, sizeof(ucall));
154	}
155
156	return ucall.cmd;
157	}

tools/testing/selftests/kvm/lib/x86_64/ucall.c+56−0

@@ -0,0 +1,56 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* ucall support. A ucall is a "hypercall to userspace".
	4	*
	5	* Copyright (C) 2018, Red Hat, Inc.
	6	*/
	7	#include "kvm_util.h"
	8
	9	#define UCALL_PIO_PORT ((uint16_t)0x1000)
	10
	11	void ucall_init(struct kvm_vm vm, void arg)
	12	{
	13	}
	14
	15	void ucall_uninit(struct kvm_vm *vm)
	16	{
	17	}
	18
	19	void ucall(uint64_t cmd, int nargs, ...)
	20	{
	21	struct ucall uc = {
	22	.cmd = cmd,
	23	};
	24	va_list va;
	25	int i;
	26
	27	nargs = nargs <= UCALL_MAX_ARGS ? nargs : UCALL_MAX_ARGS;
	28
	29	va_start(va, nargs);
	30	for (i = 0; i < nargs; ++i)
	31	uc.args[i] = va_arg(va, uint64_t);
	32	va_end(va);
	33
	34	asm volatile("in %[port], %%al"
	35	: : [port] "d" (UCALL_PIO_PORT), "D" (&uc) : "rax");
	36	}
	37
	38	uint64_t get_ucall(struct kvm_vm vm, uint32_t vcpu_id, struct ucall uc)
	39	{
	40	struct kvm_run *run = vcpu_state(vm, vcpu_id);
	41	struct ucall ucall = {};
	42
	43	if (run->exit_reason == KVM_EXIT_IO && run->io.port == UCALL_PIO_PORT) {
	44	struct kvm_regs regs;
	45
	46	vcpu_regs_get(vm, vcpu_id, &regs);
	47	memcpy(&ucall, addr_gva2hva(vm, (vm_vaddr_t)regs.rdi),
	48	sizeof(ucall));
	49
	50	vcpu_run_complete_io(vm, vcpu_id);
	51	if (uc)
	52	memcpy(uc, &ucall, sizeof(ucall));
	53	}
	54
	55	return ucall.cmd;
	56	}

tools/testing/selftests/kvm/s390x/memop.c+166−0

@@ -0,0 +1,166 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/*
	3	* Test for s390x KVM_S390_MEM_OP
	4	*
	5	* Copyright (C) 2019, Red Hat, Inc.
	6	*/
	7
	8	#include <stdio.h>
	9	#include <stdlib.h>
	10	#include <string.h>
	11	#include <sys/ioctl.h>
	12
	13	#include "test_util.h"
	14	#include "kvm_util.h"
	15
	16	#define VCPU_ID 1
	17
	18	static uint8_t mem1[65536];
	19	static uint8_t mem2[65536];
	20
	21	static void guest_code(void)
	22	{
	23	int i;
	24
	25	for (;;) {
	26	for (i = 0; i < sizeof(mem2); i++)
	27	mem2[i] = mem1[i];
	28	GUEST_SYNC(0);
	29	}
	30	}
	31
	32	int main(int argc, char *argv[])
	33	{
	34	struct kvm_vm *vm;
	35	struct kvm_run *run;
	36	struct kvm_s390_mem_op ksmo;
	37	int rv, i, maxsize;
	38
	39	setbuf(stdout, NULL); /* Tell stdout not to buffer its content */
	40
	41	maxsize = kvm_check_cap(KVM_CAP_S390_MEM_OP);
	42	if (!maxsize) {
	43	fprintf(stderr, "CAP_S390_MEM_OP not supported -> skip test\n");
	44	exit(KSFT_SKIP);
	45	}
	46	if (maxsize > sizeof(mem1))
	47	maxsize = sizeof(mem1);
	48
	49	/* Create VM */
	50	vm = vm_create_default(VCPU_ID, 0, guest_code);
	51	run = vcpu_state(vm, VCPU_ID);
	52
	53	for (i = 0; i < sizeof(mem1); i++)
	54	mem1[i] = i * i + i;
	55
	56	/* Set the first array */
	57	ksmo.gaddr = addr_gva2gpa(vm, (uintptr_t)mem1);
	58	ksmo.flags = 0;
	59	ksmo.size = maxsize;
	60	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	61	ksmo.buf = (uintptr_t)mem1;
	62	ksmo.ar = 0;
	63	vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	64
	65	/* Let the guest code copy the first array to the second */
	66	vcpu_run(vm, VCPU_ID);
	67	TEST_ASSERT(run->exit_reason == KVM_EXIT_S390_SIEIC,
	68	"Unexpected exit reason: %u (%s)\n",
	69	run->exit_reason,
	70	exit_reason_str(run->exit_reason));
	71
	72	memset(mem2, 0xaa, sizeof(mem2));
	73
	74	/* Get the second array */
	75	ksmo.gaddr = (uintptr_t)mem2;
	76	ksmo.flags = 0;
	77	ksmo.size = maxsize;
	78	ksmo.op = KVM_S390_MEMOP_LOGICAL_READ;
	79	ksmo.buf = (uintptr_t)mem2;
	80	ksmo.ar = 0;
	81	vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	82
	83	TEST_ASSERT(!memcmp(mem1, mem2, maxsize),
	84	"Memory contents do not match!");
	85
	86	/* Check error conditions - first bad size: */
	87	ksmo.gaddr = (uintptr_t)mem1;
	88	ksmo.flags = 0;
	89	ksmo.size = -1;
	90	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	91	ksmo.buf = (uintptr_t)mem1;
	92	ksmo.ar = 0;
	93	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	94	TEST_ASSERT(rv == -1 && errno == E2BIG, "ioctl allows insane sizes");
	95
	96	/* Zero size: */
	97	ksmo.gaddr = (uintptr_t)mem1;
	98	ksmo.flags = 0;
	99	ksmo.size = 0;
	100	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	101	ksmo.buf = (uintptr_t)mem1;
	102	ksmo.ar = 0;
	103	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	104	TEST_ASSERT(rv == -1 && (errno == EINVAL \|\| errno == ENOMEM),
	105	"ioctl allows 0 as size");
	106
	107	/* Bad flags: */
	108	ksmo.gaddr = (uintptr_t)mem1;
	109	ksmo.flags = -1;
	110	ksmo.size = maxsize;
	111	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	112	ksmo.buf = (uintptr_t)mem1;
	113	ksmo.ar = 0;
	114	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	115	TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows all flags");
	116
	117	/* Bad operation: */
	118	ksmo.gaddr = (uintptr_t)mem1;
	119	ksmo.flags = 0;
	120	ksmo.size = maxsize;
	121	ksmo.op = -1;
	122	ksmo.buf = (uintptr_t)mem1;
	123	ksmo.ar = 0;
	124	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	125	TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows bad operations");
	126
	127	/* Bad guest address: */
	128	ksmo.gaddr = ~0xfffUL;
	129	ksmo.flags = KVM_S390_MEMOP_F_CHECK_ONLY;
	130	ksmo.size = maxsize;
	131	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	132	ksmo.buf = (uintptr_t)mem1;
	133	ksmo.ar = 0;
	134	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	135	TEST_ASSERT(rv > 0, "ioctl does not report bad guest memory access");
	136
	137	/* Bad host address: */
	138	ksmo.gaddr = (uintptr_t)mem1;
	139	ksmo.flags = 0;
	140	ksmo.size = maxsize;
	141	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	142	ksmo.buf = 0;
	143	ksmo.ar = 0;
	144	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	145	TEST_ASSERT(rv == -1 && errno == EFAULT,
	146	"ioctl does not report bad host memory address");
	147
	148	/* Bad access register: */
	149	run->psw_mask &= ~(3UL << (63 - 17));
	150	run->psw_mask \|= 1UL << (63 - 17); /* Enable AR mode */
	151	vcpu_run(vm, VCPU_ID); /* To sync new state to SIE block */
	152	ksmo.gaddr = (uintptr_t)mem1;
	153	ksmo.flags = 0;
	154	ksmo.size = maxsize;
	155	ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;
	156	ksmo.buf = (uintptr_t)mem1;
	157	ksmo.ar = 17;
	158	rv = _vcpu_ioctl(vm, VCPU_ID, KVM_S390_MEM_OP, &ksmo);
	159	TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows ARs > 15");
	160	run->psw_mask &= ~(3UL << (63 - 17)); /* Disable AR mode */
	161	vcpu_run(vm, VCPU_ID); /* Run to sync new state */
	162
	163	kvm_vm_free(vm);
	164
	165	return 0;
	166	}

tools/testing/selftests/kvm/s390x/sync_regs_test.c+34−2

@@ -25,9 +25,11 @@ · 24 unmodified lines
25		25
26	static void guest_code(void)	26	static void guest_code(void)
27	{	27	{
		28	register u64 stage asm("11") = 0;
		29
28	for (;;) {	30	for (;;) {
29	asm volatile ("diag 0,0,0x501");	31	GUEST_SYNC(0);
30	asm volatile ("ahi 11,1");	32	asm volatile ("ahi %0,1" : : "r"(stage));
31	}	33	}
32	}	34	}
33		35
@@ -83,6 +85,36 @@ int main(int argc, char *argv[]) · 49 unmodified lines
83		85
84	run = vcpu_state(vm, VCPU_ID);	86	run = vcpu_state(vm, VCPU_ID);
85		87
		88	/* Request reading invalid register set from VCPU. */
		89	run->kvm_valid_regs = INVALID_SYNC_FIELD;
		90	rv = _vcpu_run(vm, VCPU_ID);
		91	TEST_ASSERT(rv < 0 && errno == EINVAL,
		92	"Invalid kvm_valid_regs did not cause expected KVM_RUN error: %d\n",
		93	rv);
		94	vcpu_state(vm, VCPU_ID)->kvm_valid_regs = 0;
		95
		96	run->kvm_valid_regs = INVALID_SYNC_FIELD \| TEST_SYNC_FIELDS;
		97	rv = _vcpu_run(vm, VCPU_ID);
		98	TEST_ASSERT(rv < 0 && errno == EINVAL,
		99	"Invalid kvm_valid_regs did not cause expected KVM_RUN error: %d\n",
		100	rv);
		101	vcpu_state(vm, VCPU_ID)->kvm_valid_regs = 0;
		102
		103	/* Request setting invalid register set into VCPU. */
		104	run->kvm_dirty_regs = INVALID_SYNC_FIELD;
		105	rv = _vcpu_run(vm, VCPU_ID);
		106	TEST_ASSERT(rv < 0 && errno == EINVAL,
		107	"Invalid kvm_dirty_regs did not cause expected KVM_RUN error: %d\n",
		108	rv);
		109	vcpu_state(vm, VCPU_ID)->kvm_dirty_regs = 0;
		110
		111	run->kvm_dirty_regs = INVALID_SYNC_FIELD \| TEST_SYNC_FIELDS;
		112	rv = _vcpu_run(vm, VCPU_ID);
		113	TEST_ASSERT(rv < 0 && errno == EINVAL,
		114	"Invalid kvm_dirty_regs did not cause expected KVM_RUN error: %d\n",
		115	rv);
		116	vcpu_state(vm, VCPU_ID)->kvm_dirty_regs = 0;
		117
86	/* Request and verify all valid register sets. */	118	/* Request and verify all valid register sets. */
87	run->kvm_valid_regs = TEST_SYNC_FIELDS;	119	run->kvm_valid_regs = TEST_SYNC_FIELDS;
88	rv = _vcpu_run(vm, VCPU_ID);	120	rv = _vcpu_run(vm, VCPU_ID);

virt/kvm/arm/arm.c+2−0

 	case KVM_CAP_MP_STATE:
 	case KVM_CAP_IMMEDIATE_EXIT:
 	case KVM_CAP_VCPU_EVENTS:
+	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
 		r = 1;
 		break;
 	case KVM_CAP_ARM_SET_DEVICE_ADDR:
 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
 	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
+	vcpu_idx += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
 	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
 	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);

virt/kvm/arm/vgic/vgic-init.c+6−2

 	struct vgic_dist *dist = &kvm->arch.vgic;
 	INIT_LIST_HEAD(&dist->lpi_list_head);
+	INIT_LIST_HEAD(&dist->lpi_translation_cache);
 	raw_spin_lock_init(&dist->lpi_list_lock);
 }
 	int i;
 	vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
-	vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
 	INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
 	raw_spin_lock_init(&vgic_cpu->ap_list_lock);
 	}
 	if (vgic_has_its(kvm)) {
+		vgic_lpi_translation_cache_init(kvm);
 		ret = vgic_v4_init(kvm);
 		if (ret)
 			goto out;
 		INIT_LIST_HEAD(&dist->rd_regions);
 	}
+	if (vgic_has_its(kvm))
+		vgic_lpi_translation_cache_destroy(kvm);
 	if (vgic_supports_direct_msis(kvm))
 		vgic_v4_teardown(kvm);
 }
 		break;
 	default:
 		ret = -ENODEV;
-	};
+	}
 	if (ret)
 		return ret;

virt/kvm/arm/vgic/vgic-irqfd.c+30−6

@@ -66,6 +66,15 @@ out: · 65 unmodified lines
66	return r;	66	return r;
67	}	67	}
68		68
		69	static void kvm_populate_msi(struct kvm_kernel_irq_routing_entry *e,
		70	struct kvm_msi *msi)
		71	{
		72	msi->address_lo = e->msi.address_lo;
		73	msi->address_hi = e->msi.address_hi;
		74	msi->data = e->msi.data;
		75	msi->flags = e->msi.flags;
		76	msi->devid = e->msi.devid;
		77	}
69	/**	78	/**
70	* kvm_set_msi: inject the MSI corresponding to the	79	* kvm_set_msi: inject the MSI corresponding to the
71	* MSI routing entry	80	* MSI routing entry
@@ -79,21 +88,36 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, · 7 unmodified lines
79	{	88	{
80	struct kvm_msi msi;	89	struct kvm_msi msi;
81		90
82	msi.address_lo = e->msi.address_lo;
83	msi.address_hi = e->msi.address_hi;
84	msi.data = e->msi.data;
85	msi.flags = e->msi.flags;
86	msi.devid = e->msi.devid;
87
88	if (!vgic_has_its(kvm))	91	if (!vgic_has_its(kvm))
89	return -ENODEV;	92	return -ENODEV;
90		93
91	if (!level)	94	if (!level)
92	return -1;	95	return -1;
93		96
		97	kvm_populate_msi(e, &msi);
94	return vgic_its_inject_msi(kvm, &msi);	98	return vgic_its_inject_msi(kvm, &msi);
95	}	99	}
96		100
		101	/**
		102	* kvm_arch_set_irq_inatomic: fast-path for irqfd injection
		103	*
		104	* Currently only direct MSI injection is supported.
		105	*/
		106	int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
		107	struct kvm *kvm, int irq_source_id, int level,
		108	bool line_status)
		109	{
		110	if (e->type == KVM_IRQ_ROUTING_MSI && vgic_has_its(kvm) && level) {
		111	struct kvm_msi msi;
		112
		113	kvm_populate_msi(e, &msi);
		114	if (!vgic_its_inject_cached_translation(kvm, &msi))
		115	return 0;
		116	}
		117
		118	return -EWOULDBLOCK;
		119	}
		120
97	int kvm_vgic_setup_default_irq_routing(struct kvm *kvm)	121	int kvm_vgic_setup_default_irq_routing(struct kvm *kvm)
98	{	122	{
99	struct kvm_irq_routing_entry *entries;	123	struct kvm_irq_routing_entry *entries;

virt/kvm/arm/vgic/vgic-its.c+207−0

@@ -138,6 +138,14 @@ struct its_ite { · 137 unmodified lines
138	u32 event_id;	138	u32 event_id;
139	};	139	};
140		140
		141	struct vgic_translation_cache_entry {
		142	struct list_head entry;
		143	phys_addr_t db;
		144	u32 devid;
		145	u32 eventid;
		146	struct vgic_irq *irq;
		147	};
		148
141	/**	149	/**
142	* struct vgic_its_abi - ITS abi ops and settings	150	* struct vgic_its_abi - ITS abi ops and settings
143	* @cte_esz: collection table entry size	151	* @cte_esz: collection table entry size
@@ -527,6 +535,127 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm, · 383 unmodified lines
527	return 0;	535	return 0;
528	}	536	}
529		537
		538	static struct vgic_irq __vgic_its_check_cache(struct vgic_dist dist,
		539	phys_addr_t db,
		540	u32 devid, u32 eventid)
		541	{
		542	struct vgic_translation_cache_entry *cte;
		543
		544	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
		545	/*
		546	* If we hit a NULL entry, there is nothing after this
		547	* point.
		548	*/
		549	if (!cte->irq)
		550	break;
		551
		552	if (cte->db != db \|\| cte->devid != devid \|\|
		553	cte->eventid != eventid)
		554	continue;
		555
		556	/*
		557	* Move this entry to the head, as it is the most
		558	* recently used.
		559	*/
		560	if (!list_is_first(&cte->entry, &dist->lpi_translation_cache))
		561	list_move(&cte->entry, &dist->lpi_translation_cache);
		562
		563	return cte->irq;
		564	}
		565
		566	return NULL;
		567	}
		568
		569	static struct vgic_irq vgic_its_check_cache(struct kvm kvm, phys_addr_t db,
		570	u32 devid, u32 eventid)
		571	{
		572	struct vgic_dist *dist = &kvm->arch.vgic;
		573	struct vgic_irq *irq;
		574	unsigned long flags;
		575
		576	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
		577	irq = __vgic_its_check_cache(dist, db, devid, eventid);
		578	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
		579
		580	return irq;
		581	}
		582
		583	static void vgic_its_cache_translation(struct kvm kvm, struct vgic_its its,
		584	u32 devid, u32 eventid,
		585	struct vgic_irq *irq)
		586	{
		587	struct vgic_dist *dist = &kvm->arch.vgic;
		588	struct vgic_translation_cache_entry *cte;
		589	unsigned long flags;
		590	phys_addr_t db;
		591
		592	/* Do not cache a directly injected interrupt */
		593	if (irq->hw)
		594	return;
		595
		596	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
		597
		598	if (unlikely(list_empty(&dist->lpi_translation_cache)))
		599	goto out;
		600
		601	/*
		602	* We could have raced with another CPU caching the same
		603	* translation behind our back, so let's check it is not in
		604	* already
		605	*/
		606	db = its->vgic_its_base + GITS_TRANSLATER;
		607	if (__vgic_its_check_cache(dist, db, devid, eventid))
		608	goto out;
		609
		610	/* Always reuse the last entry (LRU policy) */
		611	cte = list_last_entry(&dist->lpi_translation_cache,
		612	typeof(*cte), entry);
		613
		614	/*
		615	* Caching the translation implies having an extra reference
		616	* to the interrupt, so drop the potential reference on what
		617	* was in the cache, and increment it on the new interrupt.
		618	*/
		619	if (cte->irq)
		620	__vgic_put_lpi_locked(kvm, cte->irq);
		621
		622	vgic_get_irq_kref(irq);
		623
		624	cte->db = db;
		625	cte->devid = devid;
		626	cte->eventid = eventid;
		627	cte->irq = irq;
		628
		629	/* Move the new translation to the head of the list */
		630	list_move(&cte->entry, &dist->lpi_translation_cache);
		631
		632	out:
		633	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
		634	}
		635
		636	void vgic_its_invalidate_cache(struct kvm *kvm)
		637	{
		638	struct vgic_dist *dist = &kvm->arch.vgic;
		639	struct vgic_translation_cache_entry *cte;
		640	unsigned long flags;
		641
		642	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
		643
		644	list_for_each_entry(cte, &dist->lpi_translation_cache, entry) {
		645	/*
		646	* If we hit a NULL entry, there is nothing after this
		647	* point.
		648	*/
		649	if (!cte->irq)
		650	break;
		651
		652	__vgic_put_lpi_locked(kvm, cte->irq);
		653	cte->irq = NULL;
		654	}
		655
		656	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
		657	}
		658
530	int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its,	659	int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its,
531	u32 devid, u32 eventid, struct vgic_irq **irq)	660	u32 devid, u32 eventid, struct vgic_irq **irq)
532	{	661	{
@@ -547,6 +676,8 @@ int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its, · 14 unmodified lines
547	if (!vcpu->arch.vgic_cpu.lpis_enabled)	676	if (!vcpu->arch.vgic_cpu.lpis_enabled)
548	return -EBUSY;	677	return -EBUSY;
549		678
		679	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
		680
550	*irq = ite->irq;	681	*irq = ite->irq;
551	return 0;	682	return 0;
552	}	683	}
@@ -608,6 +739,25 @@ static int vgic_its_trigger_msi(struct kvm kvm, struct vgic_its its, · 55 unmodified lines
608	return 0;	739	return 0;
609	}	740	}
610		741
		742	int vgic_its_inject_cached_translation(struct kvm kvm, struct kvm_msi msi)
		743	{
		744	struct vgic_irq *irq;
		745	unsigned long flags;
		746	phys_addr_t db;
		747
		748	db = (u64)msi->address_hi << 32 \| msi->address_lo;
		749	irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data);
		750
		751	if (!irq)
		752	return -1;
		753
		754	raw_spin_lock_irqsave(&irq->irq_lock, flags);
		755	irq->pending_latch = true;
		756	vgic_queue_irq_unlock(kvm, irq, flags);
		757
		758	return 0;
		759	}
		760
611	/*	761	/*
612	* Queries the KVM IO bus framework to get the ITS pointer from the given	762	* Queries the KVM IO bus framework to get the ITS pointer from the given
613	* doorbell address.	763	* doorbell address.
@@ -619,6 +769,9 @@ int vgic_its_inject_msi(struct kvm kvm, struct kvm_msi msi) · 5 unmodified lines
619	struct vgic_its *its;	769	struct vgic_its *its;
620	int ret;	770	int ret;
621		771
		772	if (!vgic_its_inject_cached_translation(kvm, msi))
		773	return 1;
		774
622	its = vgic_msi_to_its(kvm, msi);	775	its = vgic_msi_to_its(kvm, msi);
623	if (IS_ERR(its))	776	if (IS_ERR(its))
624	return PTR_ERR(its);	777	return PTR_ERR(its);
@@ -691,6 +844,8 @@ static int vgic_its_cmd_handle_discard(struct kvm kvm, struct vgic_its its, · 66 unmodified lines
691	* don't bother here since we clear the ITTE anyway and the	844	* don't bother here since we clear the ITTE anyway and the
692	* pending state is a property of the ITTE struct.	845	* pending state is a property of the ITTE struct.
693	*/	846	*/
		847	vgic_its_invalidate_cache(kvm);
		848
694	its_free_ite(kvm, ite);	849	its_free_ite(kvm, ite);
695	return 0;	850	return 0;
696	}	851	}
@@ -726,6 +881,8 @@ static int vgic_its_cmd_handle_movi(struct kvm kvm, struct vgic_its its, · 29 unmodified lines
726	ite->collection = collection;	881	ite->collection = collection;
727	vcpu = kvm_get_vcpu(kvm, collection->target_addr);	882	vcpu = kvm_get_vcpu(kvm, collection->target_addr);
728		883
		884	vgic_its_invalidate_cache(kvm);
		885
729	return update_affinity(ite->irq, vcpu);	886	return update_affinity(ite->irq, vcpu);
730	}	887	}
731		888
@@ -954,6 +1111,8 @@ static void vgic_its_free_device(struct kvm kvm, struct its_device device) · 222 unmodified lines
954	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)	1111	list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
955	its_free_ite(kvm, ite);	1112	its_free_ite(kvm, ite);
956		1113
		1114	vgic_its_invalidate_cache(kvm);
		1115
957	list_del(&device->dev_list);	1116	list_del(&device->dev_list);
958	kfree(device);	1117	kfree(device);
959	}	1118	}
@@ -1059,6 +1218,7 @@ static int vgic_its_cmd_handle_mapc(struct kvm kvm, struct vgic_its its, · 99 unmodified lines
1059		1218
1060	if (!valid) {	1219	if (!valid) {
1061	vgic_its_free_collection(its, coll_id);	1220	vgic_its_free_collection(its, coll_id);
		1221	vgic_its_invalidate_cache(kvm);
1062	} else {	1222	} else {
1063	collection = find_collection(its, coll_id);	1223	collection = find_collection(its, coll_id);
1064		1224
@@ -1207,6 +1367,8 @@ static int vgic_its_cmd_handle_movall(struct kvm kvm, struct vgic_its its, · 142 unmodified lines
1207	vgic_put_irq(kvm, irq);	1367	vgic_put_irq(kvm, irq);
1208	}	1368	}
1209		1369
		1370	vgic_its_invalidate_cache(kvm);
		1371
1210	kfree(intids);	1372	kfree(intids);
1211	return 0;	1373	return 0;
1212	}	1374	}
@@ -1557,6 +1719,8 @@ static void vgic_mmio_write_its_ctlr(struct kvm kvm, struct vgic_its its, · 344 unmodified lines
1557	goto out;	1719	goto out;
1558		1720
1559	its->enabled = !!(val & GITS_CTLR_ENABLE);	1721	its->enabled = !!(val & GITS_CTLR_ENABLE);
		1722	if (!its->enabled)
		1723	vgic_its_invalidate_cache(kvm);
1560		1724
1561	/*	1725	/*
1562	* Try to process any pending commands. This function bails out early	1726	* Try to process any pending commands. This function bails out early
@@ -1657,6 +1821,47 @@ out: · 94 unmodified lines
1657	return ret;	1821	return ret;
1658	}	1822	}
1659		1823
		1824	/* Default is 16 cached LPIs per vcpu */
		1825	#define LPI_DEFAULT_PCPU_CACHE_SIZE 16
		1826
		1827	void vgic_lpi_translation_cache_init(struct kvm *kvm)
		1828	{
		1829	struct vgic_dist *dist = &kvm->arch.vgic;
		1830	unsigned int sz;
		1831	int i;
		1832
		1833	if (!list_empty(&dist->lpi_translation_cache))
		1834	return;
		1835
		1836	sz = atomic_read(&kvm->online_vcpus) * LPI_DEFAULT_PCPU_CACHE_SIZE;
		1837
		1838	for (i = 0; i < sz; i++) {
		1839	struct vgic_translation_cache_entry *cte;
		1840
		1841	/* An allocation failure is not fatal */
		1842	cte = kzalloc(sizeof(*cte), GFP_KERNEL);
		1843	if (WARN_ON(!cte))
		1844	break;
		1845
		1846	INIT_LIST_HEAD(&cte->entry);
		1847	list_add(&cte->entry, &dist->lpi_translation_cache);
		1848	}
		1849	}
		1850
		1851	void vgic_lpi_translation_cache_destroy(struct kvm *kvm)
		1852	{
		1853	struct vgic_dist *dist = &kvm->arch.vgic;
		1854	struct vgic_translation_cache_entry cte, tmp;
		1855
		1856	vgic_its_invalidate_cache(kvm);
		1857
		1858	list_for_each_entry_safe(cte, tmp,
		1859	&dist->lpi_translation_cache, entry) {
		1860	list_del(&cte->entry);
		1861	kfree(cte);
		1862	}
		1863	}
		1864
1660	#define INITIAL_BASER_VALUE \	1865	#define INITIAL_BASER_VALUE \
1661	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) \| \	1866	(GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb) \| \
1662	GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) \| \	1867	GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner) \| \
@@ -1685,6 +1890,8 @@ static int vgic_its_create(struct kvm_device *dev, u32 type) · 22 unmodified lines
1685	kfree(its);	1890	kfree(its);
1686	return ret;	1891	return ret;
1687	}	1892	}
		1893
		1894	vgic_lpi_translation_cache_init(dev->kvm);
1688	}	1895	}
1689		1896
1690	mutex_init(&its->its_lock);	1897	mutex_init(&its->its_lock);

virt/kvm/arm/vgic/vgic-mmio-v3.c+27−58

@@ -192,8 +192,10 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, · 191 unmodified lines
192		192
193	vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;	193	vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
194		194
195	if (was_enabled && !vgic_cpu->lpis_enabled)	195	if (was_enabled && !vgic_cpu->lpis_enabled) {
196	vgic_flush_pending_lpis(vcpu);	196	vgic_flush_pending_lpis(vcpu);
		197	vgic_its_invalidate_cache(vcpu->kvm);
		198	}
197		199
198	if (!was_enabled && vgic_cpu->lpis_enabled)	200	if (!was_enabled && vgic_cpu->lpis_enabled)
199	vgic_enable_lpis(vcpu);	201	vgic_enable_lpis(vcpu);
@@ -515,7 +517,8 @@ static const struct vgic_register_region vgic_v3_dist_registers[] = { · 315 unmodified lines
515	VGIC_ACCESS_32bit),	517	VGIC_ACCESS_32bit),
516	};	518	};
517		519
518	static const struct vgic_register_region vgic_v3_rdbase_registers[] = {	520	static const struct vgic_register_region vgic_v3_rd_registers[] = {
		521	/* RD_base registers */
519	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,	522	REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
520	vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,	523	vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
521	VGIC_ACCESS_32bit),	524	VGIC_ACCESS_32bit),
@@ -540,44 +543,42 @@ static const struct vgic_register_region vgic_v3_rdbase_registers[] = { · 18 unmodified lines
540	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,	543	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
541	vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,	544	vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
542	VGIC_ACCESS_32bit),	545	VGIC_ACCESS_32bit),
543	};	546	/* SGI_base registers */
544		547	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
545	static const struct vgic_register_region vgic_v3_sgibase_registers[] = {
546	REGISTER_DESC_WITH_LENGTH(GICR_IGROUPR0,
547	vgic_mmio_read_group, vgic_mmio_write_group, 4,	548	vgic_mmio_read_group, vgic_mmio_write_group, 4,
548	VGIC_ACCESS_32bit),	549	VGIC_ACCESS_32bit),
549	REGISTER_DESC_WITH_LENGTH(GICR_ISENABLER0,	550	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ISENABLER0,
550	vgic_mmio_read_enable, vgic_mmio_write_senable, 4,	551	vgic_mmio_read_enable, vgic_mmio_write_senable, 4,
551	VGIC_ACCESS_32bit),	552	VGIC_ACCESS_32bit),
552	REGISTER_DESC_WITH_LENGTH(GICR_ICENABLER0,	553	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICENABLER0,
553	vgic_mmio_read_enable, vgic_mmio_write_cenable, 4,	554	vgic_mmio_read_enable, vgic_mmio_write_cenable, 4,
554	VGIC_ACCESS_32bit),	555	VGIC_ACCESS_32bit),
555	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISPENDR0,	556	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
556	vgic_mmio_read_pending, vgic_mmio_write_spending,	557	vgic_mmio_read_pending, vgic_mmio_write_spending,
557	vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,	558	vgic_v3_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
558	VGIC_ACCESS_32bit),	559	VGIC_ACCESS_32bit),
559	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICPENDR0,	560	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
560	vgic_mmio_read_pending, vgic_mmio_write_cpending,	561	vgic_mmio_read_pending, vgic_mmio_write_cpending,
561	vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,	562	vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
562	VGIC_ACCESS_32bit),	563	VGIC_ACCESS_32bit),
563	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ISACTIVER0,	564	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
564	vgic_mmio_read_active, vgic_mmio_write_sactive,	565	vgic_mmio_read_active, vgic_mmio_write_sactive,
565	NULL, vgic_mmio_uaccess_write_sactive,	566	NULL, vgic_mmio_uaccess_write_sactive,
566	4, VGIC_ACCESS_32bit),	567	4, VGIC_ACCESS_32bit),
567	REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_ICACTIVER0,	568	REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
568	vgic_mmio_read_active, vgic_mmio_write_cactive,	569	vgic_mmio_read_active, vgic_mmio_write_cactive,
569	NULL, vgic_mmio_uaccess_write_cactive,	570	NULL, vgic_mmio_uaccess_write_cactive,
570	4, VGIC_ACCESS_32bit),	571	4, VGIC_ACCESS_32bit),
571	REGISTER_DESC_WITH_LENGTH(GICR_IPRIORITYR0,	572	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
572	vgic_mmio_read_priority, vgic_mmio_write_priority, 32,	573	vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
573	VGIC_ACCESS_32bit \| VGIC_ACCESS_8bit),	574	VGIC_ACCESS_32bit \| VGIC_ACCESS_8bit),
574	REGISTER_DESC_WITH_LENGTH(GICR_ICFGR0,	575	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
575	vgic_mmio_read_config, vgic_mmio_write_config, 8,	576	vgic_mmio_read_config, vgic_mmio_write_config, 8,
576	VGIC_ACCESS_32bit),	577	VGIC_ACCESS_32bit),
577	REGISTER_DESC_WITH_LENGTH(GICR_IGRPMODR0,	578	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
578	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,	579	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
579	VGIC_ACCESS_32bit),	580	VGIC_ACCESS_32bit),
580	REGISTER_DESC_WITH_LENGTH(GICR_NSACR,	581	REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
581	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,	582	vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
582	VGIC_ACCESS_32bit),	583	VGIC_ACCESS_32bit),
583	};	584	};
@@ -607,9 +608,8 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) · 23 unmodified lines
607	struct vgic_dist *vgic = &kvm->arch.vgic;	608	struct vgic_dist *vgic = &kvm->arch.vgic;
608	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;	609	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
609	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;	610	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
610	struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
611	struct vgic_redist_region *rdreg;	611	struct vgic_redist_region *rdreg;
612	gpa_t rd_base, sgi_base;	612	gpa_t rd_base;
613	int ret;	613	int ret;
614		614
615	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))	615	if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
@@ -631,52 +631,31 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) · 15 unmodified lines
631	vgic_cpu->rdreg = rdreg;	631	vgic_cpu->rdreg = rdreg;
632		632
633	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;	633	rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
634	sgi_base = rd_base + SZ_64K;
635		634
636	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);	635	kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
637	rd_dev->base_addr = rd_base;	636	rd_dev->base_addr = rd_base;
638	rd_dev->iodev_type = IODEV_REDIST;	637	rd_dev->iodev_type = IODEV_REDIST;
639	rd_dev->regions = vgic_v3_rdbase_registers;	638	rd_dev->regions = vgic_v3_rd_registers;
640	rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);	639	rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
641	rd_dev->redist_vcpu = vcpu;	640	rd_dev->redist_vcpu = vcpu;
642		641
643	mutex_lock(&kvm->slots_lock);	642	mutex_lock(&kvm->slots_lock);
644	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,	643	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
645	SZ_64K, &rd_dev->dev);	644	2 * SZ_64K, &rd_dev->dev);
646	mutex_unlock(&kvm->slots_lock);	645	mutex_unlock(&kvm->slots_lock);
647		646
648	if (ret)	647	if (ret)
649	return ret;	648	return ret;
650		649
651	kvm_iodevice_init(&sgi_dev->dev, &kvm_io_gic_ops);
652	sgi_dev->base_addr = sgi_base;
653	sgi_dev->iodev_type = IODEV_REDIST;
654	sgi_dev->regions = vgic_v3_sgibase_registers;
655	sgi_dev->nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers);
656	sgi_dev->redist_vcpu = vcpu;
657
658	mutex_lock(&kvm->slots_lock);
659	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, sgi_base,
660	SZ_64K, &sgi_dev->dev);
661	if (ret) {
662	kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
663	&rd_dev->dev);
664	goto out;
665	}
666
667	rdreg->free_index++;	650	rdreg->free_index++;
668	out:	651	return 0;
669	mutex_unlock(&kvm->slots_lock);
670	return ret;
671	}	652	}
672		653
673	static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)	654	static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
674	{	655	{
675	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;	656	struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
676	struct vgic_io_device *sgi_dev = &vcpu->arch.vgic_cpu.sgi_iodev;
677		657
678	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);	658	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
679	kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &sgi_dev->dev);
680	}	659	}
681		660
682	static int vgic_register_all_redist_iodevs(struct kvm *kvm)	661	static int vgic_register_all_redist_iodevs(struct kvm *kvm)
@@ -826,8 +805,8 @@ int vgic_v3_has_attr_regs(struct kvm_device dev, struct kvm_device_attr attr) · 143 unmodified lines
826	iodev.base_addr = 0;	805	iodev.base_addr = 0;
827	break;	806	break;
828	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{	807	case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
829	iodev.regions = vgic_v3_rdbase_registers;	808	iodev.regions = vgic_v3_rd_registers;
830	iodev.nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers);	809	iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
831	iodev.base_addr = 0;	810	iodev.base_addr = 0;
832	break;	811	break;
833	}	812	}
@@ -985,21 +964,11 @@ int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, · 151 unmodified lines
985	int offset, u32 *val)	964	int offset, u32 *val)
986	{	965	{
987	struct vgic_io_device rd_dev = {	966	struct vgic_io_device rd_dev = {
988	.regions = vgic_v3_rdbase_registers,	967	.regions = vgic_v3_rd_registers,
989	.nr_regions = ARRAY_SIZE(vgic_v3_rdbase_registers),	968	.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
990	};	969	};
991		970
992	struct vgic_io_device sgi_dev = {	971	return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
993	.regions = vgic_v3_sgibase_registers,
994	.nr_regions = ARRAY_SIZE(vgic_v3_sgibase_registers),
995	};
996
997	/* SGI_base is the next 64K frame after RD_base */
998	if (offset >= SZ_64K)
999	return vgic_uaccess(vcpu, &sgi_dev, is_write, offset - SZ_64K,
1000	val);
1001	else
1002	return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1003	}	972	}
1004		973
1005	int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,	974	int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,

virt/kvm/arm/vgic/vgic-v2.c+4−3

 DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap);
 /**
- * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT
+ * vgic_v2_probe - probe for a VGICv2 compatible interrupt controller
- * @node:	pointer to the DT node
+ * @info:	pointer to the GIC description
  *
- * Returns 0 if a GICv2 has been found, returns an error code otherwise
+ * Returns 0 if the VGICv2 has been probed successfully, returns an error code
+ * otherwise
  */
 int vgic_v2_probe(const struct gic_kvm_info *info)
 {

virt/kvm/arm/vgic/vgic-v3.c+4−3

 early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable);
 /**
- * vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT
+ * vgic_v3_probe - probe for a VGICv3 compatible interrupt controller
- * @node:	pointer to the DT node
+ * @info:	pointer to the GIC description
  *
- * Returns 0 if a GICv3 has been found, returns an error code otherwise
+ * Returns 0 if the VGICv3 has been probed successfully, returns an error code
+ * otherwise
  */
 int vgic_v3_probe(const struct gic_kvm_info *info)
 {

virt/kvm/arm/vgic/vgic.c+17−9

 {
 }
+/*
+ * Drop the refcount on the LPI. Must be called with lpi_list_lock held.
+ */
+void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq)
+{
+	struct vgic_dist *dist = &kvm->arch.vgic;
+	if (!kref_put(&irq->refcount, vgic_irq_release))
+		return;
+	list_del(&irq->lpi_list);
+	dist->lpi_list_count--;
+	kfree(irq);
+}
 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
 {
 	struct vgic_dist *dist = &kvm->arch.vgic;
 		return;
 	raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
-	if (!kref_put(&irq->refcount, vgic_irq_release)) {
+	__vgic_put_lpi_locked(kvm, irq);
-		raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
-		return;
-	};
-	list_del(&irq->lpi_list);
-	dist->lpi_list_count--;
 	raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
-	kfree(irq);
 }
 void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)

virt/kvm/arm/vgic/vgic.h+5−0

 		     gpa_t addr, int len);
 struct vgic_irq *vgic_get_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 			      u32 intid);
+void __vgic_put_lpi_locked(struct kvm *kvm, struct vgic_irq *irq);
 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
 bool vgic_get_phys_line_level(struct vgic_irq *irq);
 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);
 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
 			 u32 devid, u32 eventid, struct vgic_irq **irq);
 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi);
+int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi);
+void vgic_lpi_translation_cache_init(struct kvm *kvm);
+void vgic_lpi_translation_cache_destroy(struct kvm *kvm);
+void vgic_its_invalidate_cache(struct kvm *kvm);
 bool vgic_supports_direct_msis(struct kvm *kvm);
 int vgic_v4_init(struct kvm *kvm);

virt/kvm/kvm_main.c+3−4

 	bool waited = false;
 	u64 block_ns;
+	kvm_arch_vcpu_blocking(vcpu);
 	start = cur = ktime_get();
 	if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
 		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
 		} while (single_task_running() && ktime_before(cur, stop));
 	}
-	kvm_arch_vcpu_blocking(vcpu);
 	for (;;) {
 		prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
 	finish_swait(&vcpu->wq, &wait);
 	cur = ktime_get();
-	kvm_arch_vcpu_unblocking(vcpu);
 out:
+	kvm_arch_vcpu_unblocking(vcpu);
 	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
 	if (!vcpu_valid_wakeup(vcpu))

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -294,7 +294,7 @@ Handling a page fault is performed as follows: · 293 unmodified lines
294	- walk shadow page table	294	- walk shadow page table
295	- check for valid generation number in the spte (see "Fast invalidation of	295	- check for valid generation number in the spte (see "Fast invalidation of
296	MMIO sptes" below)	296	MMIO sptes" below)
297	- cache the information to vcpu->arch.mmio_gva, vcpu->arch.access and	297	- cache the information to vcpu->arch.mmio_gva, vcpu->arch.mmio_access and
298	vcpu->arch.mmio_gfn, and call the emulator	298	vcpu->arch.mmio_gfn, and call the emulator
299	- If both P bit and R/W bit of error code are set, this could possibly	299	- If both P bit and R/W bit of error code are set, this could possibly
300	be handled as a "fast page fault" (fixed without taking the MMU lock). See	300	be handled as a "fast page fault" (fixed without taking the MMU lock). See
@@ -304,7 +304,7 @@ Handling a page fault is performed as follows: · 3 unmodified lines
304	- if permissions are insufficient, reflect the fault back to the guest	304	- if permissions are insufficient, reflect the fault back to the guest
305	- determine the host page	305	- determine the host page
306	- if this is an mmio request, there is no host page; cache the info to	306	- if this is an mmio request, there is no host page; cache the info to
307	vcpu->arch.mmio_gva, vcpu->arch.access and vcpu->arch.mmio_gfn	307	vcpu->arch.mmio_gva, vcpu->arch.mmio_access and vcpu->arch.mmio_gfn
308	- walk the shadow page table to find the spte for the translation,	308	- walk the shadow page table to find the spte for the translation,
309	instantiating missing intermediate page tables as necessary	309	instantiating missing intermediate page tables as necessary
310	- If this is an mmio request, cache the mmio info to the spte and set some	310	- If this is an mmio request, cache the mmio info to the spte and set some