lael-labs/linux

Documentation/virtual/kvm/api.txt+41−0

@@ -1451,6 +1451,7 @@ struct kvm_irq_routing_entry { · 1450 unmodified lines
1451	struct kvm_irq_routing_irqchip irqchip;	1451	struct kvm_irq_routing_irqchip irqchip;
1452	struct kvm_irq_routing_msi msi;	1452	struct kvm_irq_routing_msi msi;
1453	struct kvm_irq_routing_s390_adapter adapter;	1453	struct kvm_irq_routing_s390_adapter adapter;
		1454	struct kvm_irq_routing_hv_sint hv_sint;
1454	__u32 pad[8];	1455	__u32 pad[8];
1455	} u;	1456	} u;
1456	};	1457	};
@@ -1459,6 +1460,7 @@ struct kvm_irq_routing_entry { · 2 unmodified lines
1459	#define KVM_IRQ_ROUTING_IRQCHIP 1	1460	#define KVM_IRQ_ROUTING_IRQCHIP 1
1460	#define KVM_IRQ_ROUTING_MSI 2	1461	#define KVM_IRQ_ROUTING_MSI 2
1461	#define KVM_IRQ_ROUTING_S390_ADAPTER 3	1462	#define KVM_IRQ_ROUTING_S390_ADAPTER 3
		1463	#define KVM_IRQ_ROUTING_HV_SINT 4
1462		1464
1463	No flags are specified so far, the corresponding field must be set to zero.	1465	No flags are specified so far, the corresponding field must be set to zero.
1464		1466
@@ -1482,6 +1484,10 @@ struct kvm_irq_routing_s390_adapter { · 17 unmodified lines
1482	__u32 adapter_id;	1484	__u32 adapter_id;
1483	};	1485	};
1484		1486
		1487	struct kvm_irq_routing_hv_sint {
		1488	__u32 vcpu;
		1489	__u32 sint;
		1490	};
1485		1491
1486	4.53 KVM_ASSIGN_SET_MSIX_NR (deprecated)	1492	4.53 KVM_ASSIGN_SET_MSIX_NR (deprecated)
1487		1493
@@ -3331,6 +3337,28 @@ the userspace IOAPIC should process the EOI and retrigger the interrupt if · 1843 unmodified lines
3331	it is still asserted. Vector is the LAPIC interrupt vector for which the	3337	it is still asserted. Vector is the LAPIC interrupt vector for which the
3332	EOI was received.	3338	EOI was received.
3333		3339
		3340	struct kvm_hyperv_exit {
		3341	#define KVM_EXIT_HYPERV_SYNIC 1
		3342	__u32 type;
		3343	union {
		3344	struct {
		3345	__u32 msr;
		3346	__u64 control;
		3347	__u64 evt_page;
		3348	__u64 msg_page;
		3349	} synic;
		3350	} u;
		3351	};
		3352	/* KVM_EXIT_HYPERV */
		3353	struct kvm_hyperv_exit hyperv;
		3354	Indicates that the VCPU exits into userspace to process some tasks
		3355	related to Hyper-V emulation.
		3356	Valid values for 'type' are:
		3357	KVM_EXIT_HYPERV_SYNIC -- synchronously notify user-space about
		3358	Hyper-V SynIC state change. Notification is used to remap SynIC
		3359	event/message pages and to enable/disable SynIC messages/events processing
		3360	in userspace.
		3361
3334	/* Fix the size of the union. */	3362	/* Fix the size of the union. */
3335	char padding[256];	3363	char padding[256];
3336	};	3364	};
@@ -3685,3 +3713,16 @@ available, means that that the kernel has an implementation of the · 348 unmodified lines
3685	H_RANDOM hypercall backed by a hardware random-number generator.	3713	H_RANDOM hypercall backed by a hardware random-number generator.
3686	If present, the kernel H_RANDOM handler can be enabled for guest use	3714	If present, the kernel H_RANDOM handler can be enabled for guest use
3687	with the KVM_CAP_PPC_ENABLE_HCALL capability.	3715	with the KVM_CAP_PPC_ENABLE_HCALL capability.
		3716
		3717	8.2 KVM_CAP_HYPERV_SYNIC
		3718
		3719	Architectures: x86
		3720	This capability, if KVM_CHECK_EXTENSION indicates that it is
		3721	available, means that that the kernel has an implementation of the
		3722	Hyper-V Synthetic interrupt controller(SynIC). Hyper-V SynIC is
		3723	used to support Windows Hyper-V based guest paravirt drivers(VMBus).
		3724
		3725	In order to use SynIC, it has to be activated by setting this
		3726	capability via KVM_ENABLE_CAP ioctl on the vcpu fd. Note that this
		3727	will disable the use of APIC hardware virtualization even if supported
		3728	by the CPU, as it's incompatible with SynIC auto-EOI behavior.

Documentation/virtual/kvm/devices/vm.txt+2−1

@@ -37,7 +37,8 @@ Returns: -EFAULT if the given address is not accessible · 36 unmodified lines
37	Allows userspace to query the actual limit and set a new limit for	37	Allows userspace to query the actual limit and set a new limit for
38	the maximum guest memory size. The limit will be rounded up to	38	the maximum guest memory size. The limit will be rounded up to
39	2048 MB, 4096 GB, 8192 TB respectively, as this limit is governed by	39	2048 MB, 4096 GB, 8192 TB respectively, as this limit is governed by
40	the number of page table levels.	40	the number of page table levels. In the case that there is no limit we will set
		41	the limit to KVM_S390_NO_MEM_LIMIT (U64_MAX).
41		42
42	2. GROUP: KVM_S390_VM_CPU_MODEL	43	2. GROUP: KVM_S390_VM_CPU_MODEL
43	Architectures: s390	44	Architectures: s390

Documentation/virtual/kvm/mmu.txt+2−2

@@ -203,10 +203,10 @@ Shadow pages contain the following information: · 202 unmodified lines
203	page cannot be destroyed. See role.invalid.	203	page cannot be destroyed. See role.invalid.
204	parent_ptes:	204	parent_ptes:
205	The reverse mapping for the pte/ptes pointing at this page's spt. If	205	The reverse mapping for the pte/ptes pointing at this page's spt. If
206	parent_ptes bit 0 is zero, only one spte points at this pages and	206	parent_ptes bit 0 is zero, only one spte points at this page and
207	parent_ptes points at this single spte, otherwise, there exists multiple	207	parent_ptes points at this single spte, otherwise, there exists multiple
208	sptes pointing at this page and (parent_ptes & ~0x1) points at a data	208	sptes pointing at this page and (parent_ptes & ~0x1) points at a data
209	structure with a list of parent_ptes.	209	structure with a list of parent sptes.
210	unsync:	210	unsync:
211	If true, then the translations in this page may not match the guest's	211	If true, then the translations in this page may not match the guest's
212	translation. This is equivalent to the state of the tlb when a pte is	212	translation. This is equivalent to the state of the tlb when a pte is

MAINTAINERS+1−0

@@ -6089,6 +6089,7 @@ M: Marc Zyngier <marc.zyngier@arm.com> · 6088 unmodified lines
6089	L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)	6089	L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
6090	L: kvmarm@lists.cs.columbia.edu	6090	L: kvmarm@lists.cs.columbia.edu
6091	W: http://systems.cs.columbia.edu/projects/kvm-arm	6091	W: http://systems.cs.columbia.edu/projects/kvm-arm
		6092	T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git
6092	S: Supported	6093	S: Supported
6093	F: arch/arm/include/uapi/asm/kvm*	6094	F: arch/arm/include/uapi/asm/kvm*
6094	F: arch/arm/include/asm/kvm*	6095	F: arch/arm/include/asm/kvm*

arch/arm/include/asm/kvm_arm.h+17−17

 #ifndef __ARM_KVM_ARM_H__
 #define __ARM_KVM_ARM_H__
+#include <linux/const.h>
 #include <linux/types.h>
 /* Hyp Configuration Register (HCR) bits */
  * space.
  */
 #define KVM_PHYS_SHIFT	(40)
-#define KVM_PHYS_SIZE	(1ULL << KVM_PHYS_SHIFT)
+#define KVM_PHYS_SIZE	(_AC(1, ULL) << KVM_PHYS_SHIFT)
-#define KVM_PHYS_MASK	(KVM_PHYS_SIZE - 1ULL)
+#define KVM_PHYS_MASK	(KVM_PHYS_SIZE - _AC(1, ULL))
-#define PTRS_PER_S2_PGD	(1ULL << (KVM_PHYS_SHIFT - 30))
+#define PTRS_PER_S2_PGD	(_AC(1, ULL) << (KVM_PHYS_SHIFT - 30))
-#define S2_PGD_ORDER	get_order(PTRS_PER_S2_PGD * sizeof(pgd_t))
 /* Virtualization Translation Control Register (VTCR) bits */
 #define VTCR_SH0	(3 << 12)
 #define VTTBR_X		(5 - KVM_T0SZ)
 #endif
 #define VTTBR_BADDR_SHIFT (VTTBR_X - 1)
-#define VTTBR_BADDR_MASK  (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
+#define VTTBR_BADDR_MASK  (((_AC(1, ULL) << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
-#define VTTBR_VMID_SHIFT  (48LLU)
+#define VTTBR_VMID_SHIFT  _AC(48, ULL)
-#define VTTBR_VMID_MASK	  (0xffLLU << VTTBR_VMID_SHIFT)
+#define VTTBR_VMID_MASK(size)	(_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT)
 /* Hyp Syndrome Register (HSR) bits */
 #define HSR_EC_SHIFT	(26)
-#define HSR_EC		(0x3fU << HSR_EC_SHIFT)
+#define HSR_EC		(_AC(0x3f, UL) << HSR_EC_SHIFT)
-#define HSR_IL		(1U << 25)
+#define HSR_IL		(_AC(1, UL) << 25)
 #define HSR_ISS		(HSR_IL - 1)
 #define HSR_ISV_SHIFT	(24)
-#define HSR_ISV		(1U << HSR_ISV_SHIFT)
+#define HSR_ISV		(_AC(1, UL) << HSR_ISV_SHIFT)
 #define HSR_SRT_SHIFT	(16)
 #define HSR_SRT_MASK	(0xf << HSR_SRT_SHIFT)
 #define HSR_FSC		(0x3f)
 #define HSR_SSE		(1 << 21)
 #define HSR_WNR		(1 << 6)
 #define HSR_CV_SHIFT	(24)
-#define HSR_CV		(1U << HSR_CV_SHIFT)
+#define HSR_CV		(_AC(1, UL) << HSR_CV_SHIFT)
 #define HSR_COND_SHIFT	(20)
-#define HSR_COND	(0xfU << HSR_COND_SHIFT)
+#define HSR_COND	(_AC(0xf, UL) << HSR_COND_SHIFT)
 #define FSC_FAULT	(0x04)
 #define FSC_ACCESS	(0x08)
 #define HSR_EC_DABT	(0x24)
 #define HSR_EC_DABT_HYP	(0x25)
-#define HSR_WFI_IS_WFE		(1U << 0)
+#define HSR_WFI_IS_WFE		(_AC(1, UL) << 0)
-#define HSR_HVC_IMM_MASK	((1UL << 16) - 1)
+#define HSR_HVC_IMM_MASK	((_AC(1, UL) << 16) - 1)
-#define HSR_DABT_S1PTW		(1U << 7)
+#define HSR_DABT_S1PTW		(_AC(1, UL) << 7)
-#define HSR_DABT_CM		(1U << 8)
+#define HSR_DABT_CM		(_AC(1, UL) << 8)
-#define HSR_DABT_EA		(1U << 9)
+#define HSR_DABT_EA		(_AC(1, UL) << 9)
 #define kvm_arm_exception_type	\
 	{0, "RESET" }, 		\

arch/arm/include/asm/kvm_host.h+6−0

 	u32 halt_successful_poll;
 	u32 halt_attempted_poll;
 	u32 halt_wakeup;
+	u32 hvc_exit_stat;
+	u64 wfe_exit_stat;
+	u64 wfi_exit_stat;
+	u64 mmio_exit_user;
+	u64 mmio_exit_kernel;
+	u64 exits;
 };
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);

arch/arm/include/asm/kvm_mmu.h+5−0

 				       pgd_t *merged_hyp_pgd,
 				       unsigned long hyp_idmap_start) { }
+static inline unsigned int kvm_get_vmid_bits(void)
+{
+	return 8;
+}
 #endif	/* !__ASSEMBLY__ */
 #endif /* __ARM_KVM_MMU_H__ */

arch/arm/kvm/arm.c+36−4

 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_psci.h>
+#include <asm/sections.h>
 #ifdef REQUIRES_VIRT
 __asm__(".arch_extension	virt");
 /* The VMID used in the VTTBR */
 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
-static u8 kvm_next_vmid;
+static u32 kvm_next_vmid;
+static unsigned int kvm_vmid_bits __read_mostly;
 static DEFINE_SPINLOCK(kvm_vmid_lock);
+static bool vgic_present;
 static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
 {
 	BUG_ON(preemptible());
 	kvm->arch.vmid_gen = 0;
 	/* The maximum number of VCPUs is limited by the host's GIC model */
-	kvm->arch.max_vcpus = kvm_vgic_get_max_vcpus();
+	kvm->arch.max_vcpus = vgic_present ?
+				kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
 	return ret;
 out_free_stage2_pgd:
 	int r;
 	switch (ext) {
 	case KVM_CAP_IRQCHIP:
+		r = vgic_present;
+		break;
 	case KVM_CAP_IOEVENTFD:
 	case KVM_CAP_DEVICE_CTRL:
 	case KVM_CAP_USER_MEMORY:
 	kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
 	kvm->arch.vmid = kvm_next_vmid;
 	kvm_next_vmid++;
+	kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
 	/* update vttbr to be used with the new vmid */
 	pgd_phys = virt_to_phys(kvm_get_hwpgd(kvm));
 	BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
-	vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
+	vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
 	kvm->arch.vttbr = pgd_phys | vmid;
 	spin_unlock(&kvm_vmid_lock);
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 		vcpu->mode = OUTSIDE_GUEST_MODE;
+		vcpu->stat.exits++;
 		/*
 		 * Back from guest
 		 *************************************************************/
 	switch (dev_id) {
 	case KVM_ARM_DEVICE_VGIC_V2:
+		if (!vgic_present)
+			return -ENXIO;
 		return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
 	default:
 		return -ENODEV;
 	switch (ioctl) {
 	case KVM_CREATE_IRQCHIP: {
+		if (!vgic_present)
+			return -ENXIO;
 		return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
 	}
 	case KVM_ARM_SET_DEVICE_ADDR: {
 		goto out_free_mappings;
 	}
+	err = create_hyp_mappings(__start_rodata, __end_rodata);
+	if (err) {
+		kvm_err("Cannot map rodata section\n");
+		goto out_free_mappings;
+	}
 	/*
 	 * Map the Hyp stack pages
 	 */
 	 * Init HYP view of VGIC
 	 */
 	err = kvm_vgic_hyp_init();
-	if (err)
+	switch (err) {
+	case 0:
+		vgic_present = true;
+		break;
+	case -ENODEV:
+	case -ENXIO:
+		vgic_present = false;
+		break;
+	default:
 		goto out_free_context;
+	}
 	/*
 	 * Init HYP architected timer support
 	kvm_perf_init();
+	/* set size of VMID supported by CPU */
+	kvm_vmid_bits = kvm_get_vmid_bits();
+	kvm_info("%d-bit VMID\n", kvm_vmid_bits);
 	kvm_info("Hyp mode initialized successfully\n");
 	return 0;

arch/arm/kvm/emulate.c+38−36

 		return vbar;
 }
+/*
+ * Switch to an exception mode, updating both CPSR and SPSR. Follow
+ * the logic described in AArch32.EnterMode() from the ARMv8 ARM.
+ */
+static void kvm_update_psr(struct kvm_vcpu *vcpu, unsigned long mode)
+{
+	unsigned long cpsr = *vcpu_cpsr(vcpu);
+	u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
+	*vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | mode;
+	switch (mode) {
+	case FIQ_MODE:
+		*vcpu_cpsr(vcpu) |= PSR_F_BIT;
+		/* Fall through */
+	case ABT_MODE:
+	case IRQ_MODE:
+		*vcpu_cpsr(vcpu) |= PSR_A_BIT;
+		/* Fall through */
+	default:
+		*vcpu_cpsr(vcpu) |= PSR_I_BIT;
+	}
+	*vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
+	if (sctlr & SCTLR_TE)
+		*vcpu_cpsr(vcpu) |= PSR_T_BIT;
+	if (sctlr & SCTLR_EE)
+		*vcpu_cpsr(vcpu) |= PSR_E_BIT;
+	/* Note: These now point to the mode banked copies */
+	*vcpu_spsr(vcpu) = cpsr;
+}
 /**
  * kvm_inject_undefined - inject an undefined exception into the guest
  * @vcpu: The VCPU to receive the undefined exception
  */
 void kvm_inject_undefined(struct kvm_vcpu *vcpu)
 {
-	unsigned long new_lr_value;
-	unsigned long new_spsr_value;
 	unsigned long cpsr = *vcpu_cpsr(vcpu);
-	u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
 	bool is_thumb = (cpsr & PSR_T_BIT);
 	u32 vect_offset = 4;
 	u32 return_offset = (is_thumb) ? 2 : 4;
-	new_spsr_value = cpsr;
+	kvm_update_psr(vcpu, UND_MODE);
-	new_lr_value = *vcpu_pc(vcpu) - return_offset;
+	*vcpu_reg(vcpu, 14) = *vcpu_pc(vcpu) - return_offset;
-	*vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | UND_MODE;
-	*vcpu_cpsr(vcpu) |= PSR_I_BIT;
-	*vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
-	if (sctlr & SCTLR_TE)
-		*vcpu_cpsr(vcpu) |= PSR_T_BIT;
-	if (sctlr & SCTLR_EE)
-		*vcpu_cpsr(vcpu) |= PSR_E_BIT;
-	/* Note: These now point to UND banked copies */
-	*vcpu_spsr(vcpu) = cpsr;
-	*vcpu_reg(vcpu, 14) = new_lr_value;
 	/* Branch to exception vector */
 	*vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset;
  */
 static void inject_abt(struct kvm_vcpu *vcpu, bool is_pabt, unsigned long addr)
 {
-	unsigned long new_lr_value;
-	unsigned long new_spsr_value;
 	unsigned long cpsr = *vcpu_cpsr(vcpu);
-	u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
 	bool is_thumb = (cpsr & PSR_T_BIT);
 	u32 vect_offset;
 	u32 return_offset = (is_thumb) ? 4 : 0;
 	bool is_lpae;
-	new_spsr_value = cpsr;
+	kvm_update_psr(vcpu, ABT_MODE);
-	new_lr_value = *vcpu_pc(vcpu) + return_offset;
+	*vcpu_reg(vcpu, 14) = *vcpu_pc(vcpu) + return_offset;
-	*vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | ABT_MODE;
-	*vcpu_cpsr(vcpu) |= PSR_I_BIT | PSR_A_BIT;
-	*vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
-	if (sctlr & SCTLR_TE)
-		*vcpu_cpsr(vcpu) |= PSR_T_BIT;
-	if (sctlr & SCTLR_EE)
-		*vcpu_cpsr(vcpu) |= PSR_E_BIT;
-	/* Note: These now point to ABT banked copies */
-	*vcpu_spsr(vcpu) = cpsr;
-	*vcpu_reg(vcpu, 14) = new_lr_value;
 	if (is_pabt)
 		vect_offset = 12;

arch/arm/kvm/guest.c+6−0

 #define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfe_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_user),
+	VCPU_STAT(mmio_exit_kernel),
+	VCPU_STAT(exits),
 	{ NULL }
 };

arch/arm/kvm/handle_exit.c+3−0

 	trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
 		      kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {
 {
 	if (kvm_vcpu_get_hsr(vcpu) & HSR_WFI_IS_WFE) {
 		trace_kvm_wfx(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfe_exit_stat++;
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}

arch/arm/kvm/mmio.c+3−0

 	if (!ret) {
 		/* We handled the access successfully in the kernel. */
+		vcpu->stat.mmio_exit_kernel++;
 		kvm_handle_mmio_return(vcpu, run);
 		return 1;
+	} else {
+		vcpu->stat.mmio_exit_user++;
 	}
 	run->exit_reason	= KVM_EXIT_MMIO;

arch/arm/kvm/mmu.c+3−3

@@ -656,9 +656,9 @@ static void *kvm_alloc_hwpgd(void) · 655 unmodified lines
656	* kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.	656	* kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
657	* @kvm: The KVM struct pointer for the VM.	657	* @kvm: The KVM struct pointer for the VM.
658	*	658	*
659	* Allocates the 1st level table only of size defined by S2_PGD_ORDER (can	659	* Allocates only the stage-2 HW PGD level table(s) (can support either full
660	* support either full 40-bit input addresses or limited to 32-bit input	660	* 40-bit input addresses or limited to 32-bit input addresses). Clears the
661	* addresses). Clears the allocated pages.	661	* allocated pages.
662	*	662	*
663	* Note we don't need locking here as this is only called when the VM is	663	* Note we don't need locking here as this is only called when the VM is
664	* created, which can only be done once.	664	* created, which can only be done once.

arch/arm64/include/asm/kvm_arm.h+2−1

 #define VTCR_EL2_SL0_LVL1	(1 << 6)
 #define VTCR_EL2_T0SZ_MASK	0x3f
 #define VTCR_EL2_T0SZ_40B	24
+#define VTCR_EL2_VS		19
 /*
  * We configure the Stage-2 page tables to always restrict the IPA space to be
 #define VTTBR_BADDR_SHIFT (VTTBR_X - 1)
 #define VTTBR_BADDR_MASK  (((UL(1) << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
 #define VTTBR_VMID_SHIFT  (UL(48))
-#define VTTBR_VMID_MASK	  (UL(0xFF) << VTTBR_VMID_SHIFT)
+#define VTTBR_VMID_MASK(size) (_AT(u64, (1 << size) - 1) << VTTBR_VMID_SHIFT)
 /* Hyp System Trap Register */
 #define HSTR_EL2_T(x)	(1 << x)

arch/arm64/include/asm/kvm_asm.h+0−76

@@ -20,82 +20,6 @@ · 19 unmodified lines
20		20
21	#include <asm/virt.h>	21	#include <asm/virt.h>
22		22
23	/*
24	* 0 is reserved as an invalid value.
25	* Order must be kept in sync with the hyp switch code.
26	*/
27	#define MPIDR_EL1 1 /* MultiProcessor Affinity Register */
28	#define CSSELR_EL1 2 /* Cache Size Selection Register */
29	#define SCTLR_EL1 3 /* System Control Register */
30	#define ACTLR_EL1 4 /* Auxiliary Control Register */
31	#define CPACR_EL1 5 /* Coprocessor Access Control */
32	#define TTBR0_EL1 6 /* Translation Table Base Register 0 */
33	#define TTBR1_EL1 7 /* Translation Table Base Register 1 */
34	#define TCR_EL1 8 /* Translation Control Register */
35	#define ESR_EL1 9 /* Exception Syndrome Register */
36	#define AFSR0_EL1 10 /* Auxilary Fault Status Register 0 */
37	#define AFSR1_EL1 11 /* Auxilary Fault Status Register 1 */
38	#define FAR_EL1 12 /* Fault Address Register */
39	#define MAIR_EL1 13 /* Memory Attribute Indirection Register */
40	#define VBAR_EL1 14 /* Vector Base Address Register */
41	#define CONTEXTIDR_EL1 15 /* Context ID Register */
42	#define TPIDR_EL0 16 /* Thread ID, User R/W */
43	#define TPIDRRO_EL0 17 /* Thread ID, User R/O */
44	#define TPIDR_EL1 18 /* Thread ID, Privileged */
45	#define AMAIR_EL1 19 /* Aux Memory Attribute Indirection Register */
46	#define CNTKCTL_EL1 20 /* Timer Control Register (EL1) */
47	#define PAR_EL1 21 /* Physical Address Register */
48	#define MDSCR_EL1 22 /* Monitor Debug System Control Register */
49	#define MDCCINT_EL1 23 /* Monitor Debug Comms Channel Interrupt Enable Reg */
50
51	/* 32bit specific registers. Keep them at the end of the range */
52	#define DACR32_EL2 24 /* Domain Access Control Register */
53	#define IFSR32_EL2 25 /* Instruction Fault Status Register */
54	#define FPEXC32_EL2 26 /* Floating-Point Exception Control Register */
55	#define DBGVCR32_EL2 27 /* Debug Vector Catch Register */
56	#define NR_SYS_REGS 28
57
58	/* 32bit mapping */
59	#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
60	#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */
61	#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */
62	#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */
63	#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */
64	#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */
65	#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */
66	#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */
67	#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */
68	#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */
69	#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */
70	#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */
71	#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */
72	#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */
73	#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */
74	#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */
75	#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */
76	#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */
77	#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */
78	#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */
79	#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */
80	#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */
81	#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */
82	#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */
83	#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */
84	#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */
85	#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */
86	#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
87	#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
88
89	#define cp14_DBGDSCRext (MDSCR_EL1 * 2)
90	#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2)
91	#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2)
92	#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1)
93	#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2)
94	#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2)
95	#define cp14_DBGDCCINT (MDCCINT_EL1 * 2)
96
97	#define NR_COPRO_REGS (NR_SYS_REGS * 2)
98
99	#define ARM_EXCEPTION_IRQ 0	23	#define ARM_EXCEPTION_IRQ 0
100	#define ARM_EXCEPTION_TRAP 1	24	#define ARM_EXCEPTION_TRAP 1
101		25

arch/arm64/include/asm/kvm_emulate.h+0−1

@@ -26,7 +26,6 @@ · 25 unmodified lines
26		26
27	#include <asm/esr.h>	27	#include <asm/esr.h>
28	#include <asm/kvm_arm.h>	28	#include <asm/kvm_arm.h>
29	#include <asm/kvm_asm.h>
30	#include <asm/kvm_mmio.h>	29	#include <asm/kvm_mmio.h>
31	#include <asm/ptrace.h>	30	#include <asm/ptrace.h>
32	#include <asm/cputype.h>	31	#include <asm/cputype.h>

arch/arm64/include/asm/kvm_host.h+86−1

@@ -25,7 +25,6 @@ · 24 unmodified lines
25	#include <linux/types.h>	25	#include <linux/types.h>
26	#include <linux/kvm_types.h>	26	#include <linux/kvm_types.h>
27	#include <asm/kvm.h>	27	#include <asm/kvm.h>
28	#include <asm/kvm_asm.h>
29	#include <asm/kvm_mmio.h>	28	#include <asm/kvm_mmio.h>
30		29
31	#define __KVM_HAVE_ARCH_INTC_INITIALIZED	30	#define __KVM_HAVE_ARCH_INTC_INITIALIZED
@@ -85,6 +84,86 @@ struct kvm_vcpu_fault_info { · 53 unmodified lines
85	u64 hpfar_el2; /* Hyp IPA Fault Address Register */	84	u64 hpfar_el2; /* Hyp IPA Fault Address Register */
86	};	85	};
87		86
		87	/*
		88	* 0 is reserved as an invalid value.
		89	* Order should be kept in sync with the save/restore code.
		90	*/
		91	enum vcpu_sysreg {
		92	__INVALID_SYSREG__,
		93	MPIDR_EL1, /* MultiProcessor Affinity Register */
		94	CSSELR_EL1, /* Cache Size Selection Register */
		95	SCTLR_EL1, /* System Control Register */
		96	ACTLR_EL1, /* Auxiliary Control Register */
		97	CPACR_EL1, /* Coprocessor Access Control */
		98	TTBR0_EL1, /* Translation Table Base Register 0 */
		99	TTBR1_EL1, /* Translation Table Base Register 1 */
		100	TCR_EL1, /* Translation Control Register */
		101	ESR_EL1, /* Exception Syndrome Register */
		102	AFSR0_EL1, /* Auxilary Fault Status Register 0 */
		103	AFSR1_EL1, /* Auxilary Fault Status Register 1 */
		104	FAR_EL1, /* Fault Address Register */
		105	MAIR_EL1, /* Memory Attribute Indirection Register */
		106	VBAR_EL1, /* Vector Base Address Register */
		107	CONTEXTIDR_EL1, /* Context ID Register */
		108	TPIDR_EL0, /* Thread ID, User R/W */
		109	TPIDRRO_EL0, /* Thread ID, User R/O */
		110	TPIDR_EL1, /* Thread ID, Privileged */
		111	AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
		112	CNTKCTL_EL1, /* Timer Control Register (EL1) */
		113	PAR_EL1, /* Physical Address Register */
		114	MDSCR_EL1, /* Monitor Debug System Control Register */
		115	MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
		116
		117	/* 32bit specific registers. Keep them at the end of the range */
		118	DACR32_EL2, /* Domain Access Control Register */
		119	IFSR32_EL2, /* Instruction Fault Status Register */
		120	FPEXC32_EL2, /* Floating-Point Exception Control Register */
		121	DBGVCR32_EL2, /* Debug Vector Catch Register */
		122
		123	NR_SYS_REGS /* Nothing after this line! */
		124	};
		125
		126	/* 32bit mapping */
		127	#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
		128	#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */
		129	#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */
		130	#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */
		131	#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */
		132	#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */
		133	#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */
		134	#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */
		135	#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */
		136	#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */
		137	#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */
		138	#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */
		139	#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */
		140	#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */
		141	#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */
		142	#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */
		143	#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */
		144	#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */
		145	#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */
		146	#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */
		147	#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */
		148	#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */
		149	#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */
		150	#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */
		151	#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */
		152	#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */
		153	#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */
		154	#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
		155	#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
		156
		157	#define cp14_DBGDSCRext (MDSCR_EL1 * 2)
		158	#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2)
		159	#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2)
		160	#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1)
		161	#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2)
		162	#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2)
		163	#define cp14_DBGDCCINT (MDCCINT_EL1 * 2)
		164
		165	#define NR_COPRO_REGS (NR_SYS_REGS * 2)
		166
88	struct kvm_cpu_context {	167	struct kvm_cpu_context {
89	struct kvm_regs gp_regs;	168	struct kvm_regs gp_regs;
90	union {	169	union {
@@ -197,6 +276,12 @@ struct kvm_vcpu_stat { · 106 unmodified lines
197	u32 halt_successful_poll;	276	u32 halt_successful_poll;
198	u32 halt_attempted_poll;	277	u32 halt_attempted_poll;
199	u32 halt_wakeup;	278	u32 halt_wakeup;
		279	u32 hvc_exit_stat;
		280	u64 wfe_exit_stat;
		281	u64 wfi_exit_stat;
		282	u64 mmio_exit_user;
		283	u64 mmio_exit_kernel;
		284	u64 exits;
200	};	285	};
201		286
202	int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);	287	int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);

arch/arm64/include/asm/kvm_mmio.h+0−1

@@ -19,7 +19,6 @@ · 18 unmodified lines
19	#define __ARM64_KVM_MMIO_H__	19	#define __ARM64_KVM_MMIO_H__
20		20
21	#include <linux/kvm_host.h>	21	#include <linux/kvm_host.h>
22	#include <asm/kvm_asm.h>
23	#include <asm/kvm_arm.h>	22	#include <asm/kvm_arm.h>
24		23
25	/*	24	/*

arch/arm64/include/asm/kvm_mmu.h+8−1

 #include <asm/page.h>
 #include <asm/memory.h>
+#include <asm/cpufeature.h>
 /*
  * As we only have the TTBR0_EL2 register, we cannot express
 #define PTRS_PER_S2_PGD_SHIFT	(KVM_PHYS_SHIFT - PGDIR_SHIFT)
 #endif
 #define PTRS_PER_S2_PGD		(1 << PTRS_PER_S2_PGD_SHIFT)
-#define S2_PGD_ORDER		get_order(PTRS_PER_S2_PGD * sizeof(pgd_t))
 #define kvm_pgd_index(addr)	(((addr) >> PGDIR_SHIFT) & (PTRS_PER_S2_PGD - 1))
 	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
 }
+static inline unsigned int kvm_get_vmid_bits(void)
+{
+	int reg = read_system_reg(SYS_ID_AA64MMFR1_EL1);
+	return (cpuid_feature_extract_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
+}
 #endif /* __ASSEMBLY__ */
 #endif /* __ARM64_KVM_MMU_H__ */

arch/arm64/include/asm/sysreg.h+21−0

@@ -20,6 +20,8 @@ · 19 unmodified lines
20	#ifndef __ASM_SYSREG_H	20	#ifndef __ASM_SYSREG_H
21	#define __ASM_SYSREG_H	21	#define __ASM_SYSREG_H
22		22
		23	#include <linux/stringify.h>
		24
23	#include <asm/opcodes.h>	25	#include <asm/opcodes.h>
24		26
25	/*	27	/*
@@ -208,6 +210,8 @@ · 182 unmodified lines
208		210
209	#else	211	#else
210		212
		213	#include <linux/types.h>
		214
211	asm(	215	asm(
212	" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"	216	" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"
213	" .equ __reg_num_x\\num, \\num\n"	217	" .equ __reg_num_x\\num, \\num\n"
@@ -232,6 +236,23 @@ static inline void config_sctlr_el1(u32 clear, u32 set) · 18 unmodified lines
232	val \|= set;	236	val \|= set;
233	asm volatile("msr sctlr_el1, %0" : : "r" (val));	237	asm volatile("msr sctlr_el1, %0" : : "r" (val));
234	}	238	}
		239
		240	/*
		241	* Unlike read_cpuid, calls to read_sysreg are never expected to be
		242	* optimized away or replaced with synthetic values.
		243	*/
		244	#define read_sysreg(r) ({ \
		245	u64 __val; \
		246	asm volatile("mrs %0, " __stringify(r) : "=r" (__val)); \
		247	__val; \
		248	})
		249
		250	#define write_sysreg(v, r) do { \
		251	u64 __val = (u64)v; \
		252	asm volatile("msr " __stringify(r) ", %0" \
		253	: : "r" (__val)); \
		254	} while (0)
		255
235	#endif	256	#endif
236		257
237	#endif /* __ASM_SYSREG_H */	258	#endif /* __ASM_SYSREG_H */

arch/arm64/kernel/asm-offsets.c+1−39

@@ -109,49 +109,11 @@ int main(void) · 108 unmodified lines
109	DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));	109	DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));
110	DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs));	110	DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs));
111	DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));	111	DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
112	DEFINE(CPU_SP_EL1, offsetof(struct kvm_regs, sp_el1));	112	DEFINE(VCPU_FPEXC32_EL2, offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));
113	DEFINE(CPU_ELR_EL1, offsetof(struct kvm_regs, elr_el1));
114	DEFINE(CPU_SPSR, offsetof(struct kvm_regs, spsr));
115	DEFINE(CPU_SYSREGS, offsetof(struct kvm_cpu_context, sys_regs));
116	DEFINE(VCPU_ESR_EL2, offsetof(struct kvm_vcpu, arch.fault.esr_el2));	113	DEFINE(VCPU_ESR_EL2, offsetof(struct kvm_vcpu, arch.fault.esr_el2));
117	DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2));	114	DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2));
118	DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));	115	DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
119	DEFINE(VCPU_DEBUG_FLAGS, offsetof(struct kvm_vcpu, arch.debug_flags));
120	DEFINE(VCPU_DEBUG_PTR, offsetof(struct kvm_vcpu, arch.debug_ptr));
121	DEFINE(DEBUG_BCR, offsetof(struct kvm_guest_debug_arch, dbg_bcr));
122	DEFINE(DEBUG_BVR, offsetof(struct kvm_guest_debug_arch, dbg_bvr));
123	DEFINE(DEBUG_WCR, offsetof(struct kvm_guest_debug_arch, dbg_wcr));
124	DEFINE(DEBUG_WVR, offsetof(struct kvm_guest_debug_arch, dbg_wvr));
125	DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
126	DEFINE(VCPU_MDCR_EL2, offsetof(struct kvm_vcpu, arch.mdcr_el2));
127	DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines));
128	DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));	116	DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
129	DEFINE(VCPU_HOST_DEBUG_STATE, offsetof(struct kvm_vcpu, arch.host_debug_state));
130	DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
131	DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
132	DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
133	DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
134	DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm));
135	DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
136	DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr));
137	DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr));
138	DEFINE(VGIC_V2_CPU_MISR, offsetof(struct vgic_cpu, vgic_v2.vgic_misr));
139	DEFINE(VGIC_V2_CPU_EISR, offsetof(struct vgic_cpu, vgic_v2.vgic_eisr));
140	DEFINE(VGIC_V2_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr));
141	DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
142	DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
143	DEFINE(VGIC_V3_CPU_SRE, offsetof(struct vgic_cpu, vgic_v3.vgic_sre));
144	DEFINE(VGIC_V3_CPU_HCR, offsetof(struct vgic_cpu, vgic_v3.vgic_hcr));
145	DEFINE(VGIC_V3_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr));
146	DEFINE(VGIC_V3_CPU_MISR, offsetof(struct vgic_cpu, vgic_v3.vgic_misr));
147	DEFINE(VGIC_V3_CPU_EISR, offsetof(struct vgic_cpu, vgic_v3.vgic_eisr));
148	DEFINE(VGIC_V3_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v3.vgic_elrsr));
149	DEFINE(VGIC_V3_CPU_AP0R, offsetof(struct vgic_cpu, vgic_v3.vgic_ap0r));
150	DEFINE(VGIC_V3_CPU_AP1R, offsetof(struct vgic_cpu, vgic_v3.vgic_ap1r));
151	DEFINE(VGIC_V3_CPU_LR, offsetof(struct vgic_cpu, vgic_v3.vgic_lr));
152	DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
153	DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
154	DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
155	#endif	117	#endif
156	#ifdef CONFIG_CPU_PM	118	#ifdef CONFIG_CPU_PM
157	DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));	119	DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));

arch/arm64/kvm/Makefile+1−2

 ARM=../../../arch/arm/kvm
 obj-$(CONFIG_KVM_ARM_HOST) += kvm.o
+obj-$(CONFIG_KVM_ARM_HOST) += hyp/
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o $(KVM)/vfio.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o
-kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
-kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o

arch/arm64/kvm/guest.c+9−1

 #include <asm/cputype.h>
 #include <asm/uaccess.h>
 #include <asm/kvm.h>
-#include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include "trace.h"
+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
 struct kvm_stats_debugfs_item debugfs_entries[] = {
+	VCPU_STAT(hvc_exit_stat),
+	VCPU_STAT(wfe_exit_stat),
+	VCPU_STAT(wfi_exit_stat),
+	VCPU_STAT(mmio_exit_user),
+	VCPU_STAT(mmio_exit_kernel),
+	VCPU_STAT(exits),
 	{ NULL }
 };

arch/arm64/kvm/handle_exit.c+4−0

 #include <linux/kvm_host.h>
 #include <asm/esr.h>
+#include <asm/kvm_asm.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_mmu.h>
 	trace_kvm_hvc_arm64(*vcpu_pc(vcpu), vcpu_get_reg(vcpu, 0),
 			    kvm_vcpu_hvc_get_imm(vcpu));
+	vcpu->stat.hvc_exit_stat++;
 	ret = kvm_psci_call(vcpu);
 	if (ret < 0) {
 {
 	if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true);
+		vcpu->stat.wfe_exit_stat++;
 		kvm_vcpu_on_spin(vcpu);
 	} else {
 		trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false);
+		vcpu->stat.wfi_exit_stat++;
 		kvm_vcpu_block(vcpu);
 	}

arch/arm64/kvm/hyp-init.S+9−0

@@ -94,6 +94,15 @@ __do_hyp_init: · 93 unmodified lines
94	*/	94	*/
95	mrs x5, ID_AA64MMFR0_EL1	95	mrs x5, ID_AA64MMFR0_EL1
96	bfi x4, x5, #16, #3	96	bfi x4, x5, #16, #3
		97	/*
		98	* Read the VMIDBits bits from ID_AA64MMFR1_EL1 and set the VS bit in
		99	* VTCR_EL2.
		100	*/
		101	mrs x5, ID_AA64MMFR1_EL1
		102	ubfx x5, x5, #5, #1
		103	lsl x5, x5, #VTCR_EL2_VS
		104	orr x4, x4, x5
		105
97	msr vtcr_el2, x4	106	msr vtcr_el2, x4
98		107
99	mrs x4, mair_el1	108	mrs x4, mair_el1

arch/arm64/kvm/hyp.S+1−1080

@@ -17,910 +17,7 @@ · 16 unmodified lines
17		17
18	#include <linux/linkage.h>	18	#include <linux/linkage.h>
19		19
20	#include <asm/alternative.h>
21	#include <asm/asm-offsets.h>
22	#include <asm/assembler.h>	20	#include <asm/assembler.h>
23	#include <asm/cpufeature.h>
24	#include <asm/debug-monitors.h>
25	#include <asm/esr.h>
26	#include <asm/fpsimdmacros.h>
27	#include <asm/kvm.h>
28	#include <asm/kvm_arm.h>
29	#include <asm/kvm_asm.h>
30	#include <asm/kvm_mmu.h>
31	#include <asm/memory.h>
32
33	#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
34	#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
35	#define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
36	#define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
37
38	.text
39	.pushsection .hyp.text, "ax"
40	.align PAGE_SHIFT
41
42	.macro save_common_regs
43	// x2: base address for cpu context
44	// x3: tmp register
45
46	add x3, x2, #CPU_XREG_OFFSET(19)
47	stp x19, x20, [x3]
48	stp x21, x22, [x3, #16]
49	stp x23, x24, [x3, #32]
50	stp x25, x26, [x3, #48]
51	stp x27, x28, [x3, #64]
52	stp x29, lr, [x3, #80]
53
54	mrs x19, sp_el0
55	mrs x20, elr_el2 // pc before entering el2
56	mrs x21, spsr_el2 // pstate before entering el2
57
58	stp x19, x20, [x3, #96]
59	str x21, [x3, #112]
60
61	mrs x22, sp_el1
62	mrs x23, elr_el1
63	mrs x24, spsr_el1
64
65	str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
66	str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
67	str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
68	.endm
69
70	.macro restore_common_regs
71	// x2: base address for cpu context
72	// x3: tmp register
73
74	ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
75	ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
76	ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
77
78	msr sp_el1, x22
79	msr elr_el1, x23
80	msr spsr_el1, x24
81
82	add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
83	ldp x19, x20, [x3]
84	ldr x21, [x3, #16]
85
86	msr sp_el0, x19
87	msr elr_el2, x20 // pc on return from el2
88	msr spsr_el2, x21 // pstate on return from el2
89
90	add x3, x2, #CPU_XREG_OFFSET(19)
91	ldp x19, x20, [x3]
92	ldp x21, x22, [x3, #16]
93	ldp x23, x24, [x3, #32]
94	ldp x25, x26, [x3, #48]
95	ldp x27, x28, [x3, #64]
96	ldp x29, lr, [x3, #80]
97	.endm
98
99	.macro save_host_regs
100	save_common_regs
101	.endm
102
103	.macro restore_host_regs
104	restore_common_regs
105	.endm
106
107	.macro save_fpsimd
108	// x2: cpu context address
109	// x3, x4: tmp regs
110	add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
111	fpsimd_save x3, 4
112	.endm
113
114	.macro restore_fpsimd
115	// x2: cpu context address
116	// x3, x4: tmp regs
117	add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
118	fpsimd_restore x3, 4
119	.endm
120
121	.macro save_guest_regs
122	// x0 is the vcpu address
123	// x1 is the return code, do not corrupt!
124	// x2 is the cpu context
125	// x3 is a tmp register
126	// Guest's x0-x3 are on the stack
127
128	// Compute base to save registers
129	add x3, x2, #CPU_XREG_OFFSET(4)
130	stp x4, x5, [x3]
131	stp x6, x7, [x3, #16]
132	stp x8, x9, [x3, #32]
133	stp x10, x11, [x3, #48]
134	stp x12, x13, [x3, #64]
135	stp x14, x15, [x3, #80]
136	stp x16, x17, [x3, #96]
137	str x18, [x3, #112]
138
139	pop x6, x7 // x2, x3
140	pop x4, x5 // x0, x1
141
142	add x3, x2, #CPU_XREG_OFFSET(0)
143	stp x4, x5, [x3]
144	stp x6, x7, [x3, #16]
145
146	save_common_regs
147	.endm
148
149	.macro restore_guest_regs
150	// x0 is the vcpu address.
151	// x2 is the cpu context
152	// x3 is a tmp register
153
154	// Prepare x0-x3 for later restore
155	add x3, x2, #CPU_XREG_OFFSET(0)
156	ldp x4, x5, [x3]
157	ldp x6, x7, [x3, #16]
158	push x4, x5 // Push x0-x3 on the stack
159	push x6, x7
160
161	// x4-x18
162	ldp x4, x5, [x3, #32]
163	ldp x6, x7, [x3, #48]
164	ldp x8, x9, [x3, #64]
165	ldp x10, x11, [x3, #80]
166	ldp x12, x13, [x3, #96]
167	ldp x14, x15, [x3, #112]
168	ldp x16, x17, [x3, #128]
169	ldr x18, [x3, #144]
170
171	// x19-x29, lr, sp, elr, spsr*
172	restore_common_regs
173
174	// Last bits of the 64bit state
175	pop x2, x3
176	pop x0, x1
177
178	// Do not touch any register after this!
179	.endm
180
181	/*
182	* Macros to perform system register save/restore.
183	*
184	* Ordering here is absolutely critical, and must be kept consistent
185	* in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
186	* and in kvm_asm.h.
187	*
188	* In other words, don't touch any of these unless you know what
189	* you are doing.
190	*/
191	.macro save_sysregs
192	// x2: base address for cpu context
193	// x3: tmp register
194
195	add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
196
197	mrs x4, vmpidr_el2
198	mrs x5, csselr_el1
199	mrs x6, sctlr_el1
200	mrs x7, actlr_el1
201	mrs x8, cpacr_el1
202	mrs x9, ttbr0_el1
203	mrs x10, ttbr1_el1
204	mrs x11, tcr_el1
205	mrs x12, esr_el1
206	mrs x13, afsr0_el1
207	mrs x14, afsr1_el1
208	mrs x15, far_el1
209	mrs x16, mair_el1
210	mrs x17, vbar_el1
211	mrs x18, contextidr_el1
212	mrs x19, tpidr_el0
213	mrs x20, tpidrro_el0
214	mrs x21, tpidr_el1
215	mrs x22, amair_el1
216	mrs x23, cntkctl_el1
217	mrs x24, par_el1
218	mrs x25, mdscr_el1
219
220	stp x4, x5, [x3]
221	stp x6, x7, [x3, #16]
222	stp x8, x9, [x3, #32]
223	stp x10, x11, [x3, #48]
224	stp x12, x13, [x3, #64]
225	stp x14, x15, [x3, #80]
226	stp x16, x17, [x3, #96]
227	stp x18, x19, [x3, #112]
228	stp x20, x21, [x3, #128]
229	stp x22, x23, [x3, #144]
230	stp x24, x25, [x3, #160]
231	.endm
232
233	.macro save_debug type
234	// x4: pointer to register set
235	// x5: number of registers to skip
236	// x6..x22 trashed
237
238	adr x22, 1f
239	add x22, x22, x5, lsl #2
240	br x22
241	1:
242	mrs x21, \type\()15_el1
243	mrs x20, \type\()14_el1
244	mrs x19, \type\()13_el1
245	mrs x18, \type\()12_el1
246	mrs x17, \type\()11_el1
247	mrs x16, \type\()10_el1
248	mrs x15, \type\()9_el1
249	mrs x14, \type\()8_el1
250	mrs x13, \type\()7_el1
251	mrs x12, \type\()6_el1
252	mrs x11, \type\()5_el1
253	mrs x10, \type\()4_el1
254	mrs x9, \type\()3_el1
255	mrs x8, \type\()2_el1
256	mrs x7, \type\()1_el1
257	mrs x6, \type\()0_el1
258
259	adr x22, 1f
260	add x22, x22, x5, lsl #2
261	br x22
262	1:
263	str x21, [x4, #(15 * 8)]
264	str x20, [x4, #(14 * 8)]
265	str x19, [x4, #(13 * 8)]
266	str x18, [x4, #(12 * 8)]
267	str x17, [x4, #(11 * 8)]
268	str x16, [x4, #(10 * 8)]
269	str x15, [x4, #(9 * 8)]
270	str x14, [x4, #(8 * 8)]
271	str x13, [x4, #(7 * 8)]
272	str x12, [x4, #(6 * 8)]
273	str x11, [x4, #(5 * 8)]
274	str x10, [x4, #(4 * 8)]
275	str x9, [x4, #(3 * 8)]
276	str x8, [x4, #(2 * 8)]
277	str x7, [x4, #(1 * 8)]
278	str x6, [x4, #(0 * 8)]
279	.endm
280
281	.macro restore_sysregs
282	// x2: base address for cpu context
283	// x3: tmp register
284
285	add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
286
287	ldp x4, x5, [x3]
288	ldp x6, x7, [x3, #16]
289	ldp x8, x9, [x3, #32]
290	ldp x10, x11, [x3, #48]
291	ldp x12, x13, [x3, #64]
292	ldp x14, x15, [x3, #80]
293	ldp x16, x17, [x3, #96]
294	ldp x18, x19, [x3, #112]
295	ldp x20, x21, [x3, #128]
296	ldp x22, x23, [x3, #144]
297	ldp x24, x25, [x3, #160]
298
299	msr vmpidr_el2, x4
300	msr csselr_el1, x5
301	msr sctlr_el1, x6
302	msr actlr_el1, x7
303	msr cpacr_el1, x8
304	msr ttbr0_el1, x9
305	msr ttbr1_el1, x10
306	msr tcr_el1, x11
307	msr esr_el1, x12
308	msr afsr0_el1, x13
309	msr afsr1_el1, x14
310	msr far_el1, x15
311	msr mair_el1, x16
312	msr vbar_el1, x17
313	msr contextidr_el1, x18
314	msr tpidr_el0, x19
315	msr tpidrro_el0, x20
316	msr tpidr_el1, x21
317	msr amair_el1, x22
318	msr cntkctl_el1, x23
319	msr par_el1, x24
320	msr mdscr_el1, x25
321	.endm
322
323	.macro restore_debug type
324	// x4: pointer to register set
325	// x5: number of registers to skip
326	// x6..x22 trashed
327
328	adr x22, 1f
329	add x22, x22, x5, lsl #2
330	br x22
331	1:
332	ldr x21, [x4, #(15 * 8)]
333	ldr x20, [x4, #(14 * 8)]
334	ldr x19, [x4, #(13 * 8)]
335	ldr x18, [x4, #(12 * 8)]
336	ldr x17, [x4, #(11 * 8)]
337	ldr x16, [x4, #(10 * 8)]
338	ldr x15, [x4, #(9 * 8)]
339	ldr x14, [x4, #(8 * 8)]
340	ldr x13, [x4, #(7 * 8)]
341	ldr x12, [x4, #(6 * 8)]
342	ldr x11, [x4, #(5 * 8)]
343	ldr x10, [x4, #(4 * 8)]
344	ldr x9, [x4, #(3 * 8)]
345	ldr x8, [x4, #(2 * 8)]
346	ldr x7, [x4, #(1 * 8)]
347	ldr x6, [x4, #(0 * 8)]
348
349	adr x22, 1f
350	add x22, x22, x5, lsl #2
351	br x22
352	1:
353	msr \type\()15_el1, x21
354	msr \type\()14_el1, x20
355	msr \type\()13_el1, x19
356	msr \type\()12_el1, x18
357	msr \type\()11_el1, x17
358	msr \type\()10_el1, x16
359	msr \type\()9_el1, x15
360	msr \type\()8_el1, x14
361	msr \type\()7_el1, x13
362	msr \type\()6_el1, x12
363	msr \type\()5_el1, x11
364	msr \type\()4_el1, x10
365	msr \type\()3_el1, x9
366	msr \type\()2_el1, x8
367	msr \type\()1_el1, x7
368	msr \type\()0_el1, x6
369	.endm
370
371	.macro skip_32bit_state tmp, target
372	// Skip 32bit state if not needed
373	mrs \tmp, hcr_el2
374	tbnz \tmp, #HCR_RW_SHIFT, \target
375	.endm
376
377	.macro skip_tee_state tmp, target
378	// Skip ThumbEE state if not needed
379	mrs \tmp, id_pfr0_el1
380	tbz \tmp, #12, \target
381	.endm
382
383	.macro skip_debug_state tmp, target
384	ldr \tmp, [x0, #VCPU_DEBUG_FLAGS]
385	tbz \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target
386	.endm
387
388	/*
389	* Branch to target if CPTR_EL2.TFP bit is set (VFP/SIMD trapping enabled)
390	*/
391	.macro skip_fpsimd_state tmp, target
392	mrs \tmp, cptr_el2
393	tbnz \tmp, #CPTR_EL2_TFP_SHIFT, \target
394	.endm
395
396	.macro compute_debug_state target
397	// Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY
398	// is set, we do a full save/restore cycle and disable trapping.
399	add x25, x0, #VCPU_CONTEXT
400
401	// Check the state of MDSCR_EL1
402	ldr x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)]
403	and x26, x25, #DBG_MDSCR_KDE
404	and x25, x25, #DBG_MDSCR_MDE
405	adds xzr, x25, x26
406	b.eq 9998f // Nothing to see there
407
408	// If any interesting bits was set, we must set the flag
409	mov x26, #KVM_ARM64_DEBUG_DIRTY
410	str x26, [x0, #VCPU_DEBUG_FLAGS]
411	b 9999f // Don't skip restore
412
413	9998:
414	// Otherwise load the flags from memory in case we recently
415	// trapped
416	skip_debug_state x25, \target
417	9999:
418	.endm
419
420	.macro save_guest_32bit_state
421	skip_32bit_state x3, 1f
422
423	add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
424	mrs x4, spsr_abt
425	mrs x5, spsr_und
426	mrs x6, spsr_irq
427	mrs x7, spsr_fiq
428	stp x4, x5, [x3]
429	stp x6, x7, [x3, #16]
430
431	add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
432	mrs x4, dacr32_el2
433	mrs x5, ifsr32_el2
434	stp x4, x5, [x3]
435
436	skip_fpsimd_state x8, 2f
437	mrs x6, fpexc32_el2
438	str x6, [x3, #16]
439	2:
440	skip_debug_state x8, 1f
441	mrs x7, dbgvcr32_el2
442	str x7, [x3, #24]
443	1:
444	.endm
445
446	.macro restore_guest_32bit_state
447	skip_32bit_state x3, 1f
448
449	add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
450	ldp x4, x5, [x3]
451	ldp x6, x7, [x3, #16]
452	msr spsr_abt, x4
453	msr spsr_und, x5
454	msr spsr_irq, x6
455	msr spsr_fiq, x7
456
457	add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
458	ldp x4, x5, [x3]
459	msr dacr32_el2, x4
460	msr ifsr32_el2, x5
461
462	skip_debug_state x8, 1f
463	ldr x7, [x3, #24]
464	msr dbgvcr32_el2, x7
465	1:
466	.endm
467
468	.macro activate_traps
469	ldr x2, [x0, #VCPU_HCR_EL2]
470
471	/*
472	* We are about to set CPTR_EL2.TFP to trap all floating point
473	* register accesses to EL2, however, the ARM ARM clearly states that
474	* traps are only taken to EL2 if the operation would not otherwise
475	* trap to EL1. Therefore, always make sure that for 32-bit guests,
476	* we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
477	*/
478	tbnz x2, #HCR_RW_SHIFT, 99f // open code skip_32bit_state
479	mov x3, #(1 << 30)
480	msr fpexc32_el2, x3
481	isb
482	99:
483	msr hcr_el2, x2
484	mov x2, #CPTR_EL2_TTA
485	orr x2, x2, #CPTR_EL2_TFP
486	msr cptr_el2, x2
487
488	mov x2, #(1 << 15) // Trap CP15 Cr=15
489	msr hstr_el2, x2
490
491	// Monitor Debug Config - see kvm_arm_setup_debug()
492	ldr x2, [x0, #VCPU_MDCR_EL2]
493	msr mdcr_el2, x2
494	.endm
495
496	.macro deactivate_traps
497	mov x2, #HCR_RW
498	msr hcr_el2, x2
499	msr hstr_el2, xzr
500
501	mrs x2, mdcr_el2
502	and x2, x2, #MDCR_EL2_HPMN_MASK
503	msr mdcr_el2, x2
504	.endm
505
506	.macro activate_vm
507	ldr x1, [x0, #VCPU_KVM]
508	kern_hyp_va x1
509	ldr x2, [x1, #KVM_VTTBR]
510	msr vttbr_el2, x2
511	.endm
512
513	.macro deactivate_vm
514	msr vttbr_el2, xzr
515	.endm
516
517	/*
518	* Call into the vgic backend for state saving
519	*/
520	.macro save_vgic_state
521	alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
522	bl __save_vgic_v2_state
523	alternative_else
524	bl __save_vgic_v3_state
525	alternative_endif
526	mrs x24, hcr_el2
527	mov x25, #HCR_INT_OVERRIDE
528	neg x25, x25
529	and x24, x24, x25
530	msr hcr_el2, x24
531	.endm
532
533	/*
534	* Call into the vgic backend for state restoring
535	*/
536	.macro restore_vgic_state
537	mrs x24, hcr_el2
538	ldr x25, [x0, #VCPU_IRQ_LINES]
539	orr x24, x24, #HCR_INT_OVERRIDE
540	orr x24, x24, x25
541	msr hcr_el2, x24
542	alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
543	bl __restore_vgic_v2_state
544	alternative_else
545	bl __restore_vgic_v3_state
546	alternative_endif
547	.endm
548
549	.macro save_timer_state
550	// x0: vcpu pointer
551	ldr x2, [x0, #VCPU_KVM]
552	kern_hyp_va x2
553	ldr w3, [x2, #KVM_TIMER_ENABLED]
554	cbz w3, 1f
555
556	mrs x3, cntv_ctl_el0
557	and x3, x3, #3
558	str w3, [x0, #VCPU_TIMER_CNTV_CTL]
559
560	isb
561
562	mrs x3, cntv_cval_el0
563	str x3, [x0, #VCPU_TIMER_CNTV_CVAL]
564
565	1:
566	// Disable the virtual timer
567	msr cntv_ctl_el0, xzr
568
569	// Allow physical timer/counter access for the host
570	mrs x2, cnthctl_el2
571	orr x2, x2, #3
572	msr cnthctl_el2, x2
573
574	// Clear cntvoff for the host
575	msr cntvoff_el2, xzr
576	.endm
577
578	.macro restore_timer_state
579	// x0: vcpu pointer
580	// Disallow physical timer access for the guest
581	// Physical counter access is allowed
582	mrs x2, cnthctl_el2
583	orr x2, x2, #1
584	bic x2, x2, #2
585	msr cnthctl_el2, x2
586
587	ldr x2, [x0, #VCPU_KVM]
588	kern_hyp_va x2
589	ldr w3, [x2, #KVM_TIMER_ENABLED]
590	cbz w3, 1f
591
592	ldr x3, [x2, #KVM_TIMER_CNTVOFF]
593	msr cntvoff_el2, x3
594	ldr x2, [x0, #VCPU_TIMER_CNTV_CVAL]
595	msr cntv_cval_el0, x2
596	isb
597
598	ldr w2, [x0, #VCPU_TIMER_CNTV_CTL]
599	and x2, x2, #3
600	msr cntv_ctl_el0, x2
601	1:
602	.endm
603
604	__save_sysregs:
605	save_sysregs
606	ret
607
608	__restore_sysregs:
609	restore_sysregs
610	ret
611
612	/* Save debug state */
613	__save_debug:
614	// x2: ptr to CPU context
615	// x3: ptr to debug reg struct
616	// x4/x5/x6-22/x24-26: trashed
617
618	mrs x26, id_aa64dfr0_el1
619	ubfx x24, x26, #12, #4 // Extract BRPs
620	ubfx x25, x26, #20, #4 // Extract WRPs
621	mov w26, #15
622	sub w24, w26, w24 // How many BPs to skip
623	sub w25, w26, w25 // How many WPs to skip
624
625	mov x5, x24
626	add x4, x3, #DEBUG_BCR
627	save_debug dbgbcr
628	add x4, x3, #DEBUG_BVR
629	save_debug dbgbvr
630
631	mov x5, x25
632	add x4, x3, #DEBUG_WCR
633	save_debug dbgwcr
634	add x4, x3, #DEBUG_WVR
635	save_debug dbgwvr
636
637	mrs x21, mdccint_el1
638	str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
639	ret
640
641	/* Restore debug state */
642	__restore_debug:
643	// x2: ptr to CPU context
644	// x3: ptr to debug reg struct
645	// x4/x5/x6-22/x24-26: trashed
646
647	mrs x26, id_aa64dfr0_el1
648	ubfx x24, x26, #12, #4 // Extract BRPs
649	ubfx x25, x26, #20, #4 // Extract WRPs
650	mov w26, #15
651	sub w24, w26, w24 // How many BPs to skip
652	sub w25, w26, w25 // How many WPs to skip
653
654	mov x5, x24
655	add x4, x3, #DEBUG_BCR
656	restore_debug dbgbcr
657	add x4, x3, #DEBUG_BVR
658	restore_debug dbgbvr
659
660	mov x5, x25
661	add x4, x3, #DEBUG_WCR
662	restore_debug dbgwcr
663	add x4, x3, #DEBUG_WVR
664	restore_debug dbgwvr
665
666	ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
667	msr mdccint_el1, x21
668
669	ret
670
671	__save_fpsimd:
672	skip_fpsimd_state x3, 1f
673	save_fpsimd
674	1: ret
675
676	__restore_fpsimd:
677	skip_fpsimd_state x3, 1f
678	restore_fpsimd
679	1: ret
680
681	switch_to_guest_fpsimd:
682	push x4, lr
683
684	mrs x2, cptr_el2
685	bic x2, x2, #CPTR_EL2_TFP
686	msr cptr_el2, x2
687	isb
688
689	mrs x0, tpidr_el2
690
691	ldr x2, [x0, #VCPU_HOST_CONTEXT]
692	kern_hyp_va x2
693	bl __save_fpsimd
694
695	add x2, x0, #VCPU_CONTEXT
696	bl __restore_fpsimd
697
698	skip_32bit_state x3, 1f
699	ldr x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]
700	msr fpexc32_el2, x4
701	1:
702	pop x4, lr
703	pop x2, x3
704	pop x0, x1
705
706	eret
707
708	/*
709	* u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
710	*
711	* This is the world switch. The first half of the function
712	* deals with entering the guest, and anything from __kvm_vcpu_return
713	* to the end of the function deals with reentering the host.
714	* On the enter path, only x0 (vcpu pointer) must be preserved until
715	* the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
716	* code) must both be preserved until the epilogue.
717	* In both cases, x2 points to the CPU context we're saving/restoring from/to.
718	*/
719	ENTRY(__kvm_vcpu_run)
720	kern_hyp_va x0
721	msr tpidr_el2, x0 // Save the vcpu register
722
723	// Host context
724	ldr x2, [x0, #VCPU_HOST_CONTEXT]
725	kern_hyp_va x2
726
727	save_host_regs
728	bl __save_sysregs
729
730	compute_debug_state 1f
731	add x3, x0, #VCPU_HOST_DEBUG_STATE
732	bl __save_debug
733	1:
734	activate_traps
735	activate_vm
736
737	restore_vgic_state
738	restore_timer_state
739
740	// Guest context
741	add x2, x0, #VCPU_CONTEXT
742
743	// We must restore the 32-bit state before the sysregs, thanks
744	// to Cortex-A57 erratum #852523.
745	restore_guest_32bit_state
746	bl __restore_sysregs
747
748	skip_debug_state x3, 1f
749	ldr x3, [x0, #VCPU_DEBUG_PTR]
750	kern_hyp_va x3
751	bl __restore_debug
752	1:
753	restore_guest_regs
754
755	// That's it, no more messing around.
756	eret
757
758	__kvm_vcpu_return:
759	// Assume x0 is the vcpu pointer, x1 the return code
760	// Guest's x0-x3 are on the stack
761
762	// Guest context
763	add x2, x0, #VCPU_CONTEXT
764
765	save_guest_regs
766	bl __save_fpsimd
767	bl __save_sysregs
768
769	skip_debug_state x3, 1f
770	ldr x3, [x0, #VCPU_DEBUG_PTR]
771	kern_hyp_va x3
772	bl __save_debug
773	1:
774	save_guest_32bit_state
775
776	save_timer_state
777	save_vgic_state
778
779	deactivate_traps
780	deactivate_vm
781
782	// Host context
783	ldr x2, [x0, #VCPU_HOST_CONTEXT]
784	kern_hyp_va x2
785
786	bl __restore_sysregs
787	bl __restore_fpsimd
788	/* Clear FPSIMD and Trace trapping */
789	msr cptr_el2, xzr
790
791	skip_debug_state x3, 1f
792	// Clear the dirty flag for the next run, as all the state has
793	// already been saved. Note that we nuke the whole 64bit word.
794	// If we ever add more flags, we'll have to be more careful...
795	str xzr, [x0, #VCPU_DEBUG_FLAGS]
796	add x3, x0, #VCPU_HOST_DEBUG_STATE
797	bl __restore_debug
798	1:
799	restore_host_regs
800
801	mov x0, x1
802	ret
803	END(__kvm_vcpu_run)
804
805	// void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
806	ENTRY(__kvm_tlb_flush_vmid_ipa)
807	dsb ishst
808
809	kern_hyp_va x0
810	ldr x2, [x0, #KVM_VTTBR]
811	msr vttbr_el2, x2
812	isb
813
814	/*
815	* We could do so much better if we had the VA as well.
816	* Instead, we invalidate Stage-2 for this IPA, and the
817	* whole of Stage-1. Weep...
818	*/
819	lsr x1, x1, #12
820	tlbi ipas2e1is, x1
821	/*
822	* We have to ensure completion of the invalidation at Stage-2,
823	* since a table walk on another CPU could refill a TLB with a
824	* complete (S1 + S2) walk based on the old Stage-2 mapping if
825	* the Stage-1 invalidation happened first.
826	*/
827	dsb ish
828	tlbi vmalle1is
829	dsb ish
830	isb
831
832	msr vttbr_el2, xzr
833	ret
834	ENDPROC(__kvm_tlb_flush_vmid_ipa)
835
836	/**
837	* void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
838	* @struct kvm *kvm - pointer to kvm structure
839	*
840	* Invalidates all Stage 1 and 2 TLB entries for current VMID.
841	*/
842	ENTRY(__kvm_tlb_flush_vmid)
843	dsb ishst
844
845	kern_hyp_va x0
846	ldr x2, [x0, #KVM_VTTBR]
847	msr vttbr_el2, x2
848	isb
849
850	tlbi vmalls12e1is
851	dsb ish
852	isb
853
854	msr vttbr_el2, xzr
855	ret
856	ENDPROC(__kvm_tlb_flush_vmid)
857
858	ENTRY(__kvm_flush_vm_context)
859	dsb ishst
860	tlbi alle1is
861	ic ialluis
862	dsb ish
863	ret
864	ENDPROC(__kvm_flush_vm_context)
865
866	__kvm_hyp_panic:
867	// Stash PAR_EL1 before corrupting it in __restore_sysregs
868	mrs x0, par_el1
869	push x0, xzr
870
871	// Guess the context by looking at VTTBR:
872	// If zero, then we're already a host.
873	// Otherwise restore a minimal host context before panicing.
874	mrs x0, vttbr_el2
875	cbz x0, 1f
876
877	mrs x0, tpidr_el2
878
879	deactivate_traps
880	deactivate_vm
881
882	ldr x2, [x0, #VCPU_HOST_CONTEXT]
883	kern_hyp_va x2
884
885	bl __restore_sysregs
886
887	/*
888	* Make sure we have a valid host stack, and don't leave junk in the
889	* frame pointer that will give us a misleading host stack unwinding.
890	*/
891	ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
892	msr sp_el1, x22
893	mov x29, xzr
894
895	1: adr x0, __hyp_panic_str
896	adr x1, 2f
897	ldp x2, x3, [x1]
898	sub x0, x0, x2
899	add x0, x0, x3
900	mrs x1, spsr_el2
901	mrs x2, elr_el2
902	mrs x3, esr_el2
903	mrs x4, far_el2
904	mrs x5, hpfar_el2
905	pop x6, xzr // active context PAR_EL1
906	mrs x7, tpidr_el2
907
908	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
909	PSR_MODE_EL1h)
910	msr spsr_el2, lr
911	ldr lr, =panic
912	msr elr_el2, lr
913	eret
914
915	.align 3
916	2: .quad HYP_PAGE_OFFSET
917	.quad PAGE_OFFSET
918	ENDPROC(__kvm_hyp_panic)
919
920	__hyp_panic_str:
921	.ascii "HYP panic:\nPS:%08x PC:%016x ESR:%08x\nFAR:%016x HPFAR:%016x PAR:%016x\nVCPU:%p\n\0"
922
923	.align 2
924		21
925	/*	22	/*
926	* u64 kvm_call_hyp(void *hypfn, ...);	23	* u64 kvm_call_hyp(void *hypfn, ...);
@@ -934,7 +31,7 @@ __hyp_panic_str: · 7 unmodified lines
934	* passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the	31	* passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the
935	* function pointer can be passed). The function being called must be mapped	32	* function pointer can be passed). The function being called must be mapped
936	* in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are	33	* in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
937	* passed in r0 and r1.	34	* passed in x0.
938	*	35	*
939	* A function pointer with a value of 0 has a special meaning, and is	36	* A function pointer with a value of 0 has a special meaning, and is
940	* used to implement __hyp_get_vectors in the same way as in	37	* used to implement __hyp_get_vectors in the same way as in
@@ -944,179 +41,3 @@ ENTRY(kvm_call_hyp) · 3 unmodified lines
944	hvc #0	41	hvc #0
945	ret	42	ret
946	ENDPROC(kvm_call_hyp)	43	ENDPROC(kvm_call_hyp)
947
948	.macro invalid_vector label, target
949	.align 2
950	\label:
951	b \target
952	ENDPROC(\label)
953	.endm
954
955	/* None of these should ever happen */
956	invalid_vector el2t_sync_invalid, __kvm_hyp_panic
957	invalid_vector el2t_irq_invalid, __kvm_hyp_panic
958	invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
959	invalid_vector el2t_error_invalid, __kvm_hyp_panic
960	invalid_vector el2h_sync_invalid, __kvm_hyp_panic
961	invalid_vector el2h_irq_invalid, __kvm_hyp_panic
962	invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
963	invalid_vector el2h_error_invalid, __kvm_hyp_panic
964	invalid_vector el1_sync_invalid, __kvm_hyp_panic
965	invalid_vector el1_irq_invalid, __kvm_hyp_panic
966	invalid_vector el1_fiq_invalid, __kvm_hyp_panic
967	invalid_vector el1_error_invalid, __kvm_hyp_panic
968
969	el1_sync: // Guest trapped into EL2
970	push x0, x1
971	push x2, x3
972
973	mrs x1, esr_el2
974	lsr x2, x1, #ESR_ELx_EC_SHIFT
975
976	cmp x2, #ESR_ELx_EC_HVC64
977	b.ne el1_trap
978
979	mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
980	cbnz x3, el1_trap // called HVC
981
982	/* Here, we're pretty sure the host called HVC. */
983	pop x2, x3
984	pop x0, x1
985
986	/* Check for __hyp_get_vectors */
987	cbnz x0, 1f
988	mrs x0, vbar_el2
989	b 2f
990
991	1: push lr, xzr
992
993	/*
994	* Compute the function address in EL2, and shuffle the parameters.
995	*/
996	kern_hyp_va x0
997	mov lr, x0
998	mov x0, x1
999	mov x1, x2
1000	mov x2, x3
1001	blr lr
1002
1003	pop lr, xzr
1004	2: eret
1005
1006	el1_trap:
1007	/*
1008	* x1: ESR
1009	* x2: ESR_EC
1010	*/
1011
1012	/* Guest accessed VFP/SIMD registers, save host, restore Guest */
1013	cmp x2, #ESR_ELx_EC_FP_ASIMD
1014	b.eq switch_to_guest_fpsimd
1015
1016	cmp x2, #ESR_ELx_EC_DABT_LOW
1017	mov x0, #ESR_ELx_EC_IABT_LOW
1018	ccmp x2, x0, #4, ne
1019	b.ne 1f // Not an abort we care about
1020
1021	/* This is an abort. Check for permission fault */
1022	alternative_if_not ARM64_WORKAROUND_834220
1023	and x2, x1, #ESR_ELx_FSC_TYPE
1024	cmp x2, #FSC_PERM
1025	b.ne 1f // Not a permission fault
1026	alternative_else
1027	nop // Use the permission fault path to
1028	nop // check for a valid S1 translation,
1029	nop // regardless of the ESR value.
1030	alternative_endif
1031
1032	/*
1033	* Check for Stage-1 page table walk, which is guaranteed
1034	* to give a valid HPFAR_EL2.
1035	*/
1036	tbnz x1, #7, 1f // S1PTW is set
1037
1038	/* Preserve PAR_EL1 */
1039	mrs x3, par_el1
1040	push x3, xzr
1041
1042	/*
1043	* Permission fault, HPFAR_EL2 is invalid.
1044	* Resolve the IPA the hard way using the guest VA.
1045	* Stage-1 translation already validated the memory access rights.
1046	* As such, we can use the EL1 translation regime, and don't have
1047	* to distinguish between EL0 and EL1 access.
1048	*/
1049	mrs x2, far_el2
1050	at s1e1r, x2
1051	isb
1052
1053	/* Read result */
1054	mrs x3, par_el1
1055	pop x0, xzr // Restore PAR_EL1 from the stack
1056	msr par_el1, x0
1057	tbnz x3, #0, 3f // Bail out if we failed the translation
1058	ubfx x3, x3, #12, #36 // Extract IPA
1059	lsl x3, x3, #4 // and present it like HPFAR
1060	b 2f
1061
1062	1: mrs x3, hpfar_el2
1063	mrs x2, far_el2
1064
1065	2: mrs x0, tpidr_el2
1066	str w1, [x0, #VCPU_ESR_EL2]
1067	str x2, [x0, #VCPU_FAR_EL2]
1068	str x3, [x0, #VCPU_HPFAR_EL2]
1069
1070	mov x1, #ARM_EXCEPTION_TRAP
1071	b __kvm_vcpu_return
1072
1073	/*
1074	* Translation failed. Just return to the guest and
1075	* let it fault again. Another CPU is probably playing
1076	* behind our back.
1077	*/
1078	3: pop x2, x3
1079	pop x0, x1
1080
1081	eret
1082
1083	el1_irq:
1084	push x0, x1
1085	push x2, x3
1086	mrs x0, tpidr_el2
1087	mov x1, #ARM_EXCEPTION_IRQ
1088	b __kvm_vcpu_return
1089
1090	.ltorg
1091
1092	.align 11
1093
1094	ENTRY(__kvm_hyp_vector)
1095	ventry el2t_sync_invalid // Synchronous EL2t
1096	ventry el2t_irq_invalid // IRQ EL2t
1097	ventry el2t_fiq_invalid // FIQ EL2t
1098	ventry el2t_error_invalid // Error EL2t
1099
1100	ventry el2h_sync_invalid // Synchronous EL2h
1101	ventry el2h_irq_invalid // IRQ EL2h
1102	ventry el2h_fiq_invalid // FIQ EL2h
1103	ventry el2h_error_invalid // Error EL2h
1104
1105	ventry el1_sync // Synchronous 64-bit EL1
1106	ventry el1_irq // IRQ 64-bit EL1
1107	ventry el1_fiq_invalid // FIQ 64-bit EL1
1108	ventry el1_error_invalid // Error 64-bit EL1
1109
1110	ventry el1_sync // Synchronous 32-bit EL1
1111	ventry el1_irq // IRQ 32-bit EL1
1112	ventry el1_fiq_invalid // FIQ 32-bit EL1
1113	ventry el1_error_invalid // Error 32-bit EL1
1114	ENDPROC(__kvm_hyp_vector)
1115
1116
1117	ENTRY(__kvm_get_mdcr_el2)
1118	mrs x0, mdcr_el2
1119	ret
1120	ENDPROC(__kvm_get_mdcr_el2)
1121
1122	.popsection

arch/arm64/kvm/hyp/Makefile+14−0

@@ -0,0 +1,14 @@
	1	#
	2	# Makefile for Kernel-based Virtual Machine module, HYP part
	3	#
	4
	5	obj-$(CONFIG_KVM_ARM_HOST) += vgic-v2-sr.o
	6	obj-$(CONFIG_KVM_ARM_HOST) += vgic-v3-sr.o
	7	obj-$(CONFIG_KVM_ARM_HOST) += timer-sr.o
	8	obj-$(CONFIG_KVM_ARM_HOST) += sysreg-sr.o
	9	obj-$(CONFIG_KVM_ARM_HOST) += debug-sr.o
	10	obj-$(CONFIG_KVM_ARM_HOST) += entry.o
	11	obj-$(CONFIG_KVM_ARM_HOST) += switch.o
	12	obj-$(CONFIG_KVM_ARM_HOST) += fpsimd.o
	13	obj-$(CONFIG_KVM_ARM_HOST) += tlb.o
	14	obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o

arch/arm64/kvm/hyp/debug-sr.c+140−0

@@ -0,0 +1,140 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/compiler.h>
	19	#include <linux/kvm_host.h>
	20
	21	#include <asm/kvm_asm.h>
	22	#include <asm/kvm_mmu.h>
	23
	24	#include "hyp.h"
	25
	26	#define read_debug(r,n) read_sysreg(r##n##_el1)
	27	#define write_debug(v,r,n) write_sysreg(v, r##n##_el1)
	28
	29	#define save_debug(ptr,reg,nr) \
	30	switch (nr) { \
	31	case 15: ptr[15] = read_debug(reg, 15); \
	32	case 14: ptr[14] = read_debug(reg, 14); \
	33	case 13: ptr[13] = read_debug(reg, 13); \
	34	case 12: ptr[12] = read_debug(reg, 12); \
	35	case 11: ptr[11] = read_debug(reg, 11); \
	36	case 10: ptr[10] = read_debug(reg, 10); \
	37	case 9: ptr[9] = read_debug(reg, 9); \
	38	case 8: ptr[8] = read_debug(reg, 8); \
	39	case 7: ptr[7] = read_debug(reg, 7); \
	40	case 6: ptr[6] = read_debug(reg, 6); \
	41	case 5: ptr[5] = read_debug(reg, 5); \
	42	case 4: ptr[4] = read_debug(reg, 4); \
	43	case 3: ptr[3] = read_debug(reg, 3); \
	44	case 2: ptr[2] = read_debug(reg, 2); \
	45	case 1: ptr[1] = read_debug(reg, 1); \
	46	default: ptr[0] = read_debug(reg, 0); \
	47	}
	48
	49	#define restore_debug(ptr,reg,nr) \
	50	switch (nr) { \
	51	case 15: write_debug(ptr[15], reg, 15); \
	52	case 14: write_debug(ptr[14], reg, 14); \
	53	case 13: write_debug(ptr[13], reg, 13); \
	54	case 12: write_debug(ptr[12], reg, 12); \
	55	case 11: write_debug(ptr[11], reg, 11); \
	56	case 10: write_debug(ptr[10], reg, 10); \
	57	case 9: write_debug(ptr[9], reg, 9); \
	58	case 8: write_debug(ptr[8], reg, 8); \
	59	case 7: write_debug(ptr[7], reg, 7); \
	60	case 6: write_debug(ptr[6], reg, 6); \
	61	case 5: write_debug(ptr[5], reg, 5); \
	62	case 4: write_debug(ptr[4], reg, 4); \
	63	case 3: write_debug(ptr[3], reg, 3); \
	64	case 2: write_debug(ptr[2], reg, 2); \
	65	case 1: write_debug(ptr[1], reg, 1); \
	66	default: write_debug(ptr[0], reg, 0); \
	67	}
	68
	69	void __hyp_text __debug_save_state(struct kvm_vcpu *vcpu,
	70	struct kvm_guest_debug_arch *dbg,
	71	struct kvm_cpu_context *ctxt)
	72	{
	73	u64 aa64dfr0;
	74	int brps, wrps;
	75
	76	if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
	77	return;
	78
	79	aa64dfr0 = read_sysreg(id_aa64dfr0_el1);
	80	brps = (aa64dfr0 >> 12) & 0xf;
	81	wrps = (aa64dfr0 >> 20) & 0xf;
	82
	83	save_debug(dbg->dbg_bcr, dbgbcr, brps);
	84	save_debug(dbg->dbg_bvr, dbgbvr, brps);
	85	save_debug(dbg->dbg_wcr, dbgwcr, wrps);
	86	save_debug(dbg->dbg_wvr, dbgwvr, wrps);
	87
	88	ctxt->sys_regs[MDCCINT_EL1] = read_sysreg(mdccint_el1);
	89	}
	90
	91	void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
	92	struct kvm_guest_debug_arch *dbg,
	93	struct kvm_cpu_context *ctxt)
	94	{
	95	u64 aa64dfr0;
	96	int brps, wrps;
	97
	98	if (!(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY))
	99	return;
	100
	101	aa64dfr0 = read_sysreg(id_aa64dfr0_el1);
	102
	103	brps = (aa64dfr0 >> 12) & 0xf;
	104	wrps = (aa64dfr0 >> 20) & 0xf;
	105
	106	restore_debug(dbg->dbg_bcr, dbgbcr, brps);
	107	restore_debug(dbg->dbg_bvr, dbgbvr, brps);
	108	restore_debug(dbg->dbg_wcr, dbgwcr, wrps);
	109	restore_debug(dbg->dbg_wvr, dbgwvr, wrps);
	110
	111	write_sysreg(ctxt->sys_regs[MDCCINT_EL1], mdccint_el1);
	112	}
	113
	114	void __hyp_text __debug_cond_save_host_state(struct kvm_vcpu *vcpu)
	115	{
	116	/* If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY is set, perform
	117	* a full save/restore cycle. */
	118	if ((vcpu->arch.ctxt.sys_regs[MDSCR_EL1] & DBG_MDSCR_KDE) \|\|
	119	(vcpu->arch.ctxt.sys_regs[MDSCR_EL1] & DBG_MDSCR_MDE))
	120	vcpu->arch.debug_flags \|= KVM_ARM64_DEBUG_DIRTY;
	121
	122	__debug_save_state(vcpu, &vcpu->arch.host_debug_state,
	123	kern_hyp_va(vcpu->arch.host_cpu_context));
	124	}
	125
	126	void __hyp_text __debug_cond_restore_host_state(struct kvm_vcpu *vcpu)
	127	{
	128	__debug_restore_state(vcpu, &vcpu->arch.host_debug_state,
	129	kern_hyp_va(vcpu->arch.host_cpu_context));
	130
	131	if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
	132	vcpu->arch.debug_flags &= ~KVM_ARM64_DEBUG_DIRTY;
	133	}
	134
	135	static u32 __hyp_text __debug_read_mdcr_el2(void)
	136	{
	137	return read_sysreg(mdcr_el2);
	138	}
	139
	140	__alias(__debug_read_mdcr_el2) u32 __kvm_get_mdcr_el2(void);

arch/arm64/kvm/hyp/entry.S+160−0

@@ -0,0 +1,160 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/linkage.h>
	19
	20	#include <asm/asm-offsets.h>
	21	#include <asm/assembler.h>
	22	#include <asm/fpsimdmacros.h>
	23	#include <asm/kvm.h>
	24	#include <asm/kvm_arm.h>
	25	#include <asm/kvm_asm.h>
	26	#include <asm/kvm_mmu.h>
	27
	28	#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
	29	#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
	30
	31	.text
	32	.pushsection .hyp.text, "ax"
	33
	34	.macro save_callee_saved_regs ctxt
	35	stp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	36	stp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	37	stp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	38	stp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	39	stp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	40	stp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
	41	.endm
	42
	43	.macro restore_callee_saved_regs ctxt
	44	ldp x19, x20, [\ctxt, #CPU_XREG_OFFSET(19)]
	45	ldp x21, x22, [\ctxt, #CPU_XREG_OFFSET(21)]
	46	ldp x23, x24, [\ctxt, #CPU_XREG_OFFSET(23)]
	47	ldp x25, x26, [\ctxt, #CPU_XREG_OFFSET(25)]
	48	ldp x27, x28, [\ctxt, #CPU_XREG_OFFSET(27)]
	49	ldp x29, lr, [\ctxt, #CPU_XREG_OFFSET(29)]
	50	.endm
	51
	52	/*
	53	* u64 __guest_enter(struct kvm_vcpu *vcpu,
	54	* struct kvm_cpu_context *host_ctxt);
	55	*/
	56	ENTRY(__guest_enter)
	57	// x0: vcpu
	58	// x1: host/guest context
	59	// x2-x18: clobbered by macros
	60
	61	// Store the host regs
	62	save_callee_saved_regs x1
	63
	64	// Preserve vcpu & host_ctxt for use at exit time
	65	stp x0, x1, [sp, #-16]!
	66
	67	add x1, x0, #VCPU_CONTEXT
	68
	69	// Prepare x0-x1 for later restore by pushing them onto the stack
	70	ldp x2, x3, [x1, #CPU_XREG_OFFSET(0)]
	71	stp x2, x3, [sp, #-16]!
	72
	73	// x2-x18
	74	ldp x2, x3, [x1, #CPU_XREG_OFFSET(2)]
	75	ldp x4, x5, [x1, #CPU_XREG_OFFSET(4)]
	76	ldp x6, x7, [x1, #CPU_XREG_OFFSET(6)]
	77	ldp x8, x9, [x1, #CPU_XREG_OFFSET(8)]
	78	ldp x10, x11, [x1, #CPU_XREG_OFFSET(10)]
	79	ldp x12, x13, [x1, #CPU_XREG_OFFSET(12)]
	80	ldp x14, x15, [x1, #CPU_XREG_OFFSET(14)]
	81	ldp x16, x17, [x1, #CPU_XREG_OFFSET(16)]
	82	ldr x18, [x1, #CPU_XREG_OFFSET(18)]
	83
	84	// x19-x29, lr
	85	restore_callee_saved_regs x1
	86
	87	// Last bits of the 64bit state
	88	ldp x0, x1, [sp], #16
	89
	90	// Do not touch any register after this!
	91	eret
	92	ENDPROC(__guest_enter)
	93
	94	ENTRY(__guest_exit)
	95	// x0: vcpu
	96	// x1: return code
	97	// x2-x3: free
	98	// x4-x29,lr: vcpu regs
	99	// vcpu x0-x3 on the stack
	100
	101	add x2, x0, #VCPU_CONTEXT
	102
	103	stp x4, x5, [x2, #CPU_XREG_OFFSET(4)]
	104	stp x6, x7, [x2, #CPU_XREG_OFFSET(6)]
	105	stp x8, x9, [x2, #CPU_XREG_OFFSET(8)]
	106	stp x10, x11, [x2, #CPU_XREG_OFFSET(10)]
	107	stp x12, x13, [x2, #CPU_XREG_OFFSET(12)]
	108	stp x14, x15, [x2, #CPU_XREG_OFFSET(14)]
	109	stp x16, x17, [x2, #CPU_XREG_OFFSET(16)]
	110	str x18, [x2, #CPU_XREG_OFFSET(18)]
	111
	112	ldp x6, x7, [sp], #16 // x2, x3
	113	ldp x4, x5, [sp], #16 // x0, x1
	114
	115	stp x4, x5, [x2, #CPU_XREG_OFFSET(0)]
	116	stp x6, x7, [x2, #CPU_XREG_OFFSET(2)]
	117
	118	save_callee_saved_regs x2
	119
	120	// Restore vcpu & host_ctxt from the stack
	121	// (preserving return code in x1)
	122	ldp x0, x2, [sp], #16
	123	// Now restore the host regs
	124	restore_callee_saved_regs x2
	125
	126	mov x0, x1
	127	ret
	128	ENDPROC(__guest_exit)
	129
	130	ENTRY(__fpsimd_guest_restore)
	131	stp x4, lr, [sp, #-16]!
	132
	133	mrs x2, cptr_el2
	134	bic x2, x2, #CPTR_EL2_TFP
	135	msr cptr_el2, x2
	136	isb
	137
	138	mrs x3, tpidr_el2
	139
	140	ldr x0, [x3, #VCPU_HOST_CONTEXT]
	141	kern_hyp_va x0
	142	add x0, x0, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	143	bl __fpsimd_save_state
	144
	145	add x2, x3, #VCPU_CONTEXT
	146	add x0, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
	147	bl __fpsimd_restore_state
	148
	149	// Skip restoring fpexc32 for AArch64 guests
	150	mrs x1, hcr_el2
	151	tbnz x1, #HCR_RW_SHIFT, 1f
	152	ldr x4, [x3, #VCPU_FPEXC32_EL2]
	153	msr fpexc32_el2, x4
	154	1:
	155	ldp x4, lr, [sp], #16
	156	ldp x2, x3, [sp], #16
	157	ldp x0, x1, [sp], #16
	158
	159	eret
	160	ENDPROC(__fpsimd_guest_restore)

arch/arm64/kvm/hyp/fpsimd.S+33−0

@@ -0,0 +1,33 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/linkage.h>
	19
	20	#include <asm/fpsimdmacros.h>
	21
	22	.text
	23	.pushsection .hyp.text, "ax"
	24
	25	ENTRY(__fpsimd_save_state)
	26	fpsimd_save x0, 1
	27	ret
	28	ENDPROC(__fpsimd_save_state)
	29
	30	ENTRY(__fpsimd_restore_state)
	31	fpsimd_restore x0, 1
	32	ret
	33	ENDPROC(__fpsimd_restore_state)

arch/arm64/kvm/hyp/hyp-entry.S+212−0

@@ -0,0 +1,212 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/linkage.h>
	19
	20	#include <asm/alternative.h>
	21	#include <asm/assembler.h>
	22	#include <asm/asm-offsets.h>
	23	#include <asm/cpufeature.h>
	24	#include <asm/kvm_arm.h>
	25	#include <asm/kvm_asm.h>
	26	#include <asm/kvm_mmu.h>
	27
	28	.text
	29	.pushsection .hyp.text, "ax"
	30
	31	.macro save_x0_to_x3
	32	stp x0, x1, [sp, #-16]!
	33	stp x2, x3, [sp, #-16]!
	34	.endm
	35
	36	.macro restore_x0_to_x3
	37	ldp x2, x3, [sp], #16
	38	ldp x0, x1, [sp], #16
	39	.endm
	40
	41	el1_sync: // Guest trapped into EL2
	42	save_x0_to_x3
	43
	44	mrs x1, esr_el2
	45	lsr x2, x1, #ESR_ELx_EC_SHIFT
	46
	47	cmp x2, #ESR_ELx_EC_HVC64
	48	b.ne el1_trap
	49
	50	mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
	51	cbnz x3, el1_trap // called HVC
	52
	53	/* Here, we're pretty sure the host called HVC. */
	54	restore_x0_to_x3
	55
	56	/* Check for __hyp_get_vectors */
	57	cbnz x0, 1f
	58	mrs x0, vbar_el2
	59	b 2f
	60
	61	1: stp lr, xzr, [sp, #-16]!
	62
	63	/*
	64	* Compute the function address in EL2, and shuffle the parameters.
	65	*/
	66	kern_hyp_va x0
	67	mov lr, x0
	68	mov x0, x1
	69	mov x1, x2
	70	mov x2, x3
	71	blr lr
	72
	73	ldp lr, xzr, [sp], #16
	74	2: eret
	75
	76	el1_trap:
	77	/*
	78	* x1: ESR
	79	* x2: ESR_EC
	80	*/
	81
	82	/* Guest accessed VFP/SIMD registers, save host, restore Guest */
	83	cmp x2, #ESR_ELx_EC_FP_ASIMD
	84	b.eq __fpsimd_guest_restore
	85
	86	cmp x2, #ESR_ELx_EC_DABT_LOW
	87	mov x0, #ESR_ELx_EC_IABT_LOW
	88	ccmp x2, x0, #4, ne
	89	b.ne 1f // Not an abort we care about
	90
	91	/* This is an abort. Check for permission fault */
	92	alternative_if_not ARM64_WORKAROUND_834220
	93	and x2, x1, #ESR_ELx_FSC_TYPE
	94	cmp x2, #FSC_PERM
	95	b.ne 1f // Not a permission fault
	96	alternative_else
	97	nop // Use the permission fault path to
	98	nop // check for a valid S1 translation,
	99	nop // regardless of the ESR value.
	100	alternative_endif
	101
	102	/*
	103	* Check for Stage-1 page table walk, which is guaranteed
	104	* to give a valid HPFAR_EL2.
	105	*/
	106	tbnz x1, #7, 1f // S1PTW is set
	107
	108	/* Preserve PAR_EL1 */
	109	mrs x3, par_el1
	110	stp x3, xzr, [sp, #-16]!
	111
	112	/*
	113	* Permission fault, HPFAR_EL2 is invalid.
	114	* Resolve the IPA the hard way using the guest VA.
	115	* Stage-1 translation already validated the memory access rights.
	116	* As such, we can use the EL1 translation regime, and don't have
	117	* to distinguish between EL0 and EL1 access.
	118	*/
	119	mrs x2, far_el2
	120	at s1e1r, x2
	121	isb
	122
	123	/* Read result */
	124	mrs x3, par_el1
	125	ldp x0, xzr, [sp], #16 // Restore PAR_EL1 from the stack
	126	msr par_el1, x0
	127	tbnz x3, #0, 3f // Bail out if we failed the translation
	128	ubfx x3, x3, #12, #36 // Extract IPA
	129	lsl x3, x3, #4 // and present it like HPFAR
	130	b 2f
	131
	132	1: mrs x3, hpfar_el2
	133	mrs x2, far_el2
	134
	135	2: mrs x0, tpidr_el2
	136	str w1, [x0, #VCPU_ESR_EL2]
	137	str x2, [x0, #VCPU_FAR_EL2]
	138	str x3, [x0, #VCPU_HPFAR_EL2]
	139
	140	mov x1, #ARM_EXCEPTION_TRAP
	141	b __guest_exit
	142
	143	/*
	144	* Translation failed. Just return to the guest and
	145	* let it fault again. Another CPU is probably playing
	146	* behind our back.
	147	*/
	148	3: restore_x0_to_x3
	149
	150	eret
	151
	152	el1_irq:
	153	save_x0_to_x3
	154	mrs x0, tpidr_el2
	155	mov x1, #ARM_EXCEPTION_IRQ
	156	b __guest_exit
	157
	158	ENTRY(__hyp_do_panic)
	159	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
	160	PSR_MODE_EL1h)
	161	msr spsr_el2, lr
	162	ldr lr, =panic
	163	msr elr_el2, lr
	164	eret
	165	ENDPROC(__hyp_do_panic)
	166
	167	.macro invalid_vector label, target = __hyp_panic
	168	.align 2
	169	\label:
	170	b \target
	171	ENDPROC(\label)
	172	.endm
	173
	174	/* None of these should ever happen */
	175	invalid_vector el2t_sync_invalid
	176	invalid_vector el2t_irq_invalid
	177	invalid_vector el2t_fiq_invalid
	178	invalid_vector el2t_error_invalid
	179	invalid_vector el2h_sync_invalid
	180	invalid_vector el2h_irq_invalid
	181	invalid_vector el2h_fiq_invalid
	182	invalid_vector el2h_error_invalid
	183	invalid_vector el1_sync_invalid
	184	invalid_vector el1_irq_invalid
	185	invalid_vector el1_fiq_invalid
	186	invalid_vector el1_error_invalid
	187
	188	.ltorg
	189
	190	.align 11
	191
	192	ENTRY(__kvm_hyp_vector)
	193	ventry el2t_sync_invalid // Synchronous EL2t
	194	ventry el2t_irq_invalid // IRQ EL2t
	195	ventry el2t_fiq_invalid // FIQ EL2t
	196	ventry el2t_error_invalid // Error EL2t
	197
	198	ventry el2h_sync_invalid // Synchronous EL2h
	199	ventry el2h_irq_invalid // IRQ EL2h
	200	ventry el2h_fiq_invalid // FIQ EL2h
	201	ventry el2h_error_invalid // Error EL2h
	202
	203	ventry el1_sync // Synchronous 64-bit EL1
	204	ventry el1_irq // IRQ 64-bit EL1
	205	ventry el1_fiq_invalid // FIQ 64-bit EL1
	206	ventry el1_error_invalid // Error 64-bit EL1
	207
	208	ventry el1_sync // Synchronous 32-bit EL1
	209	ventry el1_irq // IRQ 32-bit EL1
	210	ventry el1_fiq_invalid // FIQ 32-bit EL1
	211	ventry el1_error_invalid // Error 32-bit EL1
	212	ENDPROC(__kvm_hyp_vector)

arch/arm64/kvm/hyp/hyp.h+90−0

@@ -0,0 +1,90 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#ifndef __ARM64_KVM_HYP_H__
	19	#define __ARM64_KVM_HYP_H__
	20
	21	#include <linux/compiler.h>
	22	#include <linux/kvm_host.h>
	23	#include <asm/kvm_mmu.h>
	24	#include <asm/sysreg.h>
	25
	26	#define __hyp_text __section(.hyp.text) notrace
	27
	28	#define kern_hyp_va(v) (typeof(v))((unsigned long)(v) & HYP_PAGE_OFFSET_MASK)
	29	#define hyp_kern_va(v) (typeof(v))((unsigned long)(v) - HYP_PAGE_OFFSET \
	30	+ PAGE_OFFSET)
	31
	32	/**
	33	* hyp_alternate_select - Generates patchable code sequences that are
	34	* used to switch between two implementations of a function, depending
	35	* on the availability of a feature.
	36	*
	37	* @fname: a symbol name that will be defined as a function returning a
	38	* function pointer whose type will match @orig and @alt
	39	* @orig: A pointer to the default function, as returned by @fname when
	40	* @cond doesn't hold
	41	* @alt: A pointer to the alternate function, as returned by @fname
	42	* when @cond holds
	43	* @cond: a CPU feature (as described in asm/cpufeature.h)
	44	*/
	45	#define hyp_alternate_select(fname, orig, alt, cond) \
	46	typeof(orig) * __hyp_text fname(void) \
	47	{ \
	48	typeof(alt) *val = orig; \
	49	asm volatile(ALTERNATIVE("nop \n", \
	50	"mov %0, %1 \n", \
	51	cond) \
	52	: "+r" (val) : "r" (alt)); \
	53	return val; \
	54	}
	55
	56	void __vgic_v2_save_state(struct kvm_vcpu *vcpu);
	57	void __vgic_v2_restore_state(struct kvm_vcpu *vcpu);
	58
	59	void __vgic_v3_save_state(struct kvm_vcpu *vcpu);
	60	void __vgic_v3_restore_state(struct kvm_vcpu *vcpu);
	61
	62	void __timer_save_state(struct kvm_vcpu *vcpu);
	63	void __timer_restore_state(struct kvm_vcpu *vcpu);
	64
	65	void __sysreg_save_state(struct kvm_cpu_context *ctxt);
	66	void __sysreg_restore_state(struct kvm_cpu_context *ctxt);
	67	void __sysreg32_save_state(struct kvm_vcpu *vcpu);
	68	void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
	69
	70	void __debug_save_state(struct kvm_vcpu *vcpu,
	71	struct kvm_guest_debug_arch *dbg,
	72	struct kvm_cpu_context *ctxt);
	73	void __debug_restore_state(struct kvm_vcpu *vcpu,
	74	struct kvm_guest_debug_arch *dbg,
	75	struct kvm_cpu_context *ctxt);
	76	void __debug_cond_save_host_state(struct kvm_vcpu *vcpu);
	77	void __debug_cond_restore_host_state(struct kvm_vcpu *vcpu);
	78
	79	void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
	80	void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
	81	static inline bool __fpsimd_enabled(void)
	82	{
	83	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
	84	}
	85
	86	u64 __guest_enter(struct kvm_vcpu vcpu, struct kvm_cpu_context host_ctxt);
	87	void __noreturn __hyp_do_panic(unsigned long, ...);
	88
	89	#endif /* __ARM64_KVM_HYP_H__ */
	90

arch/arm64/kvm/hyp/switch.c+175−0

@@ -0,0 +1,175 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include "hyp.h"
	19
	20	static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
	21	{
	22	u64 val;
	23
	24	/*
	25	* We are about to set CPTR_EL2.TFP to trap all floating point
	26	* register accesses to EL2, however, the ARM ARM clearly states that
	27	* traps are only taken to EL2 if the operation would not otherwise
	28	* trap to EL1. Therefore, always make sure that for 32-bit guests,
	29	* we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
	30	*/
	31	val = vcpu->arch.hcr_el2;
	32	if (!(val & HCR_RW)) {
	33	write_sysreg(1 << 30, fpexc32_el2);
	34	isb();
	35	}
	36	write_sysreg(val, hcr_el2);
	37	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	38	write_sysreg(1 << 15, hstr_el2);
	39	write_sysreg(CPTR_EL2_TTA \| CPTR_EL2_TFP, cptr_el2);
	40	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	41	}
	42
	43	static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
	44	{
	45	write_sysreg(HCR_RW, hcr_el2);
	46	write_sysreg(0, hstr_el2);
	47	write_sysreg(read_sysreg(mdcr_el2) & MDCR_EL2_HPMN_MASK, mdcr_el2);
	48	write_sysreg(0, cptr_el2);
	49	}
	50
	51	static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
	52	{
	53	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	54	write_sysreg(kvm->arch.vttbr, vttbr_el2);
	55	}
	56
	57	static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
	58	{
	59	write_sysreg(0, vttbr_el2);
	60	}
	61
	62	static hyp_alternate_select(__vgic_call_save_state,
	63	__vgic_v2_save_state, __vgic_v3_save_state,
	64	ARM64_HAS_SYSREG_GIC_CPUIF);
	65
	66	static hyp_alternate_select(__vgic_call_restore_state,
	67	__vgic_v2_restore_state, __vgic_v3_restore_state,
	68	ARM64_HAS_SYSREG_GIC_CPUIF);
	69
	70	static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
	71	{
	72	__vgic_call_save_state()(vcpu);
	73	write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
	74	}
	75
	76	static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
	77	{
	78	u64 val;
	79
	80	val = read_sysreg(hcr_el2);
	81	val \|= HCR_INT_OVERRIDE;
	82	val \|= vcpu->arch.irq_lines;
	83	write_sysreg(val, hcr_el2);
	84
	85	__vgic_call_restore_state()(vcpu);
	86	}
	87
	88	static int __hyp_text __guest_run(struct kvm_vcpu *vcpu)
	89	{
	90	struct kvm_cpu_context *host_ctxt;
	91	struct kvm_cpu_context *guest_ctxt;
	92	bool fp_enabled;
	93	u64 exit_code;
	94
	95	vcpu = kern_hyp_va(vcpu);
	96	write_sysreg(vcpu, tpidr_el2);
	97
	98	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	99	guest_ctxt = &vcpu->arch.ctxt;
	100
	101	__sysreg_save_state(host_ctxt);
	102	__debug_cond_save_host_state(vcpu);
	103
	104	__activate_traps(vcpu);
	105	__activate_vm(vcpu);
	106
	107	__vgic_restore_state(vcpu);
	108	__timer_restore_state(vcpu);
	109
	110	/*
	111	* We must restore the 32-bit state before the sysregs, thanks
	112	* to Cortex-A57 erratum #852523.
	113	*/
	114	__sysreg32_restore_state(vcpu);
	115	__sysreg_restore_state(guest_ctxt);
	116	__debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	117
	118	/* Jump in the fire! */
	119	exit_code = __guest_enter(vcpu, host_ctxt);
	120	/* And we're baaack! */
	121
	122	fp_enabled = __fpsimd_enabled();
	123
	124	__sysreg_save_state(guest_ctxt);
	125	__sysreg32_save_state(vcpu);
	126	__timer_save_state(vcpu);
	127	__vgic_save_state(vcpu);
	128
	129	__deactivate_traps(vcpu);
	130	__deactivate_vm(vcpu);
	131
	132	__sysreg_restore_state(host_ctxt);
	133
	134	if (fp_enabled) {
	135	__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
	136	__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	137	}
	138
	139	__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	140	__debug_cond_restore_host_state(vcpu);
	141
	142	return exit_code;
	143	}
	144
	145	__alias(__guest_run) int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
	146
	147	static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
	148
	149	void __hyp_text __noreturn __hyp_panic(void)
	150	{
	151	unsigned long str_va = (unsigned long)__hyp_panic_string;
	152	u64 spsr = read_sysreg(spsr_el2);
	153	u64 elr = read_sysreg(elr_el2);
	154	u64 par = read_sysreg(par_el1);
	155
	156	if (read_sysreg(vttbr_el2)) {
	157	struct kvm_vcpu *vcpu;
	158	struct kvm_cpu_context *host_ctxt;
	159
	160	vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
	161	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	162	__deactivate_traps(vcpu);
	163	__deactivate_vm(vcpu);
	164	__sysreg_restore_state(host_ctxt);
	165	}
	166
	167	/* Call panic for real */
	168	__hyp_do_panic(hyp_kern_va(str_va),
	169	spsr, elr,
	170	read_sysreg(esr_el2), read_sysreg(far_el2),
	171	read_sysreg(hpfar_el2), par,
	172	(void *)read_sysreg(tpidr_el2));
	173
	174	unreachable();
	175	}

arch/arm64/kvm/hyp/sysreg-sr.c+138−0

@@ -0,0 +1,138 @@
	1	/*
	2	* Copyright (C) 2012-2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/compiler.h>
	19	#include <linux/kvm_host.h>
	20
	21	#include <asm/kvm_asm.h>
	22	#include <asm/kvm_mmu.h>
	23
	24	#include "hyp.h"
	25
	26	/* ctxt is already in the HYP VA space */
	27	void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
	28	{
	29	ctxt->sys_regs[MPIDR_EL1] = read_sysreg(vmpidr_el2);
	30	ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1);
	31	ctxt->sys_regs[SCTLR_EL1] = read_sysreg(sctlr_el1);
	32	ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1);
	33	ctxt->sys_regs[CPACR_EL1] = read_sysreg(cpacr_el1);
	34	ctxt->sys_regs[TTBR0_EL1] = read_sysreg(ttbr0_el1);
	35	ctxt->sys_regs[TTBR1_EL1] = read_sysreg(ttbr1_el1);
	36	ctxt->sys_regs[TCR_EL1] = read_sysreg(tcr_el1);
	37	ctxt->sys_regs[ESR_EL1] = read_sysreg(esr_el1);
	38	ctxt->sys_regs[AFSR0_EL1] = read_sysreg(afsr0_el1);
	39	ctxt->sys_regs[AFSR1_EL1] = read_sysreg(afsr1_el1);
	40	ctxt->sys_regs[FAR_EL1] = read_sysreg(far_el1);
	41	ctxt->sys_regs[MAIR_EL1] = read_sysreg(mair_el1);
	42	ctxt->sys_regs[VBAR_EL1] = read_sysreg(vbar_el1);
	43	ctxt->sys_regs[CONTEXTIDR_EL1] = read_sysreg(contextidr_el1);
	44	ctxt->sys_regs[TPIDR_EL0] = read_sysreg(tpidr_el0);
	45	ctxt->sys_regs[TPIDRRO_EL0] = read_sysreg(tpidrro_el0);
	46	ctxt->sys_regs[TPIDR_EL1] = read_sysreg(tpidr_el1);
	47	ctxt->sys_regs[AMAIR_EL1] = read_sysreg(amair_el1);
	48	ctxt->sys_regs[CNTKCTL_EL1] = read_sysreg(cntkctl_el1);
	49	ctxt->sys_regs[PAR_EL1] = read_sysreg(par_el1);
	50	ctxt->sys_regs[MDSCR_EL1] = read_sysreg(mdscr_el1);
	51
	52	ctxt->gp_regs.regs.sp = read_sysreg(sp_el0);
	53	ctxt->gp_regs.regs.pc = read_sysreg(elr_el2);
	54	ctxt->gp_regs.regs.pstate = read_sysreg(spsr_el2);
	55	ctxt->gp_regs.sp_el1 = read_sysreg(sp_el1);
	56	ctxt->gp_regs.elr_el1 = read_sysreg(elr_el1);
	57	ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg(spsr_el1);
	58	}
	59
	60	void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
	61	{
	62	write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2);
	63	write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1);
	64	write_sysreg(ctxt->sys_regs[SCTLR_EL1], sctlr_el1);
	65	write_sysreg(ctxt->sys_regs[ACTLR_EL1], actlr_el1);
	66	write_sysreg(ctxt->sys_regs[CPACR_EL1], cpacr_el1);
	67	write_sysreg(ctxt->sys_regs[TTBR0_EL1], ttbr0_el1);
	68	write_sysreg(ctxt->sys_regs[TTBR1_EL1], ttbr1_el1);
	69	write_sysreg(ctxt->sys_regs[TCR_EL1], tcr_el1);
	70	write_sysreg(ctxt->sys_regs[ESR_EL1], esr_el1);
	71	write_sysreg(ctxt->sys_regs[AFSR0_EL1], afsr0_el1);
	72	write_sysreg(ctxt->sys_regs[AFSR1_EL1], afsr1_el1);
	73	write_sysreg(ctxt->sys_regs[FAR_EL1], far_el1);
	74	write_sysreg(ctxt->sys_regs[MAIR_EL1], mair_el1);
	75	write_sysreg(ctxt->sys_regs[VBAR_EL1], vbar_el1);
	76	write_sysreg(ctxt->sys_regs[CONTEXTIDR_EL1], contextidr_el1);
	77	write_sysreg(ctxt->sys_regs[TPIDR_EL0], tpidr_el0);
	78	write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
	79	write_sysreg(ctxt->sys_regs[TPIDR_EL1], tpidr_el1);
	80	write_sysreg(ctxt->sys_regs[AMAIR_EL1], amair_el1);
	81	write_sysreg(ctxt->sys_regs[CNTKCTL_EL1], cntkctl_el1);
	82	write_sysreg(ctxt->sys_regs[PAR_EL1], par_el1);
	83	write_sysreg(ctxt->sys_regs[MDSCR_EL1], mdscr_el1);
	84
	85	write_sysreg(ctxt->gp_regs.regs.sp, sp_el0);
	86	write_sysreg(ctxt->gp_regs.regs.pc, elr_el2);
	87	write_sysreg(ctxt->gp_regs.regs.pstate, spsr_el2);
	88	write_sysreg(ctxt->gp_regs.sp_el1, sp_el1);
	89	write_sysreg(ctxt->gp_regs.elr_el1, elr_el1);
	90	write_sysreg(ctxt->gp_regs.spsr[KVM_SPSR_EL1], spsr_el1);
	91	}
	92
	93	void __hyp_text __sysreg32_save_state(struct kvm_vcpu *vcpu)
	94	{
	95	u64 spsr, sysreg;
	96
	97	if (read_sysreg(hcr_el2) & HCR_RW)
	98	return;
	99
	100	spsr = vcpu->arch.ctxt.gp_regs.spsr;
	101	sysreg = vcpu->arch.ctxt.sys_regs;
	102
	103	spsr[KVM_SPSR_ABT] = read_sysreg(spsr_abt);
	104	spsr[KVM_SPSR_UND] = read_sysreg(spsr_und);
	105	spsr[KVM_SPSR_IRQ] = read_sysreg(spsr_irq);
	106	spsr[KVM_SPSR_FIQ] = read_sysreg(spsr_fiq);
	107
	108	sysreg[DACR32_EL2] = read_sysreg(dacr32_el2);
	109	sysreg[IFSR32_EL2] = read_sysreg(ifsr32_el2);
	110
	111	if (__fpsimd_enabled())
	112	sysreg[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
	113
	114	if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
	115	sysreg[DBGVCR32_EL2] = read_sysreg(dbgvcr32_el2);
	116	}
	117
	118	void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu)
	119	{
	120	u64 spsr, sysreg;
	121
	122	if (read_sysreg(hcr_el2) & HCR_RW)
	123	return;
	124
	125	spsr = vcpu->arch.ctxt.gp_regs.spsr;
	126	sysreg = vcpu->arch.ctxt.sys_regs;
	127
	128	write_sysreg(spsr[KVM_SPSR_ABT], spsr_abt);
	129	write_sysreg(spsr[KVM_SPSR_UND], spsr_und);
	130	write_sysreg(spsr[KVM_SPSR_IRQ], spsr_irq);
	131	write_sysreg(spsr[KVM_SPSR_FIQ], spsr_fiq);
	132
	133	write_sysreg(sysreg[DACR32_EL2], dacr32_el2);
	134	write_sysreg(sysreg[IFSR32_EL2], ifsr32_el2);
	135
	136	if (vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
	137	write_sysreg(sysreg[DBGVCR32_EL2], dbgvcr32_el2);
	138	}

arch/arm64/kvm/hyp/timer-sr.c+71−0

@@ -0,0 +1,71 @@
	1	/*
	2	* Copyright (C) 2012-2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <clocksource/arm_arch_timer.h>
	19	#include <linux/compiler.h>
	20	#include <linux/kvm_host.h>
	21
	22	#include <asm/kvm_mmu.h>
	23
	24	#include "hyp.h"
	25
	26	/* vcpu is already in the HYP VA space */
	27	void __hyp_text __timer_save_state(struct kvm_vcpu *vcpu)
	28	{
	29	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	30	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
	31	u64 val;
	32
	33	if (kvm->arch.timer.enabled) {
	34	timer->cntv_ctl = read_sysreg(cntv_ctl_el0);
	35	timer->cntv_cval = read_sysreg(cntv_cval_el0);
	36	}
	37
	38	/* Disable the virtual timer */
	39	write_sysreg(0, cntv_ctl_el0);
	40
	41	/* Allow physical timer/counter access for the host */
	42	val = read_sysreg(cnthctl_el2);
	43	val \|= CNTHCTL_EL1PCTEN \| CNTHCTL_EL1PCEN;
	44	write_sysreg(val, cnthctl_el2);
	45
	46	/* Clear cntvoff for the host */
	47	write_sysreg(0, cntvoff_el2);
	48	}
	49
	50	void __hyp_text __timer_restore_state(struct kvm_vcpu *vcpu)
	51	{
	52	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	53	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
	54	u64 val;
	55
	56	/*
	57	* Disallow physical timer access for the guest
	58	* Physical counter access is allowed
	59	*/
	60	val = read_sysreg(cnthctl_el2);
	61	val &= ~CNTHCTL_EL1PCEN;
	62	val \|= CNTHCTL_EL1PCTEN;
	63	write_sysreg(val, cnthctl_el2);
	64
	65	if (kvm->arch.timer.enabled) {
	66	write_sysreg(kvm->arch.timer.cntvoff, cntvoff_el2);
	67	write_sysreg(timer->cntv_cval, cntv_cval_el0);
	68	isb();
	69	write_sysreg(timer->cntv_ctl, cntv_ctl_el0);
	70	}
	71	}

arch/arm64/kvm/hyp/tlb.c+80−0

@@ -0,0 +1,80 @@
	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include "hyp.h"
	19
	20	static void __hyp_text __tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
	21	{
	22	dsb(ishst);
	23
	24	/* Switch to requested VMID */
	25	kvm = kern_hyp_va(kvm);
	26	write_sysreg(kvm->arch.vttbr, vttbr_el2);
	27	isb();
	28
	29	/*
	30	* We could do so much better if we had the VA as well.
	31	* Instead, we invalidate Stage-2 for this IPA, and the
	32	* whole of Stage-1. Weep...
	33	*/
	34	ipa >>= 12;
	35	asm volatile("tlbi ipas2e1is, %0" : : "r" (ipa));
	36
	37	/*
	38	* We have to ensure completion of the invalidation at Stage-2,
	39	* since a table walk on another CPU could refill a TLB with a
	40	* complete (S1 + S2) walk based on the old Stage-2 mapping if
	41	* the Stage-1 invalidation happened first.
	42	*/
	43	dsb(ish);
	44	asm volatile("tlbi vmalle1is" : : );
	45	dsb(ish);
	46	isb();
	47
	48	write_sysreg(0, vttbr_el2);
	49	}
	50
	51	__alias(__tlb_flush_vmid_ipa) void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm,
	52	phys_addr_t ipa);
	53
	54	static void __hyp_text __tlb_flush_vmid(struct kvm *kvm)
	55	{
	56	dsb(ishst);
	57
	58	/* Switch to requested VMID */
	59	kvm = kern_hyp_va(kvm);
	60	write_sysreg(kvm->arch.vttbr, vttbr_el2);
	61	isb();
	62
	63	asm volatile("tlbi vmalls12e1is" : : );
	64	dsb(ish);
	65	isb();
	66
	67	write_sysreg(0, vttbr_el2);
	68	}
	69
	70	__alias(__tlb_flush_vmid) void __kvm_tlb_flush_vmid(struct kvm *kvm);
	71
	72	static void __hyp_text __tlb_flush_vm_context(void)
	73	{
	74	dsb(ishst);
	75	asm volatile("tlbi alle1is \n"
	76	"ic ialluis ": : );
	77	dsb(ish);
	78	}
	79
	80	__alias(__tlb_flush_vm_context) void __kvm_flush_vm_context(void);

arch/arm64/kvm/hyp/vgic-v2-sr.c+84−0

@@ -0,0 +1,84 @@
	1	/*
	2	* Copyright (C) 2012-2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/compiler.h>
	19	#include <linux/irqchip/arm-gic.h>
	20	#include <linux/kvm_host.h>
	21
	22	#include <asm/kvm_mmu.h>
	23
	24	#include "hyp.h"
	25
	26	/* vcpu is already in the HYP VA space */
	27	void __hyp_text __vgic_v2_save_state(struct kvm_vcpu *vcpu)
	28	{
	29	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	30	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
	31	struct vgic_dist *vgic = &kvm->arch.vgic;
	32	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
	33	u32 eisr0, eisr1, elrsr0, elrsr1;
	34	int i, nr_lr;
	35
	36	if (!base)
	37	return;
	38
	39	nr_lr = vcpu->arch.vgic_cpu.nr_lr;
	40	cpu_if->vgic_vmcr = readl_relaxed(base + GICH_VMCR);
	41	cpu_if->vgic_misr = readl_relaxed(base + GICH_MISR);
	42	eisr0 = readl_relaxed(base + GICH_EISR0);
	43	elrsr0 = readl_relaxed(base + GICH_ELRSR0);
	44	if (unlikely(nr_lr > 32)) {
	45	eisr1 = readl_relaxed(base + GICH_EISR1);
	46	elrsr1 = readl_relaxed(base + GICH_ELRSR1);
	47	} else {
	48	eisr1 = elrsr1 = 0;
	49	}
	50	#ifdef CONFIG_CPU_BIG_ENDIAN
	51	cpu_if->vgic_eisr = ((u64)eisr0 << 32) \| eisr1;
	52	cpu_if->vgic_elrsr = ((u64)elrsr0 << 32) \| elrsr1;
	53	#else
	54	cpu_if->vgic_eisr = ((u64)eisr1 << 32) \| eisr0;
	55	cpu_if->vgic_elrsr = ((u64)elrsr1 << 32) \| elrsr0;
	56	#endif
	57	cpu_if->vgic_apr = readl_relaxed(base + GICH_APR);
	58
	59	writel_relaxed(0, base + GICH_HCR);
	60
	61	for (i = 0; i < nr_lr; i++)
	62	cpu_if->vgic_lr[i] = readl_relaxed(base + GICH_LR0 + (i * 4));
	63	}
	64
	65	/* vcpu is already in the HYP VA space */
	66	void __hyp_text __vgic_v2_restore_state(struct kvm_vcpu *vcpu)
	67	{
	68	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	69	struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2;
	70	struct vgic_dist *vgic = &kvm->arch.vgic;
	71	void __iomem *base = kern_hyp_va(vgic->vctrl_base);
	72	int i, nr_lr;
	73
	74	if (!base)
	75	return;
	76
	77	writel_relaxed(cpu_if->vgic_hcr, base + GICH_HCR);
	78	writel_relaxed(cpu_if->vgic_vmcr, base + GICH_VMCR);
	79	writel_relaxed(cpu_if->vgic_apr, base + GICH_APR);
	80
	81	nr_lr = vcpu->arch.vgic_cpu.nr_lr;
	82	for (i = 0; i < nr_lr; i++)
	83	writel_relaxed(cpu_if->vgic_lr[i], base + GICH_LR0 + (i * 4));
	84	}

arch/arm64/kvm/hyp/vgic-v3-sr.c+228−0

@@ -0,0 +1,228 @@
	1	/*
	2	* Copyright (C) 2012-2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include <linux/compiler.h>
	19	#include <linux/irqchip/arm-gic-v3.h>
	20	#include <linux/kvm_host.h>
	21
	22	#include <asm/kvm_mmu.h>
	23
	24	#include "hyp.h"
	25
	26	#define vtr_to_max_lr_idx(v) ((v) & 0xf)
	27	#define vtr_to_nr_pri_bits(v) (((u32)(v) >> 29) + 1)
	28
	29	#define read_gicreg(r) \
	30	({ \
	31	u64 reg; \
	32	asm volatile("mrs_s %0, " __stringify(r) : "=r" (reg)); \
	33	reg; \
	34	})
	35
	36	#define write_gicreg(v,r) \
	37	do { \
	38	u64 __val = (v); \
	39	asm volatile("msr_s " __stringify(r) ", %0" : : "r" (__val));\
	40	} while (0)
	41
	42	/* vcpu is already in the HYP VA space */
	43	void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
	44	{
	45	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	46	u64 val;
	47	u32 max_lr_idx, nr_pri_bits;
	48
	49	/*
	50	* Make sure stores to the GIC via the memory mapped interface
	51	* are now visible to the system register interface.
	52	*/
	53	dsb(st);
	54
	55	cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
	56	cpu_if->vgic_misr = read_gicreg(ICH_MISR_EL2);
	57	cpu_if->vgic_eisr = read_gicreg(ICH_EISR_EL2);
	58	cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
	59
	60	write_gicreg(0, ICH_HCR_EL2);
	61	val = read_gicreg(ICH_VTR_EL2);
	62	max_lr_idx = vtr_to_max_lr_idx(val);
	63	nr_pri_bits = vtr_to_nr_pri_bits(val);
	64
	65	switch (max_lr_idx) {
	66	case 15:
	67	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(15)] = read_gicreg(ICH_LR15_EL2);
	68	case 14:
	69	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(14)] = read_gicreg(ICH_LR14_EL2);
	70	case 13:
	71	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(13)] = read_gicreg(ICH_LR13_EL2);
	72	case 12:
	73	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(12)] = read_gicreg(ICH_LR12_EL2);
	74	case 11:
	75	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(11)] = read_gicreg(ICH_LR11_EL2);
	76	case 10:
	77	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(10)] = read_gicreg(ICH_LR10_EL2);
	78	case 9:
	79	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(9)] = read_gicreg(ICH_LR9_EL2);
	80	case 8:
	81	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(8)] = read_gicreg(ICH_LR8_EL2);
	82	case 7:
	83	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(7)] = read_gicreg(ICH_LR7_EL2);
	84	case 6:
	85	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(6)] = read_gicreg(ICH_LR6_EL2);
	86	case 5:
	87	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(5)] = read_gicreg(ICH_LR5_EL2);
	88	case 4:
	89	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(4)] = read_gicreg(ICH_LR4_EL2);
	90	case 3:
	91	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(3)] = read_gicreg(ICH_LR3_EL2);
	92	case 2:
	93	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(2)] = read_gicreg(ICH_LR2_EL2);
	94	case 1:
	95	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(1)] = read_gicreg(ICH_LR1_EL2);
	96	case 0:
	97	cpu_if->vgic_lr[VGIC_V3_LR_INDEX(0)] = read_gicreg(ICH_LR0_EL2);
	98	}
	99
	100	switch (nr_pri_bits) {
	101	case 7:
	102	cpu_if->vgic_ap0r[3] = read_gicreg(ICH_AP0R3_EL2);
	103	cpu_if->vgic_ap0r[2] = read_gicreg(ICH_AP0R2_EL2);
	104	case 6:
	105	cpu_if->vgic_ap0r[1] = read_gicreg(ICH_AP0R1_EL2);
	106	default:
	107	cpu_if->vgic_ap0r[0] = read_gicreg(ICH_AP0R0_EL2);
	108	}
	109
	110	switch (nr_pri_bits) {
	111	case 7:
	112	cpu_if->vgic_ap1r[3] = read_gicreg(ICH_AP1R3_EL2);
	113	cpu_if->vgic_ap1r[2] = read_gicreg(ICH_AP1R2_EL2);
	114	case 6:
	115	cpu_if->vgic_ap1r[1] = read_gicreg(ICH_AP1R1_EL2);
	116	default:
	117	cpu_if->vgic_ap1r[0] = read_gicreg(ICH_AP1R0_EL2);
	118	}
	119
	120	val = read_gicreg(ICC_SRE_EL2);
	121	write_gicreg(val \| ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
	122	isb(); /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
	123	write_gicreg(1, ICC_SRE_EL1);
	124	}
	125
	126	void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
	127	{
	128	struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
	129	u64 val;
	130	u32 max_lr_idx, nr_pri_bits;
	131
	132	/*
	133	* VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
	134	* Group0 interrupt (as generated in GICv2 mode) to be
	135	* delivered as a FIQ to the guest, with potentially fatal
	136	* consequences. So we must make sure that ICC_SRE_EL1 has
	137	* been actually programmed with the value we want before
	138	* starting to mess with the rest of the GIC.
	139	*/
	140	write_gicreg(cpu_if->vgic_sre, ICC_SRE_EL1);
	141	isb();
	142
	143	write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
	144	write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
	145
	146	val = read_gicreg(ICH_VTR_EL2);
	147	max_lr_idx = vtr_to_max_lr_idx(val);
	148	nr_pri_bits = vtr_to_nr_pri_bits(val);
	149
	150	switch (nr_pri_bits) {
	151	case 7:
	152	write_gicreg(cpu_if->vgic_ap1r[3], ICH_AP1R3_EL2);
	153	write_gicreg(cpu_if->vgic_ap1r[2], ICH_AP1R2_EL2);
	154	case 6:
	155	write_gicreg(cpu_if->vgic_ap1r[1], ICH_AP1R1_EL2);
	156	default:
	157	write_gicreg(cpu_if->vgic_ap1r[0], ICH_AP1R0_EL2);
	158	}
	159
	160	switch (nr_pri_bits) {
	161	case 7:
	162	write_gicreg(cpu_if->vgic_ap0r[3], ICH_AP0R3_EL2);
	163	write_gicreg(cpu_if->vgic_ap0r[2], ICH_AP0R2_EL2);
	164	case 6:
	165	write_gicreg(cpu_if->vgic_ap0r[1], ICH_AP0R1_EL2);
	166	default:
	167	write_gicreg(cpu_if->vgic_ap0r[0], ICH_AP0R0_EL2);
	168	}
	169
	170	switch (max_lr_idx) {
	171	case 15:
	172	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(15)], ICH_LR15_EL2);
	173	case 14:
	174	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(14)], ICH_LR14_EL2);
	175	case 13:
	176	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(13)], ICH_LR13_EL2);
	177	case 12:
	178	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(12)], ICH_LR12_EL2);
	179	case 11:
	180	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(11)], ICH_LR11_EL2);
	181	case 10:
	182	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(10)], ICH_LR10_EL2);
	183	case 9:
	184	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(9)], ICH_LR9_EL2);
	185	case 8:
	186	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(8)], ICH_LR8_EL2);
	187	case 7:
	188	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(7)], ICH_LR7_EL2);
	189	case 6:
	190	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(6)], ICH_LR6_EL2);
	191	case 5:
	192	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(5)], ICH_LR5_EL2);
	193	case 4:
	194	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(4)], ICH_LR4_EL2);
	195	case 3:
	196	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(3)], ICH_LR3_EL2);
	197	case 2:
	198	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(2)], ICH_LR2_EL2);
	199	case 1:
	200	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(1)], ICH_LR1_EL2);
	201	case 0:
	202	write_gicreg(cpu_if->vgic_lr[VGIC_V3_LR_INDEX(0)], ICH_LR0_EL2);
	203	}
	204
	205	/*
	206	* Ensures that the above will have reached the
	207	* (re)distributors. This ensure the guest will read the
	208	* correct values from the memory-mapped interface.
	209	*/
	210	isb();
	211	dsb(sy);
	212
	213	/*
	214	* Prevent the guest from touching the GIC system registers if
	215	* SRE isn't enabled for GICv3 emulation.
	216	*/
	217	if (!cpu_if->vgic_sre) {
	218	write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
	219	ICC_SRE_EL2);
	220	}
	221	}
	222
	223	static u64 __hyp_text __vgic_v3_read_ich_vtr_el2(void)
	224	{
	225	return read_gicreg(ICH_VTR_EL2);
	226	}
	227
	228	__alias(__vgic_v3_read_ich_vtr_el2) u64 __vgic_v3_get_ich_vtr_el2(void);

arch/arm64/kvm/sys_regs.c+30−29

 #include <asm/debug-monitors.h>
 #include <asm/esr.h>
 #include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_host.h>
  * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
  * hyp.S code switches between host and guest values in future.
  */
-static inline void reg_to_dbg(struct kvm_vcpu *vcpu,
+static void reg_to_dbg(struct kvm_vcpu *vcpu,
-			      struct sys_reg_params *p,
+		       struct sys_reg_params *p,
-			      u64 *dbg_reg)
+		       u64 *dbg_reg)
 {
 	u64 val = p->regval;
 	vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 }
-static inline void dbg_to_reg(struct kvm_vcpu *vcpu,
+static void dbg_to_reg(struct kvm_vcpu *vcpu,
-			      struct sys_reg_params *p,
+		       struct sys_reg_params *p,
-			      u64 *dbg_reg)
+		       u64 *dbg_reg)
 {
 	p->regval = *dbg_reg;
 	if (p->is_32bit)
 		p->regval &= 0xffffffffUL;
 }
-static inline bool trap_bvr(struct kvm_vcpu *vcpu,
+static bool trap_bvr(struct kvm_vcpu *vcpu,
-			    struct sys_reg_params *p,
+		     struct sys_reg_params *p,
-			    const struct sys_reg_desc *rd)
+		     const struct sys_reg_desc *rd)
 {
 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
 	return 0;
 }
-static inline void reset_bvr(struct kvm_vcpu *vcpu,
+static void reset_bvr(struct kvm_vcpu *vcpu,
-			     const struct sys_reg_desc *rd)
+		      const struct sys_reg_desc *rd)
 {
 	vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
 }
-static inline bool trap_bcr(struct kvm_vcpu *vcpu,
+static bool trap_bcr(struct kvm_vcpu *vcpu,
-			    struct sys_reg_params *p,
+		     struct sys_reg_params *p,
-			    const struct sys_reg_desc *rd)
+		     const struct sys_reg_desc *rd)
 {
 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
 	return 0;
 }
-static inline void reset_bcr(struct kvm_vcpu *vcpu,
+static void reset_bcr(struct kvm_vcpu *vcpu,
-			     const struct sys_reg_desc *rd)
+		      const struct sys_reg_desc *rd)
 {
 	vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
 }
-static inline bool trap_wvr(struct kvm_vcpu *vcpu,
+static bool trap_wvr(struct kvm_vcpu *vcpu,
-			    struct sys_reg_params *p,
+		     struct sys_reg_params *p,
-			    const struct sys_reg_desc *rd)
+		     const struct sys_reg_desc *rd)
 {
 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
 	return 0;
 }
-static inline void reset_wvr(struct kvm_vcpu *vcpu,
+static void reset_wvr(struct kvm_vcpu *vcpu,
-			     const struct sys_reg_desc *rd)
+		      const struct sys_reg_desc *rd)
 {
 	vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
 }
-static inline bool trap_wcr(struct kvm_vcpu *vcpu,
+static bool trap_wcr(struct kvm_vcpu *vcpu,
-			    struct sys_reg_params *p,
+		     struct sys_reg_params *p,
-			    const struct sys_reg_desc *rd)
+		     const struct sys_reg_desc *rd)
 {
 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
 	return 0;
 }
-static inline void reset_wcr(struct kvm_vcpu *vcpu,
+static void reset_wcr(struct kvm_vcpu *vcpu,
-			     const struct sys_reg_desc *rd)
+		      const struct sys_reg_desc *rd)
 {
 	vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
 }
  * system is in.
  */
-static inline bool trap_xvr(struct kvm_vcpu *vcpu,
+static bool trap_xvr(struct kvm_vcpu *vcpu,
-			    struct sys_reg_params *p,
+		     struct sys_reg_params *p,
-			    const struct sys_reg_desc *rd)
+		     const struct sys_reg_desc *rd)
 {
 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];

arch/arm64/kvm/vgic-v2-switch.S+0−134

@@ -1,134 +0,0 @@
1	/*
2	* Copyright (C) 2012,2013 - ARM Ltd
3	* Author: Marc Zyngier <marc.zyngier@arm.com>
4	*
5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License version 2 as
7	* published by the Free Software Foundation.
8	*
9	* This program is distributed in the hope that it will be useful,
10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	* GNU General Public License for more details.
13	*
14	* You should have received a copy of the GNU General Public License
15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
16	*/
17
18	#include <linux/linkage.h>
19	#include <linux/irqchip/arm-gic.h>
20
21	#include <asm/assembler.h>
22	#include <asm/memory.h>
23	#include <asm/asm-offsets.h>
24	#include <asm/kvm.h>
25	#include <asm/kvm_asm.h>
26	#include <asm/kvm_arm.h>
27	#include <asm/kvm_mmu.h>
28
29	.text
30	.pushsection .hyp.text, "ax"
31
32	/*
33	* Save the VGIC CPU state into memory
34	* x0: Register pointing to VCPU struct
35	* Do not corrupt x1!!!
36	*/
37	ENTRY(__save_vgic_v2_state)
38	__save_vgic_v2_state:
39	/* Get VGIC VCTRL base into x2 */
40	ldr x2, [x0, #VCPU_KVM]
41	kern_hyp_va x2
42	ldr x2, [x2, #KVM_VGIC_VCTRL]
43	kern_hyp_va x2
44	cbz x2, 2f // disabled
45
46	/* Compute the address of struct vgic_cpu */
47	add x3, x0, #VCPU_VGIC_CPU
48
49	/* Save all interesting registers */
50	ldr w5, [x2, #GICH_VMCR]
51	ldr w6, [x2, #GICH_MISR]
52	ldr w7, [x2, #GICH_EISR0]
53	ldr w8, [x2, #GICH_EISR1]
54	ldr w9, [x2, #GICH_ELRSR0]
55	ldr w10, [x2, #GICH_ELRSR1]
56	ldr w11, [x2, #GICH_APR]
57	CPU_BE( rev w5, w5 )
58	CPU_BE( rev w6, w6 )
59	CPU_BE( rev w7, w7 )
60	CPU_BE( rev w8, w8 )
61	CPU_BE( rev w9, w9 )
62	CPU_BE( rev w10, w10 )
63	CPU_BE( rev w11, w11 )
64
65	str w5, [x3, #VGIC_V2_CPU_VMCR]
66	str w6, [x3, #VGIC_V2_CPU_MISR]
67	CPU_LE( str w7, [x3, #VGIC_V2_CPU_EISR] )
68	CPU_LE( str w8, [x3, #(VGIC_V2_CPU_EISR + 4)] )
69	CPU_LE( str w9, [x3, #VGIC_V2_CPU_ELRSR] )
70	CPU_LE( str w10, [x3, #(VGIC_V2_CPU_ELRSR + 4)] )
71	CPU_BE( str w7, [x3, #(VGIC_V2_CPU_EISR + 4)] )
72	CPU_BE( str w8, [x3, #VGIC_V2_CPU_EISR] )
73	CPU_BE( str w9, [x3, #(VGIC_V2_CPU_ELRSR + 4)] )
74	CPU_BE( str w10, [x3, #VGIC_V2_CPU_ELRSR] )
75	str w11, [x3, #VGIC_V2_CPU_APR]
76
77	/* Clear GICH_HCR */
78	str wzr, [x2, #GICH_HCR]
79
80	/* Save list registers */
81	add x2, x2, #GICH_LR0
82	ldr w4, [x3, #VGIC_CPU_NR_LR]
83	add x3, x3, #VGIC_V2_CPU_LR
84	1: ldr w5, [x2], #4
85	CPU_BE( rev w5, w5 )
86	str w5, [x3], #4
87	sub w4, w4, #1
88	cbnz w4, 1b
89	2:
90	ret
91	ENDPROC(__save_vgic_v2_state)
92
93	/*
94	* Restore the VGIC CPU state from memory
95	* x0: Register pointing to VCPU struct
96	*/
97	ENTRY(__restore_vgic_v2_state)
98	__restore_vgic_v2_state:
99	/* Get VGIC VCTRL base into x2 */
100	ldr x2, [x0, #VCPU_KVM]
101	kern_hyp_va x2
102	ldr x2, [x2, #KVM_VGIC_VCTRL]
103	kern_hyp_va x2
104	cbz x2, 2f // disabled
105
106	/* Compute the address of struct vgic_cpu */
107	add x3, x0, #VCPU_VGIC_CPU
108
109	/* We only restore a minimal set of registers */
110	ldr w4, [x3, #VGIC_V2_CPU_HCR]
111	ldr w5, [x3, #VGIC_V2_CPU_VMCR]
112	ldr w6, [x3, #VGIC_V2_CPU_APR]
113	CPU_BE( rev w4, w4 )
114	CPU_BE( rev w5, w5 )
115	CPU_BE( rev w6, w6 )
116
117	str w4, [x2, #GICH_HCR]
118	str w5, [x2, #GICH_VMCR]
119	str w6, [x2, #GICH_APR]
120
121	/* Restore list registers */
122	add x2, x2, #GICH_LR0
123	ldr w4, [x3, #VGIC_CPU_NR_LR]
124	add x3, x3, #VGIC_V2_CPU_LR
125	1: ldr w5, [x3], #4
126	CPU_BE( rev w5, w5 )
127	str w5, [x2], #4
128	sub w4, w4, #1
129	cbnz w4, 1b
130	2:
131	ret
132	ENDPROC(__restore_vgic_v2_state)
133
134	.popsection

arch/arm64/kvm/vgic-v3-switch.S+0−269

@@ -1,269 +0,0 @@
1	/*
2	* Copyright (C) 2012,2013 - ARM Ltd
3	* Author: Marc Zyngier <marc.zyngier@arm.com>
4	*
5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License version 2 as
7	* published by the Free Software Foundation.
8	*
9	* This program is distributed in the hope that it will be useful,
10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	* GNU General Public License for more details.
13	*
14	* You should have received a copy of the GNU General Public License
15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
16	*/
17
18	#include <linux/linkage.h>
19	#include <linux/irqchip/arm-gic-v3.h>
20
21	#include <asm/assembler.h>
22	#include <asm/memory.h>
23	#include <asm/asm-offsets.h>
24	#include <asm/kvm.h>
25	#include <asm/kvm_asm.h>
26	#include <asm/kvm_arm.h>
27
28	.text
29	.pushsection .hyp.text, "ax"
30
31	/*
32	* We store LRs in reverse order to let the CPU deal with streaming
33	* access. Use this macro to make it look saner...
34	*/
35	#define LR_OFFSET(n) (VGIC_V3_CPU_LR + (15 - n) * 8)
36
37	/*
38	* Save the VGIC CPU state into memory
39	* x0: Register pointing to VCPU struct
40	* Do not corrupt x1!!!
41	*/
42	.macro save_vgic_v3_state
43	// Compute the address of struct vgic_cpu
44	add x3, x0, #VCPU_VGIC_CPU
45
46	// Make sure stores to the GIC via the memory mapped interface
47	// are now visible to the system register interface
48	dsb st
49
50	// Save all interesting registers
51	mrs_s x5, ICH_VMCR_EL2
52	mrs_s x6, ICH_MISR_EL2
53	mrs_s x7, ICH_EISR_EL2
54	mrs_s x8, ICH_ELSR_EL2
55
56	str w5, [x3, #VGIC_V3_CPU_VMCR]
57	str w6, [x3, #VGIC_V3_CPU_MISR]
58	str w7, [x3, #VGIC_V3_CPU_EISR]
59	str w8, [x3, #VGIC_V3_CPU_ELRSR]
60
61	msr_s ICH_HCR_EL2, xzr
62
63	mrs_s x21, ICH_VTR_EL2
64	mvn w22, w21
65	ubfiz w23, w22, 2, 4 // w23 = (15 - ListRegs) * 4
66
67	adr x24, 1f
68	add x24, x24, x23
69	br x24
70
71	1:
72	mrs_s x20, ICH_LR15_EL2
73	mrs_s x19, ICH_LR14_EL2
74	mrs_s x18, ICH_LR13_EL2
75	mrs_s x17, ICH_LR12_EL2
76	mrs_s x16, ICH_LR11_EL2
77	mrs_s x15, ICH_LR10_EL2
78	mrs_s x14, ICH_LR9_EL2
79	mrs_s x13, ICH_LR8_EL2
80	mrs_s x12, ICH_LR7_EL2
81	mrs_s x11, ICH_LR6_EL2
82	mrs_s x10, ICH_LR5_EL2
83	mrs_s x9, ICH_LR4_EL2
84	mrs_s x8, ICH_LR3_EL2
85	mrs_s x7, ICH_LR2_EL2
86	mrs_s x6, ICH_LR1_EL2
87	mrs_s x5, ICH_LR0_EL2
88
89	adr x24, 1f
90	add x24, x24, x23
91	br x24
92
93	1:
94	str x20, [x3, #LR_OFFSET(15)]
95	str x19, [x3, #LR_OFFSET(14)]
96	str x18, [x3, #LR_OFFSET(13)]
97	str x17, [x3, #LR_OFFSET(12)]
98	str x16, [x3, #LR_OFFSET(11)]
99	str x15, [x3, #LR_OFFSET(10)]
100	str x14, [x3, #LR_OFFSET(9)]
101	str x13, [x3, #LR_OFFSET(8)]
102	str x12, [x3, #LR_OFFSET(7)]
103	str x11, [x3, #LR_OFFSET(6)]
104	str x10, [x3, #LR_OFFSET(5)]
105	str x9, [x3, #LR_OFFSET(4)]
106	str x8, [x3, #LR_OFFSET(3)]
107	str x7, [x3, #LR_OFFSET(2)]
108	str x6, [x3, #LR_OFFSET(1)]
109	str x5, [x3, #LR_OFFSET(0)]
110
111	tbnz w21, #29, 6f // 6 bits
112	tbz w21, #30, 5f // 5 bits
113	// 7 bits
114	mrs_s x20, ICH_AP0R3_EL2
115	str w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
116	mrs_s x19, ICH_AP0R2_EL2
117	str w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
118	6: mrs_s x18, ICH_AP0R1_EL2
119	str w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
120	5: mrs_s x17, ICH_AP0R0_EL2
121	str w17, [x3, #VGIC_V3_CPU_AP0R]
122
123	tbnz w21, #29, 6f // 6 bits
124	tbz w21, #30, 5f // 5 bits
125	// 7 bits
126	mrs_s x20, ICH_AP1R3_EL2
127	str w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
128	mrs_s x19, ICH_AP1R2_EL2
129	str w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
130	6: mrs_s x18, ICH_AP1R1_EL2
131	str w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
132	5: mrs_s x17, ICH_AP1R0_EL2
133	str w17, [x3, #VGIC_V3_CPU_AP1R]
134
135	// Restore SRE_EL1 access and re-enable SRE at EL1.
136	mrs_s x5, ICC_SRE_EL2
137	orr x5, x5, #ICC_SRE_EL2_ENABLE
138	msr_s ICC_SRE_EL2, x5
139	isb
140	mov x5, #1
141	msr_s ICC_SRE_EL1, x5
142	.endm
143
144	/*
145	* Restore the VGIC CPU state from memory
146	* x0: Register pointing to VCPU struct
147	*/
148	.macro restore_vgic_v3_state
149	// Compute the address of struct vgic_cpu
150	add x3, x0, #VCPU_VGIC_CPU
151
152	// Restore all interesting registers
153	ldr w4, [x3, #VGIC_V3_CPU_HCR]
154	ldr w5, [x3, #VGIC_V3_CPU_VMCR]
155	ldr w25, [x3, #VGIC_V3_CPU_SRE]
156
157	msr_s ICC_SRE_EL1, x25
158
159	// make sure SRE is valid before writing the other registers
160	isb
161
162	msr_s ICH_HCR_EL2, x4
163	msr_s ICH_VMCR_EL2, x5
164
165	mrs_s x21, ICH_VTR_EL2
166
167	tbnz w21, #29, 6f // 6 bits
168	tbz w21, #30, 5f // 5 bits
169	// 7 bits
170	ldr w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
171	msr_s ICH_AP1R3_EL2, x20
172	ldr w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
173	msr_s ICH_AP1R2_EL2, x19
174	6: ldr w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
175	msr_s ICH_AP1R1_EL2, x18
176	5: ldr w17, [x3, #VGIC_V3_CPU_AP1R]
177	msr_s ICH_AP1R0_EL2, x17
178
179	tbnz w21, #29, 6f // 6 bits
180	tbz w21, #30, 5f // 5 bits
181	// 7 bits
182	ldr w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
183	msr_s ICH_AP0R3_EL2, x20
184	ldr w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
185	msr_s ICH_AP0R2_EL2, x19
186	6: ldr w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
187	msr_s ICH_AP0R1_EL2, x18
188	5: ldr w17, [x3, #VGIC_V3_CPU_AP0R]
189	msr_s ICH_AP0R0_EL2, x17
190
191	and w22, w21, #0xf
192	mvn w22, w21
193	ubfiz w23, w22, 2, 4 // w23 = (15 - ListRegs) * 4
194
195	adr x24, 1f
196	add x24, x24, x23
197	br x24
198
199	1:
200	ldr x20, [x3, #LR_OFFSET(15)]
201	ldr x19, [x3, #LR_OFFSET(14)]
202	ldr x18, [x3, #LR_OFFSET(13)]
203	ldr x17, [x3, #LR_OFFSET(12)]
204	ldr x16, [x3, #LR_OFFSET(11)]
205	ldr x15, [x3, #LR_OFFSET(10)]
206	ldr x14, [x3, #LR_OFFSET(9)]
207	ldr x13, [x3, #LR_OFFSET(8)]
208	ldr x12, [x3, #LR_OFFSET(7)]
209	ldr x11, [x3, #LR_OFFSET(6)]
210	ldr x10, [x3, #LR_OFFSET(5)]
211	ldr x9, [x3, #LR_OFFSET(4)]
212	ldr x8, [x3, #LR_OFFSET(3)]
213	ldr x7, [x3, #LR_OFFSET(2)]
214	ldr x6, [x3, #LR_OFFSET(1)]
215	ldr x5, [x3, #LR_OFFSET(0)]
216
217	adr x24, 1f
218	add x24, x24, x23
219	br x24
220
221	1:
222	msr_s ICH_LR15_EL2, x20
223	msr_s ICH_LR14_EL2, x19
224	msr_s ICH_LR13_EL2, x18
225	msr_s ICH_LR12_EL2, x17
226	msr_s ICH_LR11_EL2, x16
227	msr_s ICH_LR10_EL2, x15
228	msr_s ICH_LR9_EL2, x14
229	msr_s ICH_LR8_EL2, x13
230	msr_s ICH_LR7_EL2, x12
231	msr_s ICH_LR6_EL2, x11
232	msr_s ICH_LR5_EL2, x10
233	msr_s ICH_LR4_EL2, x9
234	msr_s ICH_LR3_EL2, x8
235	msr_s ICH_LR2_EL2, x7
236	msr_s ICH_LR1_EL2, x6
237	msr_s ICH_LR0_EL2, x5
238
239	// Ensure that the above will have reached the
240	// (re)distributors. This ensure the guest will read
241	// the correct values from the memory-mapped interface.
242	isb
243	dsb sy
244
245	// Prevent the guest from touching the GIC system registers
246	// if SRE isn't enabled for GICv3 emulation
247	cbnz x25, 1f
248	mrs_s x5, ICC_SRE_EL2
249	and x5, x5, #~ICC_SRE_EL2_ENABLE
250	msr_s ICC_SRE_EL2, x5
251	1:
252	.endm
253
254	ENTRY(__save_vgic_v3_state)
255	save_vgic_v3_state
256	ret
257	ENDPROC(__save_vgic_v3_state)
258
259	ENTRY(__restore_vgic_v3_state)
260	restore_vgic_v3_state
261	ret
262	ENDPROC(__restore_vgic_v3_state)
263
264	ENTRY(__vgic_v3_get_ich_vtr_el2)
265	mrs_s x0, ICH_VTR_EL2
266	ret
267	ENDPROC(__vgic_v3_get_ich_vtr_el2)
268
269	.popsection

arch/powerpc/include/asm/kvm_host.h+4−0

@@ -50,6 +50,10 @@ · 49 unmodified lines
50	#define KVM_NR_IRQCHIPS 1	50	#define KVM_NR_IRQCHIPS 1
51	#define KVM_IRQCHIP_NUM_PINS 256	51	#define KVM_IRQCHIP_NUM_PINS 256
52		52
		53	/* PPC-specific vcpu->requests bit members */
		54	#define KVM_REQ_WATCHDOG 8
		55	#define KVM_REQ_EPR_EXIT 9
		56
53	#include <linux/mmu_notifier.h>	57	#include <linux/mmu_notifier.h>
54		58
55	#define KVM_ARCH_WANT_MMU_NOTIFIER	59	#define KVM_ARCH_WANT_MMU_NOTIFIER

arch/powerpc/kvm/book3s_hv.c+2−8

 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
 {
-	int r;
+	struct kvm_vcpu *ret;
-	struct kvm_vcpu *v, *ret = NULL;
 	mutex_lock(&kvm->lock);
-	kvm_for_each_vcpu(r, v, kvm) {
+	ret = kvm_get_vcpu_by_id(kvm, id);
-		if (v->vcpu_id == id) {
-			ret = v;
-			break;
-		}
-	}
 	mutex_unlock(&kvm->lock);
 	return ret;
 }

arch/powerpc/kvm/book3s_pr.c+2−2

@@ -512,7 +512,7 @@ static void kvmppc_patch_dcbz(struct kvm_vcpu vcpu, struct kvmppc_pte pte) · 511 unmodified lines
512	put_page(hpage);	512	put_page(hpage);
513	}	513	}
514		514
515	static int kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)	515	static bool kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
516	{	516	{
517	ulong mp_pa = vcpu->arch.magic_page_pa;	517	ulong mp_pa = vcpu->arch.magic_page_pa;
518		518
@@ -521,7 +521,7 @@ static int kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) · 2 unmodified lines
521		521
522	gpa &= ~0xFFFULL;	522	gpa &= ~0xFFFULL;
523	if (unlikely(mp_pa) && unlikely((mp_pa & KVM_PAM) == (gpa & KVM_PAM))) {	523	if (unlikely(mp_pa) && unlikely((mp_pa & KVM_PAM) == (gpa & KVM_PAM))) {
524	return 1;	524	return true;
525	}	525	}
526		526
527	return kvm_is_visible_gfn(vcpu->kvm, gpa >> PAGE_SHIFT);	527	return kvm_is_visible_gfn(vcpu->kvm, gpa >> PAGE_SHIFT);

arch/s390/include/asm/elf.h+7−0

 #define HWCAP_S390_TE		1024
 #define HWCAP_S390_VXRS		2048
+/* Internal bits, not exposed via elf */
+#define HWCAP_INT_SIE		1UL
 /*
  * These are used to set parameters in the core dumps.
  */
 extern unsigned long elf_hwcap;
 #define ELF_HWCAP (elf_hwcap)
+/* Internal hardware capabilities, not exposed via elf */
+extern unsigned long int_hwcap;
 /* This yields a string that ld.so will use to load implementation
    specific libraries for optimization.  This is more specific in
    intent than poking at uname or /proc/cpuinfo.

arch/s390/include/asm/kvm_host.h+50−7

 #include <asm/fpu/api.h>
 #include <asm/isc.h>
-#define KVM_MAX_VCPUS 64
+#define KVM_S390_BSCA_CPU_SLOTS 64
+#define KVM_S390_ESCA_CPU_SLOTS 248
+#define KVM_MAX_VCPUS KVM_S390_ESCA_CPU_SLOTS
 #define KVM_USER_MEM_SLOTS 32
 /*
 #define KVM_IRQCHIP_NUM_PINS 4096
 #define KVM_HALT_POLL_NS_DEFAULT 0
+/* s390-specific vcpu->requests bit members */
+#define KVM_REQ_ENABLE_IBS         8
+#define KVM_REQ_DISABLE_IBS        9
 #define SIGP_CTRL_C		0x80
 #define SIGP_CTRL_SCN_MASK	0x3f
-struct sca_entry {
+union bsca_sigp_ctrl {
+	__u8 value;
+	struct {
+		__u8 c : 1;
+		__u8 r : 1;
+		__u8 scn : 6;
+	};
+} __packed;
+union esca_sigp_ctrl {
+	__u16 value;
+	struct {
+		__u8 c : 1;
+		__u8 reserved: 7;
+		__u8 scn;
+	};
+} __packed;
+struct esca_entry {
+	union esca_sigp_ctrl sigp_ctrl;
+	__u16   reserved1[3];
+	__u64   sda;
+	__u64   reserved2[6];
+} __packed;
+struct bsca_entry {
 	__u8	reserved0;
-	__u8	sigp_ctrl;
+	union bsca_sigp_ctrl	sigp_ctrl;
 	__u16	reserved[3];
 	__u64	sda;
 	__u64	reserved2[2];
 	};
 };
-struct sca_block {
+struct bsca_block {
 	union ipte_control ipte_control;
 	__u64	reserved[5];
 	__u64	mcn;
 	__u64	reserved2;
-	struct sca_entry cpu[64];
+	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
 } __attribute__((packed));
+struct esca_block {
+	union ipte_control ipte_control;
+	__u64   reserved1[7];
+	__u64   mcn[4];
+	__u64   reserved2[20];
+	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];
+} __packed;
 #define CPUSTAT_STOPPED    0x80000000
 #define CPUSTAT_WAIT       0x10000000
 #define CPUSTAT_ECALL_PEND 0x08000000
 	__u64	pp;			/* 0x01de */
 	__u8	reserved1e6[2];		/* 0x01e6 */
 	__u64	itdba;			/* 0x01e8 */
-	__u8	reserved1f0[16];	/* 0x01f0 */
+	__u64   riccbd;			/* 0x01f0 */
+	__u8    reserved1f8[8];		/* 0x01f8 */
 } __attribute__((packed));
 struct kvm_s390_itdb {
 };
 struct kvm_arch{
-	struct sca_block *sca;
+	void *sca;
+	int use_esca;
+	rwlock_t sca_lock;
 	debug_info_t *dbf;
 	struct kvm_s390_float_interrupt float_int;
 	struct kvm_device *flic;
 	struct gmap *gmap;
+	unsigned long mem_limit;
 	int css_support;
 	int use_irqchip;
 	int use_cmma;

arch/s390/include/asm/sclp.h+7−1

 struct sclp_core_entry {
 	u8 core_id;
-	u8 reserved0[2];
+	u8 reserved0;
+	u8 : 4;
+	u8 sief2 : 1;
+	u8 : 3;
 	u8 : 3;
 	u8 siif : 1;
 	u8 sigpif : 1;
 	unsigned char has_sigpif : 1;
 	unsigned char has_core_type : 1;
 	unsigned char has_sprp : 1;
+	unsigned char has_hvs : 1;
+	unsigned char has_esca : 1;
+	unsigned char has_sief2 : 1;
 	unsigned int ibc;
 	unsigned int mtid;
 	unsigned int mtid_cp;

arch/s390/include/uapi/asm/kvm.h+5−0

 #define KVM_S390_VM_MEM_CLR_CMMA	1
 #define KVM_S390_VM_MEM_LIMIT_SIZE	2
+#define KVM_S390_NO_MEM_LIMIT		U64_MAX
 /* kvm attributes for KVM_S390_VM_TOD */
 #define KVM_S390_VM_TOD_LOW		0
 #define KVM_S390_VM_TOD_HIGH		1
 #define KVM_SYNC_ARCH0  (1UL << 4)
 #define KVM_SYNC_PFAULT (1UL << 5)
 #define KVM_SYNC_VRS    (1UL << 6)
+#define KVM_SYNC_RICCB  (1UL << 7)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
 	__u64 vrs[32][2];	/* vector registers */
 	__u8  reserved[512];	/* for future vector expansion */
 	__u32 fpc;	/* only valid with vector registers */
+	__u8 padding[52];	/* riccb needs to be 64byte aligned */
+	__u8 riccb[64];		/* runtime instrumentation controls block */
 };
 #define KVM_REG_S390_TODPR	(KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)

arch/s390/kernel/processor.c+6−0

 		"esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp",
 		"edat", "etf3eh", "highgprs", "te", "vx"
 	};
+	static const char * const int_hwcap_str[] = {
+		"sie"
+	};
 	unsigned long n = (unsigned long) v - 1;
 	int i;
 		for (i = 0; i < ARRAY_SIZE(hwcap_str); i++)
 			if (hwcap_str[i] && (elf_hwcap & (1UL << i)))
 				seq_printf(m, "%s ", hwcap_str[i]);
+		for (i = 0; i < ARRAY_SIZE(int_hwcap_str); i++)
+			if (int_hwcap_str[i] && (int_hwcap & (1UL << i)))
+				seq_printf(m, "%s ", int_hwcap_str[i]);
 		seq_puts(m, "\n");
 		show_cacheinfo(m);
 	}

arch/s390/kernel/setup.c+9−0

 unsigned long elf_hwcap __read_mostly = 0;
 char elf_platform[ELF_PLATFORM_SIZE];
+unsigned long int_hwcap = 0;
 int __initdata memory_end_set;
 unsigned long __initdata memory_end;
 unsigned long __initdata max_physmem_end;
 		strcpy(elf_platform, "z13");
 		break;
 	}
+	/*
+	 * Virtualization support HWCAP_INT_SIE is bit 0.
+	 */
+	if (sclp.has_sief2)
+		int_hwcap |= HWCAP_INT_SIE;
 	return 0;
 }
 arch_initcall(setup_hwcaps);

arch/s390/kvm/diag.c+3−8

 static int __diag_time_slice_end_directed(struct kvm_vcpu *vcpu)
 {
-	struct kvm *kvm = vcpu->kvm;
 	struct kvm_vcpu *tcpu;
 	int tid;
-	int i;
 	tid = vcpu->run->s.regs.gprs[(vcpu->arch.sie_block->ipa & 0xf0) >> 4];
 	vcpu->stat.diagnose_9c++;
 	if (tid == vcpu->vcpu_id)
 		return 0;
-	kvm_for_each_vcpu(i, tcpu, kvm)
+	tcpu = kvm_get_vcpu_by_id(vcpu->kvm, tid);
-		if (tcpu->vcpu_id == tid) {
+	if (tcpu)
-			kvm_vcpu_yield_to(tcpu);
+		kvm_vcpu_yield_to(tcpu);
-			break;
-		}
 	return 0;
 }

arch/s390/kvm/gaccess.c+27−11

 int ipte_lock_held(struct kvm_vcpu *vcpu)
 {
-	union ipte_control *ic = &vcpu->kvm->arch.sca->ipte_control;
+	if (vcpu->arch.sie_block->eca & 1) {
+		int rc;
-	if (vcpu->arch.sie_block->eca & 1)
+		read_lock(&vcpu->kvm->arch.sca_lock);
-		return ic->kh != 0;
+		rc = kvm_s390_get_ipte_control(vcpu->kvm)->kh != 0;
+		read_unlock(&vcpu->kvm->arch.sca_lock);
+		return rc;
+	}
 	return vcpu->kvm->arch.ipte_lock_count != 0;
 }
 	vcpu->kvm->arch.ipte_lock_count++;
 	if (vcpu->kvm->arch.ipte_lock_count > 1)
 		goto out;
-	ic = &vcpu->kvm->arch.sca->ipte_control;
+retry:
+	read_lock(&vcpu->kvm->arch.sca_lock);
+	ic = kvm_s390_get_ipte_control(vcpu->kvm);
 	do {
 		old = READ_ONCE(*ic);
-		while (old.k) {
+		if (old.k) {
+			read_unlock(&vcpu->kvm->arch.sca_lock);
 			cond_resched();
-			old = READ_ONCE(*ic);
+			goto retry;
 		}
 		new = old;
 		new.k = 1;
 	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	read_unlock(&vcpu->kvm->arch.sca_lock);
 out:
 	mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
 }
 	vcpu->kvm->arch.ipte_lock_count--;
 	if (vcpu->kvm->arch.ipte_lock_count)
 		goto out;
-	ic = &vcpu->kvm->arch.sca->ipte_control;
+	read_lock(&vcpu->kvm->arch.sca_lock);
+	ic = kvm_s390_get_ipte_control(vcpu->kvm);
 	do {
 		old = READ_ONCE(*ic);
 		new = old;
 		new.k = 0;
 	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	read_unlock(&vcpu->kvm->arch.sca_lock);
 	wake_up(&vcpu->kvm->arch.ipte_wq);
 out:
 	mutex_unlock(&vcpu->kvm->arch.ipte_mutex);
 {
 	union ipte_control old, new, *ic;
-	ic = &vcpu->kvm->arch.sca->ipte_control;
+retry:
+	read_lock(&vcpu->kvm->arch.sca_lock);
+	ic = kvm_s390_get_ipte_control(vcpu->kvm);
 	do {
 		old = READ_ONCE(*ic);
-		while (old.kg) {
+		if (old.kg) {
+			read_unlock(&vcpu->kvm->arch.sca_lock);
 			cond_resched();
-			old = READ_ONCE(*ic);
+			goto retry;
 		}
 		new = old;
 		new.k = 1;
 		new.kh++;
 	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	read_unlock(&vcpu->kvm->arch.sca_lock);
 }
 static void ipte_unlock_siif(struct kvm_vcpu *vcpu)
 {
 	union ipte_control old, new, *ic;
-	ic = &vcpu->kvm->arch.sca->ipte_control;
+	read_lock(&vcpu->kvm->arch.sca_lock);
+	ic = kvm_s390_get_ipte_control(vcpu->kvm);
 	do {
 		old = READ_ONCE(*ic);
 		new = old;
 		if (!new.kh)
 			new.k = 0;
 	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	read_unlock(&vcpu->kvm->arch.sca_lock);
 	if (!new.kh)
 		wake_up(&vcpu->kvm->arch.ipte_wq);
 }

arch/s390/kvm/intercept.c+3−4

 static int handle_noop(struct kvm_vcpu *vcpu)
 {
 	switch (vcpu->arch.sie_block->icptcode) {
-	case 0x0:
-		vcpu->stat.exit_null++;
-		break;
 	case 0x10:
 		vcpu->stat.exit_external_request++;
 		break;
 int kvm_handle_sie_intercept(struct kvm_vcpu *vcpu)
 {
+	if (kvm_is_ucontrol(vcpu->kvm))
+		return -EOPNOTSUPP;
 	switch (vcpu->arch.sie_block->icptcode) {
-	case 0x00:
 	case 0x10:
 	case 0x18:
 		return handle_noop(vcpu);

arch/s390/kvm/interrupt.c+106−27

@@ -34,6 +34,106 @@ · 33 unmodified lines
34	#define PFAULT_DONE 0x0680	34	#define PFAULT_DONE 0x0680
35	#define VIRTIO_PARAM 0x0d00	35	#define VIRTIO_PARAM 0x0d00
36		36
		37	/* handle external calls via sigp interpretation facility */
		38	static int sca_ext_call_pending(struct kvm_vcpu vcpu, int src_id)
		39	{
		40	int c, scn;
		41
		42	if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND))
		43	return 0;
		44
		45	read_lock(&vcpu->kvm->arch.sca_lock);
		46	if (vcpu->kvm->arch.use_esca) {
		47	struct esca_block *sca = vcpu->kvm->arch.sca;
		48	union esca_sigp_ctrl sigp_ctrl =
		49	sca->cpu[vcpu->vcpu_id].sigp_ctrl;
		50
		51	c = sigp_ctrl.c;
		52	scn = sigp_ctrl.scn;
		53	} else {
		54	struct bsca_block *sca = vcpu->kvm->arch.sca;
		55	union bsca_sigp_ctrl sigp_ctrl =
		56	sca->cpu[vcpu->vcpu_id].sigp_ctrl;
		57
		58	c = sigp_ctrl.c;
		59	scn = sigp_ctrl.scn;
		60	}
		61	read_unlock(&vcpu->kvm->arch.sca_lock);
		62
		63	if (src_id)
		64	*src_id = scn;
		65
		66	return c;
		67	}
		68
		69	static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
		70	{
		71	int expect, rc;
		72
		73	read_lock(&vcpu->kvm->arch.sca_lock);
		74	if (vcpu->kvm->arch.use_esca) {
		75	struct esca_block *sca = vcpu->kvm->arch.sca;
		76	union esca_sigp_ctrl *sigp_ctrl =
		77	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		78	union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
		79
		80	new_val.scn = src_id;
		81	new_val.c = 1;
		82	old_val.c = 0;
		83
		84	expect = old_val.value;
		85	rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
		86	} else {
		87	struct bsca_block *sca = vcpu->kvm->arch.sca;
		88	union bsca_sigp_ctrl *sigp_ctrl =
		89	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		90	union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
		91
		92	new_val.scn = src_id;
		93	new_val.c = 1;
		94	old_val.c = 0;
		95
		96	expect = old_val.value;
		97	rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
		98	}
		99	read_unlock(&vcpu->kvm->arch.sca_lock);
		100
		101	if (rc != expect) {
		102	/* another external call is pending */
		103	return -EBUSY;
		104	}
		105	atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags);
		106	return 0;
		107	}
		108
		109	static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
		110	{
		111	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
		112	int rc, expect;
		113
		114	atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags);
		115	read_lock(&vcpu->kvm->arch.sca_lock);
		116	if (vcpu->kvm->arch.use_esca) {
		117	struct esca_block *sca = vcpu->kvm->arch.sca;
		118	union esca_sigp_ctrl *sigp_ctrl =
		119	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		120	union esca_sigp_ctrl old = *sigp_ctrl;
		121
		122	expect = old.value;
		123	rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
		124	} else {
		125	struct bsca_block *sca = vcpu->kvm->arch.sca;
		126	union bsca_sigp_ctrl *sigp_ctrl =
		127	&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
		128	union bsca_sigp_ctrl old = *sigp_ctrl;
		129
		130	expect = old.value;
		131	rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
		132	}
		133	read_unlock(&vcpu->kvm->arch.sca_lock);
		134	WARN_ON(rc != expect); /* cannot clear? */
		135	}
		136
37	int psw_extint_disabled(struct kvm_vcpu *vcpu)	137	int psw_extint_disabled(struct kvm_vcpu *vcpu)
38	{	138	{
39	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);	139	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
@@ -792,13 +892,11 @@ static const deliver_irq_t deliver_irq_funcs[] = { · 752 unmodified lines
792	int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)	892	int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
793	{	893	{
794	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;	894	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
795	uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
796		895
797	if (!sclp.has_sigpif)	896	if (!sclp.has_sigpif)
798	return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);	897	return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
799		898
800	return (sigp_ctrl & SIGP_CTRL_C) &&	899	return sca_ext_call_pending(vcpu, NULL);
801	(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND);
802	}	900	}
803		901
804	int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)	902	int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
@@ -909,9 +1007,7 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) · 104 unmodified lines
909	memset(&li->irq, 0, sizeof(li->irq));	1007	memset(&li->irq, 0, sizeof(li->irq));
910	spin_unlock(&li->lock);	1008	spin_unlock(&li->lock);
911		1009
912	/* clear pending external calls set by sigp interpretation facility */	1010	sca_clear_ext_call(vcpu);
913	atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags);
914	vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl = 0;
915	}	1011	}
916		1012
917	int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)	1013	int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
@@ -1003,21 +1099,6 @@ static int __inject_pfault_init(struct kvm_vcpu vcpu, struct kvm_s390_irq irq) · 85 unmodified lines
1003	return 0;	1099	return 0;
1004	}	1100	}
1005		1101
1006	static int __inject_extcall_sigpif(struct kvm_vcpu *vcpu, uint16_t src_id)
1007	{
1008	unsigned char new_val, old_val;
1009	uint8_t *sigp_ctrl = &vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;
1010
1011	new_val = SIGP_CTRL_C \| (src_id & SIGP_CTRL_SCN_MASK);
1012	old_val = *sigp_ctrl & ~SIGP_CTRL_C;
1013	if (cmpxchg(sigp_ctrl, old_val, new_val) != old_val) {
1014	/* another external call is pending */
1015	return -EBUSY;
1016	}
1017	atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags);
1018	return 0;
1019	}
1020
1021	static int __inject_extcall(struct kvm_vcpu vcpu, struct kvm_s390_irq irq)	1102	static int __inject_extcall(struct kvm_vcpu vcpu, struct kvm_s390_irq irq)
1022	{	1103	{
1023	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;	1104	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
@@ -1034,7 +1115,7 @@ static int __inject_extcall(struct kvm_vcpu vcpu, struct kvm_s390_irq irq) · 10 unmodified lines
1034	return -EINVAL;	1115	return -EINVAL;
1035		1116
1036	if (sclp.has_sigpif)	1117	if (sclp.has_sigpif)
1037	return __inject_extcall_sigpif(vcpu, src_id);	1118	return sca_inject_ext_call(vcpu, src_id);
1038		1119
1039	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))	1120	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1040	return -EBUSY;	1121	return -EBUSY;
@@ -2203,7 +2284,7 @@ static void store_local_irq(struct kvm_s390_local_interrupt *li, · 1162 unmodified lines
2203		2284
2204	int kvm_s390_get_irq_state(struct kvm_vcpu vcpu, __u8 __user buf, int len)	2285	int kvm_s390_get_irq_state(struct kvm_vcpu vcpu, __u8 __user buf, int len)
2205	{	2286	{
2206	uint8_t sigp_ctrl = vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl;	2287	int scn;
2207	unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];	2288	unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)];
2208	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;	2289	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2209	unsigned long pending_irqs;	2290	unsigned long pending_irqs;
@@ -2243,14 +2324,12 @@ int kvm_s390_get_irq_state(struct kvm_vcpu vcpu, __u8 __user buf, int len) · 33 unmodified lines
2243	}	2324	}
2244	}	2325	}
2245		2326
2246	if ((sigp_ctrl & SIGP_CTRL_C) &&	2327	if (sca_ext_call_pending(vcpu, &scn)) {
2247	(atomic_read(&vcpu->arch.sie_block->cpuflags) &
2248	CPUSTAT_ECALL_PEND)) {
2249	if (n + sizeof(irq) > len)	2328	if (n + sizeof(irq) > len)
2250	return -ENOBUFS;	2329	return -ENOBUFS;
2251	memset(&irq, 0, sizeof(irq));	2330	memset(&irq, 0, sizeof(irq));
2252	irq.type = KVM_S390_INT_EXTERNAL_CALL;	2331	irq.type = KVM_S390_INT_EXTERNAL_CALL;
2253	irq.u.extcall.code = sigp_ctrl & SIGP_CTRL_SCN_MASK;	2332	irq.u.extcall.code = scn;
2254	if (copy_to_user(&buf[n], &irq, sizeof(irq)))	2333	if (copy_to_user(&buf[n], &irq, sizeof(irq)))
2255	return -EFAULT;	2334	return -EFAULT;
2256	n += sizeof(irq);	2335	n += sizeof(irq);

arch/s390/kvm/kvm-s390.c+219−81

@@ -246,7 +246,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 245 unmodified lines
246	break;	246	break;
247	case KVM_CAP_NR_VCPUS:	247	case KVM_CAP_NR_VCPUS:
248	case KVM_CAP_MAX_VCPUS:	248	case KVM_CAP_MAX_VCPUS:
249	r = KVM_MAX_VCPUS;	249	r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
		250	: KVM_S390_BSCA_CPU_SLOTS;
250	break;	251	break;
251	case KVM_CAP_NR_MEMSLOTS:	252	case KVM_CAP_NR_MEMSLOTS:
252	r = KVM_USER_MEM_SLOTS;	253	r = KVM_USER_MEM_SLOTS;
@@ -257,6 +258,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 4 unmodified lines
257	case KVM_CAP_S390_VECTOR_REGISTERS:	258	case KVM_CAP_S390_VECTOR_REGISTERS:
258	r = MACHINE_HAS_VX;	259	r = MACHINE_HAS_VX;
259	break;	260	break;
		261	case KVM_CAP_S390_RI:
		262	r = test_facility(64);
		263	break;
260	default:	264	default:
261	r = 0;	265	r = 0;
262	}	266	}
@@ -283,6 +287,8 @@ static void kvm_s390_sync_dirty_log(struct kvm *kvm, · 20 unmodified lines
283	}	287	}
284		288
285	/* Section: vm related */	289	/* Section: vm related */
		290	static void sca_del_vcpu(struct kvm_vcpu *vcpu);
		291
286	/*	292	/*
287	* Get (and clear) the dirty memory log for a memory slot.	293	* Get (and clear) the dirty memory log for a memory slot.
288	*/	294	*/
@@ -355,6 +361,20 @@ static int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap) · 66 unmodified lines
355	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",	361	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
356	r ? "(not available)" : "(success)");	362	r ? "(not available)" : "(success)");
357	break;	363	break;
		364	case KVM_CAP_S390_RI:
		365	r = -EINVAL;
		366	mutex_lock(&kvm->lock);
		367	if (atomic_read(&kvm->online_vcpus)) {
		368	r = -EBUSY;
		369	} else if (test_facility(64)) {
		370	set_kvm_facility(kvm->arch.model.fac->mask, 64);
		371	set_kvm_facility(kvm->arch.model.fac->list, 64);
		372	r = 0;
		373	}
		374	mutex_unlock(&kvm->lock);
		375	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
		376	r ? "(not available)" : "(success)");
		377	break;
358	case KVM_CAP_S390_USER_STSI:	378	case KVM_CAP_S390_USER_STSI:
359	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");	379	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
360	kvm->arch.user_stsi = 1;	380	kvm->arch.user_stsi = 1;
@@ -375,8 +395,8 @@ static int kvm_s390_get_mem_control(struct kvm kvm, struct kvm_device_attr att · 14 unmodified lines
375	case KVM_S390_VM_MEM_LIMIT_SIZE:	395	case KVM_S390_VM_MEM_LIMIT_SIZE:
376	ret = 0;	396	ret = 0;
377	VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",	397	VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
378	kvm->arch.gmap->asce_end);	398	kvm->arch.mem_limit);
379	if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))	399	if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
380	ret = -EFAULT;	400	ret = -EFAULT;
381	break;	401	break;
382	default:	402	default:
@@ -428,9 +448,17 @@ static int kvm_s390_set_mem_control(struct kvm kvm, struct kvm_device_attr att · 45 unmodified lines
428	if (get_user(new_limit, (u64 __user *)attr->addr))	448	if (get_user(new_limit, (u64 __user *)attr->addr))
429	return -EFAULT;	449	return -EFAULT;
430		450
431	if (new_limit > kvm->arch.gmap->asce_end)	451	if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
		452	new_limit > kvm->arch.mem_limit)
432	return -E2BIG;	453	return -E2BIG;
433		454
		455	if (!new_limit)
		456	return -EINVAL;
		457
		458	/* gmap_alloc takes last usable address */
		459	if (new_limit != KVM_S390_NO_MEM_LIMIT)
		460	new_limit -= 1;
		461
434	ret = -EBUSY;	462	ret = -EBUSY;
435	mutex_lock(&kvm->lock);	463	mutex_lock(&kvm->lock);
436	if (atomic_read(&kvm->online_vcpus) == 0) {	464	if (atomic_read(&kvm->online_vcpus) == 0) {
@@ -447,7 +475,9 @@ static int kvm_s390_set_mem_control(struct kvm kvm, struct kvm_device_attr att · 10 unmodified lines
447	}	475	}
448	}	476	}
449	mutex_unlock(&kvm->lock);	477	mutex_unlock(&kvm->lock);
450	VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);	478	VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
		479	VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
		480	(void *) kvm->arch.gmap->asce);
451	break;	481	break;
452	}	482	}
453	default:	483	default:
@@ -1024,7 +1054,7 @@ static int kvm_s390_apxa_installed(void) · 570 unmodified lines
1024	u8 config[128];	1054	u8 config[128];
1025	int cc;	1055	int cc;
1026		1056
1027	if (test_facility(2) && test_facility(12)) {	1057	if (test_facility(12)) {
1028	cc = kvm_s390_query_ap_config(config);	1058	cc = kvm_s390_query_ap_config(config);
1029		1059
1030	if (cc)	1060	if (cc)
@@ -1075,6 +1105,15 @@ static int kvm_s390_crypto_init(struct kvm *kvm) · 44 unmodified lines
1075	return 0;	1105	return 0;
1076	}	1106	}
1077		1107
		1108	static void sca_dispose(struct kvm *kvm)
		1109	{
		1110	if (kvm->arch.use_esca)
		1111	free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
		1112	else
		1113	free_page((unsigned long)(kvm->arch.sca));
		1114	kvm->arch.sca = NULL;
		1115	}
		1116
1078	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)	1117	int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1079	{	1118	{
1080	int i, rc;	1119	int i, rc;
@@ -1098,14 +1137,17 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 17 unmodified lines
1098		1137
1099	rc = -ENOMEM;	1138	rc = -ENOMEM;
1100		1139
1101	kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);	1140	kvm->arch.use_esca = 0; /* start with basic SCA */
		1141	rwlock_init(&kvm->arch.sca_lock);
		1142	kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1102	if (!kvm->arch.sca)	1143	if (!kvm->arch.sca)
1103	goto out_err;	1144	goto out_err;
1104	spin_lock(&kvm_lock);	1145	spin_lock(&kvm_lock);
1105	sca_offset += 16;	1146	sca_offset += 16;
1106	if (sca_offset + sizeof(struct sca_block) > PAGE_SIZE)	1147	if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1107	sca_offset = 0;	1148	sca_offset = 0;
1108	kvm->arch.sca = (struct sca_block ) ((char ) kvm->arch.sca + sca_offset);	1149	kvm->arch.sca = (struct bsca_block *)
		1150	((char *) kvm->arch.sca + sca_offset);
1109	spin_unlock(&kvm_lock);	1151	spin_unlock(&kvm_lock);
1110		1152
1111	sprintf(debug_name, "kvm-%u", current->pid);	1153	sprintf(debug_name, "kvm-%u", current->pid);
@@ -1157,8 +1199,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 45 unmodified lines
1157		1199
1158	if (type & KVM_VM_S390_UCONTROL) {	1200	if (type & KVM_VM_S390_UCONTROL) {
1159	kvm->arch.gmap = NULL;	1201	kvm->arch.gmap = NULL;
		1202	kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1160	} else {	1203	} else {
1161	kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);	1204	if (sclp.hamax == U64_MAX)
		1205	kvm->arch.mem_limit = TASK_MAX_SIZE;
		1206	else
		1207	kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
		1208	sclp.hamax + 1);
		1209	kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
1162	if (!kvm->arch.gmap)	1210	if (!kvm->arch.gmap)
1163	goto out_err;	1211	goto out_err;
1164	kvm->arch.gmap->private = kvm;	1212	kvm->arch.gmap->private = kvm;
@@ -1170,14 +1218,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 5 unmodified lines
1170	kvm->arch.epoch = 0;	1218	kvm->arch.epoch = 0;
1171		1219
1172	spin_lock_init(&kvm->arch.start_stop_lock);	1220	spin_lock_init(&kvm->arch.start_stop_lock);
1173	KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);	1221	KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1174		1222
1175	return 0;	1223	return 0;
1176	out_err:	1224	out_err:
1177	kfree(kvm->arch.crypto.crycb);	1225	kfree(kvm->arch.crypto.crycb);
1178	free_page((unsigned long)kvm->arch.model.fac);	1226	free_page((unsigned long)kvm->arch.model.fac);
1179	debug_unregister(kvm->arch.dbf);	1227	debug_unregister(kvm->arch.dbf);
1180	free_page((unsigned long)(kvm->arch.sca));	1228	sca_dispose(kvm);
1181	KVM_EVENT(3, "creation of vm failed: %d", rc);	1229	KVM_EVENT(3, "creation of vm failed: %d", rc);
1182	return rc;	1230	return rc;
1183	}	1231	}
@@ -1188,14 +1236,8 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) · 4 unmodified lines
1188	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);	1236	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1189	kvm_s390_clear_local_irqs(vcpu);	1237	kvm_s390_clear_local_irqs(vcpu);
1190	kvm_clear_async_pf_completion_queue(vcpu);	1238	kvm_clear_async_pf_completion_queue(vcpu);
1191	if (!kvm_is_ucontrol(vcpu->kvm)) {	1239	if (!kvm_is_ucontrol(vcpu->kvm))
1192	clear_bit(63 - vcpu->vcpu_id,	1240	sca_del_vcpu(vcpu);
1193	(unsigned long *) &vcpu->kvm->arch.sca->mcn);
1194	if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1195	(__u64) vcpu->arch.sie_block)
1196	vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1197	}
1198	smp_mb();
1199		1241
1200	if (kvm_is_ucontrol(vcpu->kvm))	1242	if (kvm_is_ucontrol(vcpu->kvm))
1201	gmap_free(vcpu->arch.gmap);	1243	gmap_free(vcpu->arch.gmap);
@@ -1228,14 +1270,14 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 26 unmodified lines
1228	{	1270	{
1229	kvm_free_vcpus(kvm);	1271	kvm_free_vcpus(kvm);
1230	free_page((unsigned long)kvm->arch.model.fac);	1272	free_page((unsigned long)kvm->arch.model.fac);
1231	free_page((unsigned long)(kvm->arch.sca));	1273	sca_dispose(kvm);
1232	debug_unregister(kvm->arch.dbf);	1274	debug_unregister(kvm->arch.dbf);
1233	kfree(kvm->arch.crypto.crycb);	1275	kfree(kvm->arch.crypto.crycb);
1234	if (!kvm_is_ucontrol(kvm))	1276	if (!kvm_is_ucontrol(kvm))
1235	gmap_free(kvm->arch.gmap);	1277	gmap_free(kvm->arch.gmap);
1236	kvm_s390_destroy_adapters(kvm);	1278	kvm_s390_destroy_adapters(kvm);
1237	kvm_s390_clear_float_irqs(kvm);	1279	kvm_s390_clear_float_irqs(kvm);
1238	KVM_EVENT(3, "vm 0x%p destroyed", kvm);	1280	KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1239	}	1281	}
1240		1282
1241	/* Section: vcpu related */	1283	/* Section: vcpu related */
@@ -1249,6 +1291,117 @@ static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) · 7 unmodified lines
1249	return 0;	1291	return 0;
1250	}	1292	}
1251		1293
		1294	static void sca_del_vcpu(struct kvm_vcpu *vcpu)
		1295	{
		1296	read_lock(&vcpu->kvm->arch.sca_lock);
		1297	if (vcpu->kvm->arch.use_esca) {
		1298	struct esca_block *sca = vcpu->kvm->arch.sca;
		1299
		1300	clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
		1301	sca->cpu[vcpu->vcpu_id].sda = 0;
		1302	} else {
		1303	struct bsca_block *sca = vcpu->kvm->arch.sca;
		1304
		1305	clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
		1306	sca->cpu[vcpu->vcpu_id].sda = 0;
		1307	}
		1308	read_unlock(&vcpu->kvm->arch.sca_lock);
		1309	}
		1310
		1311	static void sca_add_vcpu(struct kvm_vcpu *vcpu)
		1312	{
		1313	read_lock(&vcpu->kvm->arch.sca_lock);
		1314	if (vcpu->kvm->arch.use_esca) {
		1315	struct esca_block *sca = vcpu->kvm->arch.sca;
		1316
		1317	sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
		1318	vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
		1319	vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
		1320	vcpu->arch.sie_block->ecb2 \|= 0x04U;
		1321	set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
		1322	} else {
		1323	struct bsca_block *sca = vcpu->kvm->arch.sca;
		1324
		1325	sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
		1326	vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
		1327	vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
		1328	set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
		1329	}
		1330	read_unlock(&vcpu->kvm->arch.sca_lock);
		1331	}
		1332
		1333	/* Basic SCA to Extended SCA data copy routines */
		1334	static inline void sca_copy_entry(struct esca_entry d, struct bsca_entry s)
		1335	{
		1336	d->sda = s->sda;
		1337	d->sigp_ctrl.c = s->sigp_ctrl.c;
		1338	d->sigp_ctrl.scn = s->sigp_ctrl.scn;
		1339	}
		1340
		1341	static void sca_copy_b_to_e(struct esca_block d, struct bsca_block s)
		1342	{
		1343	int i;
		1344
		1345	d->ipte_control = s->ipte_control;
		1346	d->mcn[0] = s->mcn;
		1347	for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
		1348	sca_copy_entry(&d->cpu[i], &s->cpu[i]);
		1349	}
		1350
		1351	static int sca_switch_to_extended(struct kvm *kvm)
		1352	{
		1353	struct bsca_block *old_sca = kvm->arch.sca;
		1354	struct esca_block *new_sca;
		1355	struct kvm_vcpu *vcpu;
		1356	unsigned int vcpu_idx;
		1357	u32 scaol, scaoh;
		1358
		1359	new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL\|__GFP_ZERO);
		1360	if (!new_sca)
		1361	return -ENOMEM;
		1362
		1363	scaoh = (u32)((u64)(new_sca) >> 32);
		1364	scaol = (u32)(u64)(new_sca) & ~0x3fU;
		1365
		1366	kvm_s390_vcpu_block_all(kvm);
		1367	write_lock(&kvm->arch.sca_lock);
		1368
		1369	sca_copy_b_to_e(new_sca, old_sca);
		1370
		1371	kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
		1372	vcpu->arch.sie_block->scaoh = scaoh;
		1373	vcpu->arch.sie_block->scaol = scaol;
		1374	vcpu->arch.sie_block->ecb2 \|= 0x04U;
		1375	}
		1376	kvm->arch.sca = new_sca;
		1377	kvm->arch.use_esca = 1;
		1378
		1379	write_unlock(&kvm->arch.sca_lock);
		1380	kvm_s390_vcpu_unblock_all(kvm);
		1381
		1382	free_page((unsigned long)old_sca);
		1383
		1384	VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
		1385	old_sca, kvm->arch.sca);
		1386	return 0;
		1387	}
		1388
		1389	static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
		1390	{
		1391	int rc;
		1392
		1393	if (id < KVM_S390_BSCA_CPU_SLOTS)
		1394	return true;
		1395	if (!sclp.has_esca)
		1396	return false;
		1397
		1398	mutex_lock(&kvm->lock);
		1399	rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
		1400	mutex_unlock(&kvm->lock);
		1401
		1402	return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
		1403	}
		1404
1252	int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)	1405	int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1253	{	1406	{
1254	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;	1407	vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
@@ -1259,6 +1412,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) · 4 unmodified lines
1259	KVM_SYNC_CRS \|	1412	KVM_SYNC_CRS \|
1260	KVM_SYNC_ARCH0 \|	1413	KVM_SYNC_ARCH0 \|
1261	KVM_SYNC_PFAULT;	1414	KVM_SYNC_PFAULT;
		1415	if (test_kvm_facility(vcpu->kvm, 64))
		1416	vcpu->run->kvm_valid_regs \|= KVM_SYNC_RICCB;
1262	if (test_kvm_facility(vcpu->kvm, 129))	1417	if (test_kvm_facility(vcpu->kvm, 129))
1263	vcpu->run->kvm_valid_regs \|= KVM_SYNC_VRS;	1418	vcpu->run->kvm_valid_regs \|= KVM_SYNC_VRS;
1264		1419
@@ -1369,8 +1524,11 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 104 unmodified lines
1369	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;	1524	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1370	preempt_enable();	1525	preempt_enable();
1371	mutex_unlock(&vcpu->kvm->lock);	1526	mutex_unlock(&vcpu->kvm->lock);
1372	if (!kvm_is_ucontrol(vcpu->kvm))	1527	if (!kvm_is_ucontrol(vcpu->kvm)) {
1373	vcpu->arch.gmap = vcpu->kvm->arch.gmap;	1528	vcpu->arch.gmap = vcpu->kvm->arch.gmap;
		1529	sca_add_vcpu(vcpu);
		1530	}
		1531
1374	}	1532	}
1375		1533
1376	static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)	1534	static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
@@ -1439,10 +1597,13 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) · 62 unmodified lines
1439	vcpu->arch.sie_block->eca \|= 1;	1597	vcpu->arch.sie_block->eca \|= 1;
1440	if (sclp.has_sigpif)	1598	if (sclp.has_sigpif)
1441	vcpu->arch.sie_block->eca \|= 0x10000000U;	1599	vcpu->arch.sie_block->eca \|= 0x10000000U;
		1600	if (test_kvm_facility(vcpu->kvm, 64))
		1601	vcpu->arch.sie_block->ecb3 \|= 0x01;
1442	if (test_kvm_facility(vcpu->kvm, 129)) {	1602	if (test_kvm_facility(vcpu->kvm, 129)) {
1443	vcpu->arch.sie_block->eca \|= 0x00020000;	1603	vcpu->arch.sie_block->eca \|= 0x00020000;
1444	vcpu->arch.sie_block->ecd \|= 0x20000000;	1604	vcpu->arch.sie_block->ecd \|= 0x20000000;
1445	}	1605	}
		1606	vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1446	vcpu->arch.sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;	1607	vcpu->arch.sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;
1447		1608
1448	if (vcpu->kvm->arch.use_cmma) {	1609	if (vcpu->kvm->arch.use_cmma) {
@@ -1465,7 +1626,7 @@ struct kvm_vcpu kvm_arch_vcpu_create(struct kvm kvm, · 16 unmodified lines
1465	struct sie_page *sie_page;	1626	struct sie_page *sie_page;
1466	int rc = -EINVAL;	1627	int rc = -EINVAL;
1467		1628
1468	if (id >= KVM_MAX_VCPUS)	1629	if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1469	goto out;	1630	goto out;
1470		1631
1471	rc = -ENOMEM;	1632	rc = -ENOMEM;
@@ -1482,20 +1643,6 @@ struct kvm_vcpu kvm_arch_vcpu_create(struct kvm kvm, · 10 unmodified lines
1482	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;	1643	vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1483		1644
1484	vcpu->arch.sie_block->icpua = id;	1645	vcpu->arch.sie_block->icpua = id;
1485	if (!kvm_is_ucontrol(kvm)) {
1486	if (!kvm->arch.sca) {
1487	WARN_ON_ONCE(1);
1488	goto out_free_cpu;
1489	}
1490	if (!kvm->arch.sca->cpu[id].sda)
1491	kvm->arch.sca->cpu[id].sda =
1492	(__u64) vcpu->arch.sie_block;
1493	vcpu->arch.sie_block->scaoh =
1494	(__u32)(((__u64)kvm->arch.sca) >> 32);
1495	vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1496	set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1497	}
1498
1499	spin_lock_init(&vcpu->arch.local_int.lock);	1646	spin_lock_init(&vcpu->arch.local_int.lock);
1500	vcpu->arch.local_int.float_int = &kvm->arch.float_int;	1647	vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1501	vcpu->arch.local_int.wq = &vcpu->wq;	1648	vcpu->arch.local_int.wq = &vcpu->wq;
@@ -1509,15 +1656,13 @@ struct kvm_vcpu kvm_arch_vcpu_create(struct kvm kvm, · 7 unmodified lines
1509	*/	1656	*/
1510	vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,	1657	vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1511	GFP_KERNEL);	1658	GFP_KERNEL);
1512	if (!vcpu->arch.guest_fpregs.fprs) {	1659	if (!vcpu->arch.guest_fpregs.fprs)
1513	rc = -ENOMEM;
1514	goto out_free_sie_block;	1660	goto out_free_sie_block;
1515	}
1516		1661
1517	rc = kvm_vcpu_init(vcpu, kvm, id);	1662	rc = kvm_vcpu_init(vcpu, kvm, id);
1518	if (rc)	1663	if (rc)
1519	goto out_free_sie_block;	1664	goto out_free_sie_block;
1520	VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,	1665	VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1521	vcpu->arch.sie_block);	1666	vcpu->arch.sie_block);
1522	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);	1667	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1523		1668
@@ -2013,7 +2158,8 @@ static int vcpu_pre_run(struct kvm_vcpu *vcpu) · 489 unmodified lines
2013	*/	2158	*/
2014	kvm_check_async_pf_completion(vcpu);	2159	kvm_check_async_pf_completion(vcpu);
2015		2160
2016	memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);	2161	vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
		2162	vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2017		2163
2018	if (need_resched())	2164	if (need_resched())
2019	schedule();	2165	schedule();
@@ -2071,8 +2217,6 @@ static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu) · 51 unmodified lines
2071		2217
2072	static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)	2218	static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2073	{	2219	{
2074	int rc = -1;
2075
2076	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",	2220	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2077	vcpu->arch.sie_block->icptcode);	2221	vcpu->arch.sie_block->icptcode);
2078	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);	2222	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
@@ -2080,40 +2224,36 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) · 1 unmodified line
2080	if (guestdbg_enabled(vcpu))	2224	if (guestdbg_enabled(vcpu))
2081	kvm_s390_restore_guest_per_regs(vcpu);	2225	kvm_s390_restore_guest_per_regs(vcpu);
2082		2226
2083	if (exit_reason >= 0) {	2227	vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2084	rc = 0;	2228	vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
		2229
		2230	if (vcpu->arch.sie_block->icptcode > 0) {
		2231	int rc = kvm_handle_sie_intercept(vcpu);
		2232
		2233	if (rc != -EOPNOTSUPP)
		2234	return rc;
		2235	vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
		2236	vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
		2237	vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
		2238	vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
		2239	return -EREMOTE;
		2240	} else if (exit_reason != -EFAULT) {
		2241	vcpu->stat.exit_null++;
		2242	return 0;
2085	} else if (kvm_is_ucontrol(vcpu->kvm)) {	2243	} else if (kvm_is_ucontrol(vcpu->kvm)) {
2086	vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;	2244	vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2087	vcpu->run->s390_ucontrol.trans_exc_code =	2245	vcpu->run->s390_ucontrol.trans_exc_code =
2088	current->thread.gmap_addr;	2246	current->thread.gmap_addr;
2089	vcpu->run->s390_ucontrol.pgm_code = 0x10;	2247	vcpu->run->s390_ucontrol.pgm_code = 0x10;
2090	rc = -EREMOTE;	2248	return -EREMOTE;
2091
2092	} else if (current->thread.gmap_pfault) {	2249	} else if (current->thread.gmap_pfault) {
2093	trace_kvm_s390_major_guest_pfault(vcpu);	2250	trace_kvm_s390_major_guest_pfault(vcpu);
2094	current->thread.gmap_pfault = 0;	2251	current->thread.gmap_pfault = 0;
2095	if (kvm_arch_setup_async_pf(vcpu)) {	2252	if (kvm_arch_setup_async_pf(vcpu))
2096	rc = 0;	2253	return 0;
2097	} else {	2254	return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2098	gpa_t gpa = current->thread.gmap_addr;
2099	rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
2100	}
2101	}	2255	}
2102		2256	return vcpu_post_run_fault_in_sie(vcpu);
2103	if (rc == -1)
2104	rc = vcpu_post_run_fault_in_sie(vcpu);
2105
2106	memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
2107
2108	if (rc == 0) {
2109	if (kvm_is_ucontrol(vcpu->kvm))
2110	/* Don't exit for host interrupts. */
2111	rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
2112	else
2113	rc = kvm_handle_sie_intercept(vcpu);
2114	}
2115
2116	return rc;
2117	}	2257	}
2118		2258
2119	static int __vcpu_run(struct kvm_vcpu *vcpu)	2259	static int __vcpu_run(struct kvm_vcpu *vcpu)
@@ -2233,18 +2373,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 113 unmodified lines
2233	rc = 0;	2373	rc = 0;
2234	}	2374	}
2235		2375
2236	if (rc == -EOPNOTSUPP) {
2237	/* intercept cannot be handled in-kernel, prepare kvm-run */
2238	kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2239	kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2240	kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2241	kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2242	rc = 0;
2243	}
2244
2245	if (rc == -EREMOTE) {	2376	if (rc == -EREMOTE) {
2246	/* intercept was handled, but userspace support is needed	2377	/* userspace support is needed, kvm_run has been prepared */
2247	* kvm_run has been prepared by the handler */
2248	rc = 0;	2378	rc = 0;
2249	}	2379	}
2250		2380
@@ -2736,6 +2866,9 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, · 485 unmodified lines
2736	if (mem->memory_size & 0xffffful)	2866	if (mem->memory_size & 0xffffful)
2737	return -EINVAL;	2867	return -EINVAL;
2738		2868
		2869	if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
		2870	return -EINVAL;
		2871
2739	return 0;	2872	return 0;
2740	}	2873	}
2741		2874
@@ -2767,6 +2900,11 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, · 25 unmodified lines
2767		2900
2768	static int __init kvm_s390_init(void)	2901	static int __init kvm_s390_init(void)
2769	{	2902	{
		2903	if (!sclp.has_sief2) {
		2904	pr_info("SIE not available\n");
		2905	return -ENODEV;
		2906	}
		2907
2770	return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);	2908	return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2771	}	2909	}
2772		2910

arch/s390/kvm/kvm-s390.h+7−0

@@ -340,4 +340,11 @@ void kvm_s390_clear_bp_data(struct kvm_vcpu *vcpu); · 339 unmodified lines
340	void kvm_s390_prepare_debug_exit(struct kvm_vcpu *vcpu);	340	void kvm_s390_prepare_debug_exit(struct kvm_vcpu *vcpu);
341	void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu);	341	void kvm_s390_handle_per_event(struct kvm_vcpu *vcpu);
342		342
		343	/* support for Basic/Extended SCA handling */
		344	static inline union ipte_control kvm_s390_get_ipte_control(struct kvm kvm)
		345	{
		346	struct bsca_block sca = kvm->arch.sca; / SCA version doesn't matter */
		347
		348	return &sca->ipte_control;
		349	}
343	#endif	350	#endif

arch/s390/kvm/trace-s390.h+3−3

@@ -55,8 +55,8 @@ TRACE_EVENT(kvm_s390_create_vcpu, · 54 unmodified lines
55	__entry->sie_block = sie_block;	55	__entry->sie_block = sie_block;
56	),	56	),
57		57
58	TP_printk("create cpu %d at %p, sie block at %p", __entry->id,	58	TP_printk("create cpu %d at 0x%pK, sie block at 0x%pK",
59	__entry->vcpu, __entry->sie_block)	59	__entry->id, __entry->vcpu, __entry->sie_block)
60	);	60	);
61		61
62	TRACE_EVENT(kvm_s390_destroy_vcpu,	62	TRACE_EVENT(kvm_s390_destroy_vcpu,
@@ -254,7 +254,7 @@ TRACE_EVENT(kvm_s390_enable_css, · 191 unmodified lines
254	__entry->kvm = kvm;	254	__entry->kvm = kvm;
255	),	255	),
256		256
257	TP_printk("enabling channel I/O support (kvm @ %p)\n",	257	TP_printk("enabling channel I/O support (kvm @ %pK)\n",
258	__entry->kvm)	258	__entry->kvm)
259	);	259	);
260		260

arch/s390/mm/pgtable.c+2−2

@@ -133,7 +133,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) · 132 unmodified lines
133	/**	133	/**
134	* gmap_alloc - allocate a guest address space	134	* gmap_alloc - allocate a guest address space
135	* @mm: pointer to the parent mm_struct	135	* @mm: pointer to the parent mm_struct
136	* @limit: maximum size of the gmap address space	136	* @limit: maximum address of the gmap address space
137	*	137	*
138	* Returns a guest address space structure.	138	* Returns a guest address space structure.
139	*/	139	*/
@@ -402,7 +402,7 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from, · 262 unmodified lines
402	if ((from \| to \| len) & (PMD_SIZE - 1))	402	if ((from \| to \| len) & (PMD_SIZE - 1))
403	return -EINVAL;	403	return -EINVAL;
404	if (len == 0 \|\| from + len < from \|\| to + len < to \|\|	404	if (len == 0 \|\| from + len < from \|\| to + len < to \|\|
405	from + len > TASK_MAX_SIZE \|\| to + len > gmap->asce_end)	405	from + len - 1 > TASK_MAX_SIZE \|\| to + len - 1 > gmap->asce_end)
406	return -EINVAL;	406	return -EINVAL;
407		407
408	flush = 0;	408	flush = 0;

arch/x86/include/asm/kvm_host.h+70−5

 #include <linux/pvclock_gtod.h>
 #include <linux/clocksource.h>
 #include <linux/irqbypass.h>
+#include <linux/hyperv.h>
 #include <asm/pvclock-abi.h>
 #include <asm/desc.h>
 #define KVM_IRQCHIP_NUM_PINS  KVM_IOAPIC_NUM_PINS
+/* x86-specific vcpu->requests bit members */
+#define KVM_REQ_MIGRATE_TIMER      8
+#define KVM_REQ_REPORT_TPR_ACCESS  9
+#define KVM_REQ_TRIPLE_FAULT      10
+#define KVM_REQ_MMU_SYNC          11
+#define KVM_REQ_CLOCK_UPDATE      12
+#define KVM_REQ_DEACTIVATE_FPU    13
+#define KVM_REQ_EVENT             14
+#define KVM_REQ_APF_HALT          15
+#define KVM_REQ_STEAL_UPDATE      16
+#define KVM_REQ_NMI               17
+#define KVM_REQ_PMU               18
+#define KVM_REQ_PMI               19
+#define KVM_REQ_SMI               20
+#define KVM_REQ_MASTERCLOCK_UPDATE 21
+#define KVM_REQ_MCLOCK_INPROGRESS 22
+#define KVM_REQ_SCAN_IOAPIC       23
+#define KVM_REQ_GLOBAL_CLOCK_UPDATE 24
+#define KVM_REQ_APIC_PAGE_RELOAD  25
+#define KVM_REQ_HV_CRASH          26
+#define KVM_REQ_IOAPIC_EOI_EXIT   27
+#define KVM_REQ_HV_RESET          28
+#define KVM_REQ_HV_EXIT           29
+#define KVM_REQ_HV_STIMER         30
 #define CR0_RESERVED_BITS                                               \
 	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
 			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
 	};
 };
+struct kvm_rmap_head {
+	unsigned long val;
+};
 struct kvm_mmu_page {
 	struct list_head link;
 	struct hlist_node hash_link;
 	bool unsync;
 	int root_count;          /* Currently serving as active root */
 	unsigned int unsync_children;
-	unsigned long parent_ptes;	/* Reverse mapping for parent_pte */
+	struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
 	/* The page is obsolete if mmu_valid_gen != kvm->arch.mmu_valid_gen.  */
 	unsigned long mmu_valid_gen;
 	struct list_head head;
 };
+/* Hyper-V SynIC timer */
+struct kvm_vcpu_hv_stimer {
+	struct hrtimer timer;
+	int index;
+	u64 config;
+	u64 count;
+	u64 exp_time;
+	struct hv_message msg;
+	bool msg_pending;
+};
+/* Hyper-V synthetic interrupt controller (SynIC)*/
+struct kvm_vcpu_hv_synic {
+	u64 version;
+	u64 control;
+	u64 msg_page;
+	u64 evt_page;
+	atomic64_t sint[HV_SYNIC_SINT_COUNT];
+	atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
+	DECLARE_BITMAP(auto_eoi_bitmap, 256);
+	DECLARE_BITMAP(vec_bitmap, 256);
+	bool active;
+};
 /* Hyper-V per vcpu emulation context */
 struct kvm_vcpu_hv {
 	u64 hv_vapic;
 	s64 runtime_offset;
+	struct kvm_vcpu_hv_synic synic;
+	struct kvm_hyperv_exit exit;
+	struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
+	DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
 };
 struct kvm_vcpu_arch {
 	u64 efer;
 	u64 apic_base;
 	struct kvm_lapic *apic;    /* kernel irqchip context */
-	u64 eoi_exit_bitmap[4];
+	bool apicv_active;
+	DECLARE_BITMAP(ioapic_handled_vectors, 256);
 	unsigned long apic_attention;
 	int32_t apic_arb_prio;
 	int mp_state;
 };
 struct kvm_arch_memory_slot {
-	unsigned long *rmap[KVM_NR_PAGE_SIZES];
+	struct kvm_rmap_head *rmap[KVM_NR_PAGE_SIZES];
 	struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
 };
 	void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
 	void (*enable_irq_window)(struct kvm_vcpu *vcpu);
 	void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
-	int (*cpu_uses_apicv)(struct kvm_vcpu *vcpu);
+	bool (*get_enable_apicv)(void);
+	void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
 	void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
 	void (*hwapic_isr_update)(struct kvm *kvm, int isr);
-	void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu);
+	void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
 	void (*set_virtual_x2apic_mode)(struct kvm_vcpu *vcpu, bool set);
 	void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
 	void (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector);
 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
 				struct x86_exception *exception);
+void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu);
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code,
 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
+void kvm_make_mclock_inprogress_request(struct kvm *kvm);
+void kvm_make_scan_ioapic_request(struct kvm *kvm);
 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
 				     struct kvm_async_pf *work);
 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,

arch/x86/include/uapi/asm/hyperv.h+92−0

@@ -269,4 +269,96 @@ typedef struct _HV_REFERENCE_TSC_PAGE { · 268 unmodified lines
269	#define HV_SYNIC_SINT_AUTO_EOI (1ULL << 17)	269	#define HV_SYNIC_SINT_AUTO_EOI (1ULL << 17)
270	#define HV_SYNIC_SINT_VECTOR_MASK (0xFF)	270	#define HV_SYNIC_SINT_VECTOR_MASK (0xFF)
271		271
		272	#define HV_SYNIC_STIMER_COUNT (4)
		273
		274	/* Define synthetic interrupt controller message constants. */
		275	#define HV_MESSAGE_SIZE (256)
		276	#define HV_MESSAGE_PAYLOAD_BYTE_COUNT (240)
		277	#define HV_MESSAGE_PAYLOAD_QWORD_COUNT (30)
		278
		279	/* Define hypervisor message types. */
		280	enum hv_message_type {
		281	HVMSG_NONE = 0x00000000,
		282
		283	/* Memory access messages. */
		284	HVMSG_UNMAPPED_GPA = 0x80000000,
		285	HVMSG_GPA_INTERCEPT = 0x80000001,
		286
		287	/* Timer notification messages. */
		288	HVMSG_TIMER_EXPIRED = 0x80000010,
		289
		290	/* Error messages. */
		291	HVMSG_INVALID_VP_REGISTER_VALUE = 0x80000020,
		292	HVMSG_UNRECOVERABLE_EXCEPTION = 0x80000021,
		293	HVMSG_UNSUPPORTED_FEATURE = 0x80000022,
		294
		295	/* Trace buffer complete messages. */
		296	HVMSG_EVENTLOG_BUFFERCOMPLETE = 0x80000040,
		297
		298	/* Platform-specific processor intercept messages. */
		299	HVMSG_X64_IOPORT_INTERCEPT = 0x80010000,
		300	HVMSG_X64_MSR_INTERCEPT = 0x80010001,
		301	HVMSG_X64_CPUID_INTERCEPT = 0x80010002,
		302	HVMSG_X64_EXCEPTION_INTERCEPT = 0x80010003,
		303	HVMSG_X64_APIC_EOI = 0x80010004,
		304	HVMSG_X64_LEGACY_FP_ERROR = 0x80010005
		305	};
		306
		307	/* Define synthetic interrupt controller message flags. */
		308	union hv_message_flags {
		309	__u8 asu8;
		310	struct {
		311	__u8 msg_pending:1;
		312	__u8 reserved:7;
		313	};
		314	};
		315
		316	/* Define port identifier type. */
		317	union hv_port_id {
		318	__u32 asu32;
		319	struct {
		320	__u32 id:24;
		321	__u32 reserved:8;
		322	} u;
		323	};
		324
		325	/* Define synthetic interrupt controller message header. */
		326	struct hv_message_header {
		327	__u32 message_type;
		328	__u8 payload_size;
		329	union hv_message_flags message_flags;
		330	__u8 reserved[2];
		331	union {
		332	__u64 sender;
		333	union hv_port_id port;
		334	};
		335	};
		336
		337	/* Define synthetic interrupt controller message format. */
		338	struct hv_message {
		339	struct hv_message_header header;
		340	union {
		341	__u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
		342	} u;
		343	};
		344
		345	/* Define the synthetic interrupt message page layout. */
		346	struct hv_message_page {
		347	struct hv_message sint_message[HV_SYNIC_SINT_COUNT];
		348	};
		349
		350	/* Define timer message payload structure. */
		351	struct hv_timer_message_payload {
		352	__u32 timer_index;
		353	__u32 reserved;
		354	__u64 expiration_time; /* When the timer expired */
		355	__u64 delivery_time; /* When the message was delivered */
		356	};
		357
		358	#define HV_STIMER_ENABLE (1ULL << 0)
		359	#define HV_STIMER_PERIODIC (1ULL << 1)
		360	#define HV_STIMER_LAZY (1ULL << 2)
		361	#define HV_STIMER_AUTOENABLE (1ULL << 3)
		362	#define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F)
		363
272	#endif	364	#endif

arch/x86/kvm/hyperv.c+704−4

@@ -23,13 +23,665 @@ · 22 unmodified lines
23		23
24	#include "x86.h"	24	#include "x86.h"
25	#include "lapic.h"	25	#include "lapic.h"
		26	#include "ioapic.h"
26	#include "hyperv.h"	27	#include "hyperv.h"
27		28
28	#include <linux/kvm_host.h>	29	#include <linux/kvm_host.h>
		30	#include <linux/highmem.h>
		31	#include <asm/apicdef.h>
29	#include <trace/events/kvm.h>	32	#include <trace/events/kvm.h>
30		33
31	#include "trace.h"	34	#include "trace.h"
32		35
		36	static inline u64 synic_read_sint(struct kvm_vcpu_hv_synic *synic, int sint)
		37	{
		38	return atomic64_read(&synic->sint[sint]);
		39	}
		40
		41	static inline int synic_get_sint_vector(u64 sint_value)
		42	{
		43	if (sint_value & HV_SYNIC_SINT_MASKED)
		44	return -1;
		45	return sint_value & HV_SYNIC_SINT_VECTOR_MASK;
		46	}
		47
		48	static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic,
		49	int vector)
		50	{
		51	int i;
		52
		53	for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
		54	if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
		55	return true;
		56	}
		57	return false;
		58	}
		59
		60	static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic,
		61	int vector)
		62	{
		63	int i;
		64	u64 sint_value;
		65
		66	for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
		67	sint_value = synic_read_sint(synic, i);
		68	if (synic_get_sint_vector(sint_value) == vector &&
		69	sint_value & HV_SYNIC_SINT_AUTO_EOI)
		70	return true;
		71	}
		72	return false;
		73	}
		74
		75	static int synic_set_sint(struct kvm_vcpu_hv_synic *synic, int sint,
		76	u64 data, bool host)
		77	{
		78	int vector;
		79
		80	vector = data & HV_SYNIC_SINT_VECTOR_MASK;
		81	if (vector < 16 && !host)
		82	return 1;
		83	/*
		84	* Guest may configure multiple SINTs to use the same vector, so
		85	* we maintain a bitmap of vectors handled by synic, and a
		86	* bitmap of vectors with auto-eoi behavior. The bitmaps are
		87	* updated here, and atomically queried on fast paths.
		88	*/
		89
		90	atomic64_set(&synic->sint[sint], data);
		91
		92	if (synic_has_vector_connected(synic, vector))
		93	__set_bit(vector, synic->vec_bitmap);
		94	else
		95	__clear_bit(vector, synic->vec_bitmap);
		96
		97	if (synic_has_vector_auto_eoi(synic, vector))
		98	__set_bit(vector, synic->auto_eoi_bitmap);
		99	else
		100	__clear_bit(vector, synic->auto_eoi_bitmap);
		101
		102	/* Load SynIC vectors into EOI exit bitmap */
		103	kvm_make_request(KVM_REQ_SCAN_IOAPIC, synic_to_vcpu(synic));
		104	return 0;
		105	}
		106
		107	static struct kvm_vcpu_hv_synic synic_get(struct kvm kvm, u32 vcpu_id)
		108	{
		109	struct kvm_vcpu *vcpu;
		110	struct kvm_vcpu_hv_synic *synic;
		111
		112	if (vcpu_id >= atomic_read(&kvm->online_vcpus))
		113	return NULL;
		114	vcpu = kvm_get_vcpu(kvm, vcpu_id);
		115	if (!vcpu)
		116	return NULL;
		117	synic = vcpu_to_synic(vcpu);
		118	return (synic->active) ? synic : NULL;
		119	}
		120
		121	static void synic_clear_sint_msg_pending(struct kvm_vcpu_hv_synic *synic,
		122	u32 sint)
		123	{
		124	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
		125	struct page *page;
		126	gpa_t gpa;
		127	struct hv_message *msg;
		128	struct hv_message_page *msg_page;
		129
		130	gpa = synic->msg_page & PAGE_MASK;
		131	page = kvm_vcpu_gfn_to_page(vcpu, gpa >> PAGE_SHIFT);
		132	if (is_error_page(page)) {
		133	vcpu_err(vcpu, "Hyper-V SynIC can't get msg page, gpa 0x%llx\n",
		134	gpa);
		135	return;
		136	}
		137	msg_page = kmap_atomic(page);
		138
		139	msg = &msg_page->sint_message[sint];
		140	msg->header.message_flags.msg_pending = 0;
		141
		142	kunmap_atomic(msg_page);
		143	kvm_release_page_dirty(page);
		144	kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
		145	}
		146
		147	static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
		148	{
		149	struct kvm *kvm = vcpu->kvm;
		150	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
		151	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
		152	struct kvm_vcpu_hv_stimer *stimer;
		153	int gsi, idx, stimers_pending;
		154
		155	trace_kvm_hv_notify_acked_sint(vcpu->vcpu_id, sint);
		156
		157	if (synic->msg_page & HV_SYNIC_SIMP_ENABLE)
		158	synic_clear_sint_msg_pending(synic, sint);
		159
		160	/* Try to deliver pending Hyper-V SynIC timers messages */
		161	stimers_pending = 0;
		162	for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) {
		163	stimer = &hv_vcpu->stimer[idx];
		164	if (stimer->msg_pending &&
		165	(stimer->config & HV_STIMER_ENABLE) &&
		166	HV_STIMER_SINT(stimer->config) == sint) {
		167	set_bit(stimer->index,
		168	hv_vcpu->stimer_pending_bitmap);
		169	stimers_pending++;
		170	}
		171	}
		172	if (stimers_pending)
		173	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
		174
		175	idx = srcu_read_lock(&kvm->irq_srcu);
		176	gsi = atomic_read(&synic->sint_to_gsi[sint]);
		177	if (gsi != -1)
		178	kvm_notify_acked_gsi(kvm, gsi);
		179	srcu_read_unlock(&kvm->irq_srcu, idx);
		180	}
		181
		182	static void synic_exit(struct kvm_vcpu_hv_synic *synic, u32 msr)
		183	{
		184	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
		185	struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv;
		186
		187	hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNIC;
		188	hv_vcpu->exit.u.synic.msr = msr;
		189	hv_vcpu->exit.u.synic.control = synic->control;
		190	hv_vcpu->exit.u.synic.evt_page = synic->evt_page;
		191	hv_vcpu->exit.u.synic.msg_page = synic->msg_page;
		192
		193	kvm_make_request(KVM_REQ_HV_EXIT, vcpu);
		194	}
		195
		196	static int synic_set_msr(struct kvm_vcpu_hv_synic *synic,
		197	u32 msr, u64 data, bool host)
		198	{
		199	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
		200	int ret;
		201
		202	if (!synic->active)
		203	return 1;
		204
		205	trace_kvm_hv_synic_set_msr(vcpu->vcpu_id, msr, data, host);
		206
		207	ret = 0;
		208	switch (msr) {
		209	case HV_X64_MSR_SCONTROL:
		210	synic->control = data;
		211	if (!host)
		212	synic_exit(synic, msr);
		213	break;
		214	case HV_X64_MSR_SVERSION:
		215	if (!host) {
		216	ret = 1;
		217	break;
		218	}
		219	synic->version = data;
		220	break;
		221	case HV_X64_MSR_SIEFP:
		222	if (data & HV_SYNIC_SIEFP_ENABLE)
		223	if (kvm_clear_guest(vcpu->kvm,
		224	data & PAGE_MASK, PAGE_SIZE)) {
		225	ret = 1;
		226	break;
		227	}
		228	synic->evt_page = data;
		229	if (!host)
		230	synic_exit(synic, msr);
		231	break;
		232	case HV_X64_MSR_SIMP:
		233	if (data & HV_SYNIC_SIMP_ENABLE)
		234	if (kvm_clear_guest(vcpu->kvm,
		235	data & PAGE_MASK, PAGE_SIZE)) {
		236	ret = 1;
		237	break;
		238	}
		239	synic->msg_page = data;
		240	if (!host)
		241	synic_exit(synic, msr);
		242	break;
		243	case HV_X64_MSR_EOM: {
		244	int i;
		245
		246	for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
		247	kvm_hv_notify_acked_sint(vcpu, i);
		248	break;
		249	}
		250	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
		251	ret = synic_set_sint(synic, msr - HV_X64_MSR_SINT0, data, host);
		252	break;
		253	default:
		254	ret = 1;
		255	break;
		256	}
		257	return ret;
		258	}
		259
		260	static int synic_get_msr(struct kvm_vcpu_hv_synic synic, u32 msr, u64 pdata)
		261	{
		262	int ret;
		263
		264	if (!synic->active)
		265	return 1;
		266
		267	ret = 0;
		268	switch (msr) {
		269	case HV_X64_MSR_SCONTROL:
		270	*pdata = synic->control;
		271	break;
		272	case HV_X64_MSR_SVERSION:
		273	*pdata = synic->version;
		274	break;
		275	case HV_X64_MSR_SIEFP:
		276	*pdata = synic->evt_page;
		277	break;
		278	case HV_X64_MSR_SIMP:
		279	*pdata = synic->msg_page;
		280	break;
		281	case HV_X64_MSR_EOM:
		282	*pdata = 0;
		283	break;
		284	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
		285	*pdata = atomic64_read(&synic->sint[msr - HV_X64_MSR_SINT0]);
		286	break;
		287	default:
		288	ret = 1;
		289	break;
		290	}
		291	return ret;
		292	}
		293
		294	int synic_set_irq(struct kvm_vcpu_hv_synic *synic, u32 sint)
		295	{
		296	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
		297	struct kvm_lapic_irq irq;
		298	int ret, vector;
		299
		300	if (sint >= ARRAY_SIZE(synic->sint))
		301	return -EINVAL;
		302
		303	vector = synic_get_sint_vector(synic_read_sint(synic, sint));
		304	if (vector < 0)
		305	return -ENOENT;
		306
		307	memset(&irq, 0, sizeof(irq));
		308	irq.dest_id = kvm_apic_id(vcpu->arch.apic);
		309	irq.dest_mode = APIC_DEST_PHYSICAL;
		310	irq.delivery_mode = APIC_DM_FIXED;
		311	irq.vector = vector;
		312	irq.level = 1;
		313
		314	ret = kvm_irq_delivery_to_apic(vcpu->kvm, NULL, &irq, NULL);
		315	trace_kvm_hv_synic_set_irq(vcpu->vcpu_id, sint, irq.vector, ret);
		316	return ret;
		317	}
		318
		319	int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint)
		320	{
		321	struct kvm_vcpu_hv_synic *synic;
		322
		323	synic = synic_get(kvm, vcpu_id);
		324	if (!synic)
		325	return -EINVAL;
		326
		327	return synic_set_irq(synic, sint);
		328	}
		329
		330	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
		331	{
		332	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
		333	int i;
		334
		335	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
		336
		337	for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
		338	if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
		339	kvm_hv_notify_acked_sint(vcpu, i);
		340	}
		341
		342	static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vcpu_id, u32 sint, int gsi)
		343	{
		344	struct kvm_vcpu_hv_synic *synic;
		345
		346	synic = synic_get(kvm, vcpu_id);
		347	if (!synic)
		348	return -EINVAL;
		349
		350	if (sint >= ARRAY_SIZE(synic->sint_to_gsi))
		351	return -EINVAL;
		352
		353	atomic_set(&synic->sint_to_gsi[sint], gsi);
		354	return 0;
		355	}
		356
		357	void kvm_hv_irq_routing_update(struct kvm *kvm)
		358	{
		359	struct kvm_irq_routing_table *irq_rt;
		360	struct kvm_kernel_irq_routing_entry *e;
		361	u32 gsi;
		362
		363	irq_rt = srcu_dereference_check(kvm->irq_routing, &kvm->irq_srcu,
		364	lockdep_is_held(&kvm->irq_lock));
		365
		366	for (gsi = 0; gsi < irq_rt->nr_rt_entries; gsi++) {
		367	hlist_for_each_entry(e, &irq_rt->map[gsi], link) {
		368	if (e->type == KVM_IRQ_ROUTING_HV_SINT)
		369	kvm_hv_set_sint_gsi(kvm, e->hv_sint.vcpu,
		370	e->hv_sint.sint, gsi);
		371	}
		372	}
		373	}
		374
		375	static void synic_init(struct kvm_vcpu_hv_synic *synic)
		376	{
		377	int i;
		378
		379	memset(synic, 0, sizeof(*synic));
		380	synic->version = HV_SYNIC_VERSION_1;
		381	for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
		382	atomic64_set(&synic->sint[i], HV_SYNIC_SINT_MASKED);
		383	atomic_set(&synic->sint_to_gsi[i], -1);
		384	}
		385	}
		386
		387	static u64 get_time_ref_counter(struct kvm *kvm)
		388	{
		389	return div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
		390	}
		391
		392	static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
		393	bool vcpu_kick)
		394	{
		395	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
		396
		397	set_bit(stimer->index,
		398	vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);
		399	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
		400	if (vcpu_kick)
		401	kvm_vcpu_kick(vcpu);
		402	}
		403
		404	static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)
		405	{
		406	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
		407
		408	trace_kvm_hv_stimer_cleanup(stimer_to_vcpu(stimer)->vcpu_id,
		409	stimer->index);
		410
		411	hrtimer_cancel(&stimer->timer);
		412	clear_bit(stimer->index,
		413	vcpu_to_hv_vcpu(vcpu)->stimer_pending_bitmap);
		414	stimer->msg_pending = false;
		415	stimer->exp_time = 0;
		416	}
		417
		418	static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer)
		419	{
		420	struct kvm_vcpu_hv_stimer *stimer;
		421
		422	stimer = container_of(timer, struct kvm_vcpu_hv_stimer, timer);
		423	trace_kvm_hv_stimer_callback(stimer_to_vcpu(stimer)->vcpu_id,
		424	stimer->index);
		425	stimer_mark_pending(stimer, true);
		426
		427	return HRTIMER_NORESTART;
		428	}
		429
		430	/*
		431	* stimer_start() assumptions:
		432	* a) stimer->count is not equal to 0
		433	* b) stimer->config has HV_STIMER_ENABLE flag
		434	*/
		435	static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
		436	{
		437	u64 time_now;
		438	ktime_t ktime_now;
		439
		440	time_now = get_time_ref_counter(stimer_to_vcpu(stimer)->kvm);
		441	ktime_now = ktime_get();
		442
		443	if (stimer->config & HV_STIMER_PERIODIC) {
		444	if (stimer->exp_time) {
		445	if (time_now >= stimer->exp_time) {
		446	u64 remainder;
		447
		448	div64_u64_rem(time_now - stimer->exp_time,
		449	stimer->count, &remainder);
		450	stimer->exp_time =
		451	time_now + (stimer->count - remainder);
		452	}
		453	} else
		454	stimer->exp_time = time_now + stimer->count;
		455
		456	trace_kvm_hv_stimer_start_periodic(
		457	stimer_to_vcpu(stimer)->vcpu_id,
		458	stimer->index,
		459	time_now, stimer->exp_time);
		460
		461	hrtimer_start(&stimer->timer,
		462	ktime_add_ns(ktime_now,
		463	100 * (stimer->exp_time - time_now)),
		464	HRTIMER_MODE_ABS);
		465	return 0;
		466	}
		467	stimer->exp_time = stimer->count;
		468	if (time_now >= stimer->count) {
		469	/*
		470	* Expire timer according to Hypervisor Top-Level Functional
		471	* specification v4(15.3.1):
		472	* "If a one shot is enabled and the specified count is in
		473	* the past, it will expire immediately."
		474	*/
		475	stimer_mark_pending(stimer, false);
		476	return 0;
		477	}
		478
		479	trace_kvm_hv_stimer_start_one_shot(stimer_to_vcpu(stimer)->vcpu_id,
		480	stimer->index,
		481	time_now, stimer->count);
		482
		483	hrtimer_start(&stimer->timer,
		484	ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)),
		485	HRTIMER_MODE_ABS);
		486	return 0;
		487	}
		488
		489	static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
		490	bool host)
		491	{
		492	trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,
		493	stimer->index, config, host);
		494
		495	stimer_cleanup(stimer);
		496	if ((stimer->config & HV_STIMER_ENABLE) && HV_STIMER_SINT(config) == 0)
		497	config &= ~HV_STIMER_ENABLE;
		498	stimer->config = config;
		499	stimer_mark_pending(stimer, false);
		500	return 0;
		501	}
		502
		503	static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
		504	bool host)
		505	{
		506	trace_kvm_hv_stimer_set_count(stimer_to_vcpu(stimer)->vcpu_id,
		507	stimer->index, count, host);
		508
		509	stimer_cleanup(stimer);
		510	stimer->count = count;
		511	if (stimer->count == 0)
		512	stimer->config &= ~HV_STIMER_ENABLE;
		513	else if (stimer->config & HV_STIMER_AUTOENABLE)
		514	stimer->config \|= HV_STIMER_ENABLE;
		515	stimer_mark_pending(stimer, false);
		516	return 0;
		517	}
		518
		519	static int stimer_get_config(struct kvm_vcpu_hv_stimer stimer, u64 pconfig)
		520	{
		521	*pconfig = stimer->config;
		522	return 0;
		523	}
		524
		525	static int stimer_get_count(struct kvm_vcpu_hv_stimer stimer, u64 pcount)
		526	{
		527	*pcount = stimer->count;
		528	return 0;
		529	}
		530
		531	static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,
		532	struct hv_message *src_msg)
		533	{
		534	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
		535	struct page *page;
		536	gpa_t gpa;
		537	struct hv_message *dst_msg;
		538	int r;
		539	struct hv_message_page *msg_page;
		540
		541	if (!(synic->msg_page & HV_SYNIC_SIMP_ENABLE))
		542	return -ENOENT;
		543
		544	gpa = synic->msg_page & PAGE_MASK;
		545	page = kvm_vcpu_gfn_to_page(vcpu, gpa >> PAGE_SHIFT);
		546	if (is_error_page(page))
		547	return -EFAULT;
		548
		549	msg_page = kmap_atomic(page);
		550	dst_msg = &msg_page->sint_message[sint];
		551	if (sync_cmpxchg(&dst_msg->header.message_type, HVMSG_NONE,
		552	src_msg->header.message_type) != HVMSG_NONE) {
		553	dst_msg->header.message_flags.msg_pending = 1;
		554	r = -EAGAIN;
		555	} else {
		556	memcpy(&dst_msg->u.payload, &src_msg->u.payload,
		557	src_msg->header.payload_size);
		558	dst_msg->header.message_type = src_msg->header.message_type;
		559	dst_msg->header.payload_size = src_msg->header.payload_size;
		560	r = synic_set_irq(synic, sint);
		561	if (r >= 1)
		562	r = 0;
		563	else if (r == 0)
		564	r = -EFAULT;
		565	}
		566	kunmap_atomic(msg_page);
		567	kvm_release_page_dirty(page);
		568	kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
		569	return r;
		570	}
		571
		572	static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
		573	{
		574	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
		575	struct hv_message *msg = &stimer->msg;
		576	struct hv_timer_message_payload *payload =
		577	(struct hv_timer_message_payload *)&msg->u.payload;
		578
		579	payload->expiration_time = stimer->exp_time;
		580	payload->delivery_time = get_time_ref_counter(vcpu->kvm);
		581	return synic_deliver_msg(vcpu_to_synic(vcpu),
		582	HV_STIMER_SINT(stimer->config), msg);
		583	}
		584
		585	static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
		586	{
		587	int r;
		588
		589	stimer->msg_pending = true;
		590	r = stimer_send_msg(stimer);
		591	trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer)->vcpu_id,
		592	stimer->index, r);
		593	if (!r) {
		594	stimer->msg_pending = false;
		595	if (!(stimer->config & HV_STIMER_PERIODIC))
		596	stimer->config &= ~HV_STIMER_ENABLE;
		597	}
		598	}
		599
		600	void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
		601	{
		602	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
		603	struct kvm_vcpu_hv_stimer *stimer;
		604	u64 time_now, exp_time;
		605	int i;
		606
		607	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
		608	if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {
		609	stimer = &hv_vcpu->stimer[i];
		610	if (stimer->config & HV_STIMER_ENABLE) {
		611	exp_time = stimer->exp_time;
		612
		613	if (exp_time) {
		614	time_now =
		615	get_time_ref_counter(vcpu->kvm);
		616	if (time_now >= exp_time)
		617	stimer_expiration(stimer);
		618	}
		619
		620	if ((stimer->config & HV_STIMER_ENABLE) &&
		621	stimer->count)
		622	stimer_start(stimer);
		623	else
		624	stimer_cleanup(stimer);
		625	}
		626	}
		627	}
		628
		629	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu)
		630	{
		631	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
		632	int i;
		633
		634	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
		635	stimer_cleanup(&hv_vcpu->stimer[i]);
		636	}
		637
		638	static void stimer_prepare_msg(struct kvm_vcpu_hv_stimer *stimer)
		639	{
		640	struct hv_message *msg = &stimer->msg;
		641	struct hv_timer_message_payload *payload =
		642	(struct hv_timer_message_payload *)&msg->u.payload;
		643
		644	memset(&msg->header, 0, sizeof(msg->header));
		645	msg->header.message_type = HVMSG_TIMER_EXPIRED;
		646	msg->header.payload_size = sizeof(*payload);
		647
		648	payload->timer_index = stimer->index;
		649	payload->expiration_time = 0;
		650	payload->delivery_time = 0;
		651	}
		652
		653	static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index)
		654	{
		655	memset(stimer, 0, sizeof(*stimer));
		656	stimer->index = timer_index;
		657	hrtimer_init(&stimer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
		658	stimer->timer.function = stimer_timer_callback;
		659	stimer_prepare_msg(stimer);
		660	}
		661
		662	void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
		663	{
		664	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
		665	int i;
		666
		667	synic_init(&hv_vcpu->synic);
		668
		669	bitmap_zero(hv_vcpu->stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
		670	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
		671	stimer_init(&hv_vcpu->stimer[i], i);
		672	}
		673
		674	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu)
		675	{
		676	/*
		677	* Hyper-V SynIC auto EOI SINT's are
		678	* not compatible with APICV, so deactivate APICV
		679	*/
		680	kvm_vcpu_deactivate_apicv(vcpu);
		681	vcpu_to_synic(vcpu)->active = true;
		682	return 0;
		683	}
		684
33	static bool kvm_hv_msr_partition_wide(u32 msr)	685	static bool kvm_hv_msr_partition_wide(u32 msr)
34	{	686	{
35	bool r = false;	687	bool r = false;
@@ -226,6 +878,31 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) · 190 unmodified lines
226	return 1;	878	return 1;
227	hv->runtime_offset = data - current_task_runtime_100ns();	879	hv->runtime_offset = data - current_task_runtime_100ns();
228	break;	880	break;
		881	case HV_X64_MSR_SCONTROL:
		882	case HV_X64_MSR_SVERSION:
		883	case HV_X64_MSR_SIEFP:
		884	case HV_X64_MSR_SIMP:
		885	case HV_X64_MSR_EOM:
		886	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
		887	return synic_set_msr(vcpu_to_synic(vcpu), msr, data, host);
		888	case HV_X64_MSR_STIMER0_CONFIG:
		889	case HV_X64_MSR_STIMER1_CONFIG:
		890	case HV_X64_MSR_STIMER2_CONFIG:
		891	case HV_X64_MSR_STIMER3_CONFIG: {
		892	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
		893
		894	return stimer_set_config(vcpu_to_stimer(vcpu, timer_index),
		895	data, host);
		896	}
		897	case HV_X64_MSR_STIMER0_COUNT:
		898	case HV_X64_MSR_STIMER1_COUNT:
		899	case HV_X64_MSR_STIMER2_COUNT:
		900	case HV_X64_MSR_STIMER3_COUNT: {
		901	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
		902
		903	return stimer_set_count(vcpu_to_stimer(vcpu, timer_index),
		904	data, host);
		905	}
229	default:	906	default:
230	vcpu_unimpl(vcpu, "Hyper-V uhandled wrmsr: 0x%x data 0x%llx\n",	907	vcpu_unimpl(vcpu, "Hyper-V uhandled wrmsr: 0x%x data 0x%llx\n",
231	msr, data);	908	msr, data);
@@ -248,11 +925,9 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu vcpu, u32 msr, u64 pdata) · 16 unmodified lines
248	case HV_X64_MSR_HYPERCALL:	925	case HV_X64_MSR_HYPERCALL:
249	data = hv->hv_hypercall;	926	data = hv->hv_hypercall;
250	break;	927	break;
251	case HV_X64_MSR_TIME_REF_COUNT: {	928	case HV_X64_MSR_TIME_REF_COUNT:
252	data =	929	data = get_time_ref_counter(kvm);
253	div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
254	break;	930	break;
255	}
256	case HV_X64_MSR_REFERENCE_TSC:	931	case HV_X64_MSR_REFERENCE_TSC:
257	data = hv->hv_tsc_page;	932	data = hv->hv_tsc_page;
258	break;	933	break;
@@ -304,6 +979,31 @@ static int kvm_hv_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata) · 45 unmodified lines
304	case HV_X64_MSR_VP_RUNTIME:	979	case HV_X64_MSR_VP_RUNTIME:
305	data = current_task_runtime_100ns() + hv->runtime_offset;	980	data = current_task_runtime_100ns() + hv->runtime_offset;
306	break;	981	break;
		982	case HV_X64_MSR_SCONTROL:
		983	case HV_X64_MSR_SVERSION:
		984	case HV_X64_MSR_SIEFP:
		985	case HV_X64_MSR_SIMP:
		986	case HV_X64_MSR_EOM:
		987	case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
		988	return synic_get_msr(vcpu_to_synic(vcpu), msr, pdata);
		989	case HV_X64_MSR_STIMER0_CONFIG:
		990	case HV_X64_MSR_STIMER1_CONFIG:
		991	case HV_X64_MSR_STIMER2_CONFIG:
		992	case HV_X64_MSR_STIMER3_CONFIG: {
		993	int timer_index = (msr - HV_X64_MSR_STIMER0_CONFIG)/2;
		994
		995	return stimer_get_config(vcpu_to_stimer(vcpu, timer_index),
		996	pdata);
		997	}
		998	case HV_X64_MSR_STIMER0_COUNT:
		999	case HV_X64_MSR_STIMER1_COUNT:
		1000	case HV_X64_MSR_STIMER2_COUNT:
		1001	case HV_X64_MSR_STIMER3_COUNT: {
		1002	int timer_index = (msr - HV_X64_MSR_STIMER0_COUNT)/2;
		1003
		1004	return stimer_get_count(vcpu_to_stimer(vcpu, timer_index),
		1005	pdata);
		1006	}
307	default:	1007	default:
308	vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);	1008	vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
309	return 1;	1009	return 1;

arch/x86/kvm/hyperv.h+55−0

@@ -24,9 +24,64 @@ · 23 unmodified lines
24	#ifndef __ARCH_X86_KVM_HYPERV_H__	24	#ifndef __ARCH_X86_KVM_HYPERV_H__
25	#define __ARCH_X86_KVM_HYPERV_H__	25	#define __ARCH_X86_KVM_HYPERV_H__
26		26
		27	static inline struct kvm_vcpu_hv vcpu_to_hv_vcpu(struct kvm_vcpu vcpu)
		28	{
		29	return &vcpu->arch.hyperv;
		30	}
		31
		32	static inline struct kvm_vcpu hv_vcpu_to_vcpu(struct kvm_vcpu_hv hv_vcpu)
		33	{
		34	struct kvm_vcpu_arch *arch;
		35
		36	arch = container_of(hv_vcpu, struct kvm_vcpu_arch, hyperv);
		37	return container_of(arch, struct kvm_vcpu, arch);
		38	}
		39
		40	static inline struct kvm_vcpu_hv_synic vcpu_to_synic(struct kvm_vcpu vcpu)
		41	{
		42	return &vcpu->arch.hyperv.synic;
		43	}
		44
		45	static inline struct kvm_vcpu synic_to_vcpu(struct kvm_vcpu_hv_synic synic)
		46	{
		47	return hv_vcpu_to_vcpu(container_of(synic, struct kvm_vcpu_hv, synic));
		48	}
		49
27	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host);	50	int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host);
28	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata);	51	int kvm_hv_get_msr_common(struct kvm_vcpu vcpu, u32 msr, u64 pdata);
		52
29	bool kvm_hv_hypercall_enabled(struct kvm *kvm);	53	bool kvm_hv_hypercall_enabled(struct kvm *kvm);
30	int kvm_hv_hypercall(struct kvm_vcpu *vcpu);	54	int kvm_hv_hypercall(struct kvm_vcpu *vcpu);
31		55
		56	void kvm_hv_irq_routing_update(struct kvm *kvm);
		57	int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vcpu_id, u32 sint);
		58	void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector);
		59	int kvm_hv_activate_synic(struct kvm_vcpu *vcpu);
		60
		61	void kvm_hv_vcpu_init(struct kvm_vcpu *vcpu);
		62	void kvm_hv_vcpu_uninit(struct kvm_vcpu *vcpu);
		63
		64	static inline struct kvm_vcpu_hv_stimer vcpu_to_stimer(struct kvm_vcpu vcpu,
		65	int timer_index)
		66	{
		67	return &vcpu_to_hv_vcpu(vcpu)->stimer[timer_index];
		68	}
		69
		70	static inline struct kvm_vcpu stimer_to_vcpu(struct kvm_vcpu_hv_stimer stimer)
		71	{
		72	struct kvm_vcpu_hv *hv_vcpu;
		73
		74	hv_vcpu = container_of(stimer - stimer->index, struct kvm_vcpu_hv,
		75	stimer[0]);
		76	return hv_vcpu_to_vcpu(hv_vcpu);
		77	}
		78
		79	static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu)
		80	{
		81	return !bitmap_empty(vcpu->arch.hyperv.stimer_pending_bitmap,
		82	HV_SYNIC_STIMER_COUNT);
		83	}
		84
		85	void kvm_hv_process_stimers(struct kvm_vcpu *vcpu);
		86
32	#endif	87	#endif

arch/x86/kvm/ioapic.c+2−2

@@ -233,7 +233,7 @@ static void kvm_ioapic_inject_all(struct kvm_ioapic *ioapic, unsigned long irr) · 232 unmodified lines
233	}	233	}
234		234
235		235
236	void kvm_ioapic_scan_entry(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap)	236	void kvm_ioapic_scan_entry(struct kvm_vcpu vcpu, ulong ioapic_handled_vectors)
237	{	237	{
238	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;	238	struct kvm_ioapic *ioapic = vcpu->kvm->arch.vioapic;
239	union kvm_ioapic_redirect_entry *e;	239	union kvm_ioapic_redirect_entry *e;
@@ -250,7 +250,7 @@ void kvm_ioapic_scan_entry(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap) · 10 unmodified lines
250	(e->fields.trig_mode == IOAPIC_EDGE_TRIG &&	250	(e->fields.trig_mode == IOAPIC_EDGE_TRIG &&
251	kvm_apic_pending_eoi(vcpu, e->fields.vector)))	251	kvm_apic_pending_eoi(vcpu, e->fields.vector)))
252	__set_bit(e->fields.vector,	252	__set_bit(e->fields.vector,
253	(unsigned long *)eoi_exit_bitmap);	253	ioapic_handled_vectors);
254	}	254	}
255	}	255	}
256	spin_unlock(&ioapic->lock);	256	spin_unlock(&ioapic->lock);

arch/x86/kvm/ioapic.h+4−3

@@ -121,7 +121,8 @@ int kvm_irq_delivery_to_apic(struct kvm kvm, struct kvm_lapic src, · 120 unmodified lines
121	struct kvm_lapic_irq irq, unsigned long dest_map);	121	struct kvm_lapic_irq irq, unsigned long dest_map);
122	int kvm_get_ioapic(struct kvm kvm, struct kvm_ioapic_state state);	122	int kvm_get_ioapic(struct kvm kvm, struct kvm_ioapic_state state);
123	int kvm_set_ioapic(struct kvm kvm, struct kvm_ioapic_state state);	123	int kvm_set_ioapic(struct kvm kvm, struct kvm_ioapic_state state);
124	void kvm_ioapic_scan_entry(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap);	124	void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu,
125	void kvm_scan_ioapic_routes(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap);	125	ulong *ioapic_handled_vectors);
126		126	void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
		127	ulong *ioapic_handled_vectors);
127	#endif	128	#endif

arch/x86/kvm/irq.c+1−1

@@ -76,7 +76,7 @@ int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v) · 75 unmodified lines
76	if (kvm_cpu_has_extint(v))	76	if (kvm_cpu_has_extint(v))
77	return 1;	77	return 1;
78		78
79	if (kvm_vcpu_apic_vid_enabled(v))	79	if (kvm_vcpu_apicv_active(v))
80	return 0;	80	return 0;
81		81
82	return kvm_apic_has_interrupt(v) != -1; /* LAPIC */	82	return kvm_apic_has_interrupt(v) != -1; /* LAPIC */

arch/x86/kvm/irq_comm.c+38−3

 #include "lapic.h"
+#include "hyperv.h"
 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e,
 			   struct kvm *kvm, int irq_source_id, int level,
 			   bool line_status)
 	srcu_read_unlock(&kvm->irq_srcu, idx);
 }
+static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e,
+		    struct kvm *kvm, int irq_source_id, int level,
+		    bool line_status)
+{
+	if (!level)
+		return -1;
+	return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint);
+}
 int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e,
 			  const struct kvm_irq_routing_entry *ue)
 {
 		e->msi.address_hi = ue->u.msi.address_hi;
 		e->msi.data = ue->u.msi.data;
 		break;
+	case KVM_IRQ_ROUTING_HV_SINT:
+		e->set = kvm_hv_set_sint;
+		e->hv_sint.vcpu = ue->u.hv_sint.vcpu;
+		e->hv_sint.sint = ue->u.hv_sint.sint;
+		break;
 	default:
 		goto out;
 	}
 	return kvm_set_irq_routing(kvm, empty_routing, 0, 0);
 }
-void kvm_arch_irq_routing_update(struct kvm *kvm)
+void kvm_arch_post_irq_routing_update(struct kvm *kvm)
 {
 	if (ioapic_in_kernel(kvm) || !irqchip_in_kernel(kvm))
 		return;
 	kvm_make_scan_ioapic_request(kvm);
 }
-void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
+void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
+			    ulong *ioapic_handled_vectors)
 {
 	struct kvm *kvm = vcpu->kvm;
 	struct kvm_kernel_irq_routing_entry *entry;
 				u32 vector = entry->msi.data & 0xff;
 				__set_bit(vector,
-					  (unsigned long *) eoi_exit_bitmap);
+					  ioapic_handled_vectors);
 			}
 		}
 	}
 	srcu_read_unlock(&kvm->irq_srcu, idx);
 }
+int kvm_arch_set_irq(struct kvm_kernel_irq_routing_entry *irq, struct kvm *kvm,
+		     int irq_source_id, int level, bool line_status)
+{
+	switch (irq->type) {
+	case KVM_IRQ_ROUTING_HV_SINT:
+		return kvm_hv_set_sint(irq, kvm, irq_source_id, level,
+				       line_status);
+	default:
+		return -EWOULDBLOCK;
+	}
+}
+void kvm_arch_irq_routing_update(struct kvm *kvm)
+{
+	kvm_hv_irq_routing_update(kvm);
+}

arch/x86/kvm/lapic.c+23−17

 #include "trace.h"
 #include "x86.h"
 #include "cpuid.h"
+#include "hyperv.h"
 #ifndef CONFIG_X86_64
 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
 	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
-static inline int kvm_apic_id(struct kvm_lapic *apic)
-{
-	return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
-}
 /* The logical map is definitely wrong if we have multiple
  * modes at the same time.  (Physical map is always right.)
  */
 	if (!apic->irr_pending)
 		return -1;
-	kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
+	if (apic->vcpu->arch.apicv_active)
+		kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
 	result = apic_search_irr(apic);
 	ASSERT(result == -1 || result >= 16);
 	vcpu = apic->vcpu;
-	if (unlikely(kvm_vcpu_apic_vid_enabled(vcpu))) {
+	if (unlikely(vcpu->arch.apicv_active)) {
 		/* try to update RVI */
 		apic_clear_vector(vec, apic->regs + APIC_IRR);
 		kvm_make_request(KVM_REQ_EVENT, vcpu);
 	 * because the processor can modify ISR under the hood.  Instead
 	 * just set SVI.
 	 */
-	if (unlikely(kvm_x86_ops->hwapic_isr_update))
+	if (unlikely(vcpu->arch.apicv_active))
 		kvm_x86_ops->hwapic_isr_update(vcpu->kvm, vec);
 	else {
 		++apic->isr_count;
 	 * on the other hand isr_count and highest_isr_cache are unused
 	 * and must be left alone.
 	 */
-	if (unlikely(kvm_x86_ops->hwapic_isr_update))
+	if (unlikely(vcpu->arch.apicv_active))
 		kvm_x86_ops->hwapic_isr_update(vcpu->kvm,
 					       apic_find_highest_isr(apic));
 	else {
 				apic_clear_vector(vector, apic->regs + APIC_TMR);
 		}
-		if (kvm_x86_ops->deliver_posted_interrupt)
+		if (vcpu->arch.apicv_active)
 			kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
 		else {
 			apic_set_irr(vector, apic);
 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
 {
-	return test_bit(vector, (ulong *)apic->vcpu->arch.eoi_exit_bitmap);
+	return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
 }
 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
 	apic_clear_isr(vector, apic);
 	apic_update_ppr(apic);
+	if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
+		kvm_hv_synic_send_eoi(apic->vcpu, vector);
 	kvm_ioapic_send_eoi(apic, vector);
 	kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 	return vector;
 		int vec = reg & APIC_VECTOR_MASK;
 		void *bitmap = apic->regs + APIC_ISR;
-		if (kvm_x86_ops->deliver_posted_interrupt)
+		if (vcpu->arch.apicv_active)
 			bitmap = apic->regs + APIC_IRR;
 		if (apic_test_vector(vec, bitmap))
 		apic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
 		apic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
 	}
-	apic->irr_pending = kvm_vcpu_apic_vid_enabled(vcpu);
+	apic->irr_pending = vcpu->arch.apicv_active;
-	apic->isr_count = kvm_x86_ops->hwapic_isr_update ? 1 : 0;
+	apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
 	apic->highest_isr_cache = -1;
 	update_divide_count(apic);
 	atomic_set(&apic->lapic_timer.pending, 0);
 	apic_set_isr(vector, apic);
 	apic_update_ppr(apic);
 	apic_clear_irr(vector, apic);
+	if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
+		apic_clear_isr(vector, apic);
+		apic_update_ppr(apic);
+	}
 	return vector;
 }
 	update_divide_count(apic);
 	start_apic_timer(apic);
 	apic->irr_pending = true;
-	apic->isr_count = kvm_x86_ops->hwapic_isr_update ?
+	apic->isr_count = vcpu->arch.apicv_active ?
 				1 : count_vectors(apic->regs + APIC_ISR);
 	apic->highest_isr_cache = -1;
-	if (kvm_x86_ops->hwapic_irr_update)
+	if (vcpu->arch.apicv_active) {
 		kvm_x86_ops->hwapic_irr_update(vcpu,
 				apic_find_highest_irr(apic));
-	if (unlikely(kvm_x86_ops->hwapic_isr_update))
 		kvm_x86_ops->hwapic_isr_update(vcpu->kvm,
 				apic_find_highest_isr(apic));
+	}
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 	if (ioapic_in_kernel(vcpu->kvm))
 		kvm_rtc_eoi_tracking_restore_one(vcpu);

arch/x86/kvm/lapic.h+7−2

 	return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
 }
-static inline bool kvm_vcpu_apic_vid_enabled(struct kvm_vcpu *vcpu)
+static inline bool kvm_vcpu_apicv_active(struct kvm_vcpu *vcpu)
 {
-	return kvm_x86_ops->cpu_uses_apicv(vcpu);
+	return vcpu->arch.apic && vcpu->arch.apicv_active;
 }
 static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu)
 	return kvm_vcpu_has_lapic(vcpu) && test_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
 }
+static inline int kvm_apic_id(struct kvm_lapic *apic)
+{
+	return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
+}
 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector);
 void wait_lapic_expire(struct kvm_vcpu *vcpu);

arch/x86/kvm/mmu.c+183−214

@@ -311,11 +311,6 @@ static int is_large_pte(u64 pte) · 310 unmodified lines
311	return pte & PT_PAGE_SIZE_MASK;	311	return pte & PT_PAGE_SIZE_MASK;
312	}	312	}
313		313
314	static int is_rmap_spte(u64 pte)
315	{
316	return is_shadow_present_pte(pte);
317	}
318
319	static int is_last_spte(u64 pte, int level)	314	static int is_last_spte(u64 pte, int level)
320	{	315	{
321	if (level == PT_PAGE_TABLE_LEVEL)	316	if (level == PT_PAGE_TABLE_LEVEL)
@@ -540,7 +535,7 @@ static bool mmu_spte_update(u64 *sptep, u64 new_spte) · 218 unmodified lines
540	u64 old_spte = *sptep;	535	u64 old_spte = *sptep;
541	bool ret = false;	536	bool ret = false;
542		537
543	WARN_ON(!is_rmap_spte(new_spte));	538	WARN_ON(!is_shadow_present_pte(new_spte));
544		539
545	if (!is_shadow_present_pte(old_spte)) {	540	if (!is_shadow_present_pte(old_spte)) {
546	mmu_spte_set(sptep, new_spte);	541	mmu_spte_set(sptep, new_spte);
@@ -595,7 +590,7 @@ static int mmu_spte_clear_track_bits(u64 *sptep) · 48 unmodified lines
595	else	590	else
596	old_spte = __update_clear_spte_slow(sptep, 0ull);	591	old_spte = __update_clear_spte_slow(sptep, 0ull);
597		592
598	if (!is_rmap_spte(old_spte))	593	if (!is_shadow_present_pte(old_spte))
599	return 0;	594	return 0;
600		595
601	pfn = spte_to_pfn(old_spte);	596	pfn = spte_to_pfn(old_spte);
@@ -909,36 +904,35 @@ static int mapping_level(struct kvm_vcpu *vcpu, gfn_t large_gfn, · 307 unmodified lines
909	}	904	}
910		905
911	/*	906	/*
912	* Pte mapping structures:	907	* About rmap_head encoding:
913	*	908	*
914	* If pte_list bit zero is zero, then pte_list point to the spte.	909	* If the bit zero of rmap_head->val is clear, then it points to the only spte
915	*	910	* in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
916	* If pte_list bit zero is one, (then pte_list & ~1) points to a struct
917	* pte_list_desc containing more mappings.	911	* pte_list_desc containing more mappings.
918	*	912	*/
919	* Returns the number of pte entries before the spte was added or zero if	913
920	* the spte was not added.	914	/*
921	*	915	* Returns the number of pointers in the rmap chain, not counting the new one.
922	*/	916	*/
923	static int pte_list_add(struct kvm_vcpu vcpu, u64 spte,	917	static int pte_list_add(struct kvm_vcpu vcpu, u64 spte,
924	unsigned long *pte_list)	918	struct kvm_rmap_head *rmap_head)
925	{	919	{
926	struct pte_list_desc *desc;	920	struct pte_list_desc *desc;
927	int i, count = 0;	921	int i, count = 0;
928		922
929	if (!*pte_list) {	923	if (!rmap_head->val) {
930	rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);	924	rmap_printk("pte_list_add: %p %llx 0->1\n", spte, *spte);
931	*pte_list = (unsigned long)spte;	925	rmap_head->val = (unsigned long)spte;
932	} else if (!(*pte_list & 1)) {	926	} else if (!(rmap_head->val & 1)) {
933	rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);	927	rmap_printk("pte_list_add: %p %llx 1->many\n", spte, *spte);
934	desc = mmu_alloc_pte_list_desc(vcpu);	928	desc = mmu_alloc_pte_list_desc(vcpu);
935	desc->sptes[0] = (u64 )pte_list;	929	desc->sptes[0] = (u64 *)rmap_head->val;
936	desc->sptes[1] = spte;	930	desc->sptes[1] = spte;
937	*pte_list = (unsigned long)desc \| 1;	931	rmap_head->val = (unsigned long)desc \| 1;
938	++count;	932	++count;
939	} else {	933	} else {
940	rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);	934	rmap_printk("pte_list_add: %p %llx many->many\n", spte, *spte);
941	desc = (struct pte_list_desc )(pte_list & ~1ul);	935	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
942	while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {	936	while (desc->sptes[PTE_LIST_EXT-1] && desc->more) {
943	desc = desc->more;	937	desc = desc->more;
944	count += PTE_LIST_EXT;	938	count += PTE_LIST_EXT;
@@ -955,8 +949,9 @@ static int pte_list_add(struct kvm_vcpu vcpu, u64 spte, · 10 unmodified lines
955	}	949	}
956		950
957	static void	951	static void
958	pte_list_desc_remove_entry(unsigned long pte_list, struct pte_list_desc desc,	952	pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
959	int i, struct pte_list_desc *prev_desc)	953	struct pte_list_desc *desc, int i,
		954	struct pte_list_desc *prev_desc)
960	{	955	{
961	int j;	956	int j;
962		957
@@ -967,43 +962,43 @@ pte_list_desc_remove_entry(unsigned long pte_list, struct pte_list_desc desc, · 4 unmodified lines
967	if (j != 0)	962	if (j != 0)
968	return;	963	return;
969	if (!prev_desc && !desc->more)	964	if (!prev_desc && !desc->more)
970	*pte_list = (unsigned long)desc->sptes[0];	965	rmap_head->val = (unsigned long)desc->sptes[0];
971	else	966	else
972	if (prev_desc)	967	if (prev_desc)
973	prev_desc->more = desc->more;	968	prev_desc->more = desc->more;
974	else	969	else
975	*pte_list = (unsigned long)desc->more \| 1;	970	rmap_head->val = (unsigned long)desc->more \| 1;
976	mmu_free_pte_list_desc(desc);	971	mmu_free_pte_list_desc(desc);
977	}	972	}
978		973
979	static void pte_list_remove(u64 spte, unsigned long pte_list)	974	static void pte_list_remove(u64 spte, struct kvm_rmap_head rmap_head)
980	{	975	{
981	struct pte_list_desc *desc;	976	struct pte_list_desc *desc;
982	struct pte_list_desc *prev_desc;	977	struct pte_list_desc *prev_desc;
983	int i;	978	int i;
984		979
985	if (!*pte_list) {	980	if (!rmap_head->val) {
986	printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);	981	printk(KERN_ERR "pte_list_remove: %p 0->BUG\n", spte);
987	BUG();	982	BUG();
988	} else if (!(*pte_list & 1)) {	983	} else if (!(rmap_head->val & 1)) {
989	rmap_printk("pte_list_remove: %p 1->0\n", spte);	984	rmap_printk("pte_list_remove: %p 1->0\n", spte);
990	if ((u64 )pte_list != spte) {	985	if ((u64 *)rmap_head->val != spte) {
991	printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);	986	printk(KERN_ERR "pte_list_remove: %p 1->BUG\n", spte);
992	BUG();	987	BUG();
993	}	988	}
994	*pte_list = 0;	989	rmap_head->val = 0;
995	} else {	990	} else {
996	rmap_printk("pte_list_remove: %p many->many\n", spte);	991	rmap_printk("pte_list_remove: %p many->many\n", spte);
997	desc = (struct pte_list_desc )(pte_list & ~1ul);	992	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
998	prev_desc = NULL;	993	prev_desc = NULL;
999	while (desc) {	994	while (desc) {
1000	for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)	995	for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i) {
1001	if (desc->sptes[i] == spte) {	996	if (desc->sptes[i] == spte) {
1002	pte_list_desc_remove_entry(pte_list,	997	pte_list_desc_remove_entry(rmap_head,
1003	desc, i,	998	desc, i, prev_desc);
1004	prev_desc);
1005	return;	999	return;
1006	}	1000	}
		1001	}
1007	prev_desc = desc;	1002	prev_desc = desc;
1008	desc = desc->more;	1003	desc = desc->more;
1009	}	1004	}
@@ -1012,28 +1007,8 @@ static void pte_list_remove(u64 spte, unsigned long pte_list) · 2 unmodified lines
1012	}	1007	}
1013	}	1008	}
1014		1009
1015	typedef void (pte_list_walk_fn) (u64 spte);	1010	static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level,
1016	static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)	1011	struct kvm_memory_slot *slot)
1017	{
1018	struct pte_list_desc *desc;
1019	int i;
1020
1021	if (!*pte_list)
1022	return;
1023
1024	if (!(*pte_list & 1))
1025	return fn((u64 )pte_list);
1026
1027	desc = (struct pte_list_desc )(pte_list & ~1ul);
1028	while (desc) {
1029	for (i = 0; i < PTE_LIST_EXT && desc->sptes[i]; ++i)
1030	fn(desc->sptes[i]);
1031	desc = desc->more;
1032	}
1033	}
1034
1035	static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
1036	struct kvm_memory_slot *slot)
1037	{	1012	{
1038	unsigned long idx;	1013	unsigned long idx;
1039		1014
@@ -1041,10 +1016,8 @@ static unsigned long *__gfn_to_rmap(gfn_t gfn, int level, · 1 unmodified line
1041	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];	1016	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
1042	}	1017	}
1043		1018
1044	/*	1019	static struct kvm_rmap_head gfn_to_rmap(struct kvm kvm, gfn_t gfn,
1045	* Take gfn and return the reverse mapping to it.	1020	struct kvm_mmu_page *sp)
1046	*/
1047	static unsigned long gfn_to_rmap(struct kvm kvm, gfn_t gfn, struct kvm_mmu_page *sp)
1048	{	1021	{
1049	struct kvm_memslots *slots;	1022	struct kvm_memslots *slots;
1050	struct kvm_memory_slot *slot;	1023	struct kvm_memory_slot *slot;
@@ -1065,24 +1038,24 @@ static bool rmap_can_add(struct kvm_vcpu *vcpu) · 14 unmodified lines
1065	static int rmap_add(struct kvm_vcpu vcpu, u64 spte, gfn_t gfn)	1038	static int rmap_add(struct kvm_vcpu vcpu, u64 spte, gfn_t gfn)
1066	{	1039	{
1067	struct kvm_mmu_page *sp;	1040	struct kvm_mmu_page *sp;
1068	unsigned long *rmapp;	1041	struct kvm_rmap_head *rmap_head;
1069		1042
1070	sp = page_header(__pa(spte));	1043	sp = page_header(__pa(spte));
1071	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);	1044	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1072	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp);	1045	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1073	return pte_list_add(vcpu, spte, rmapp);	1046	return pte_list_add(vcpu, spte, rmap_head);
1074	}	1047	}
1075		1048
1076	static void rmap_remove(struct kvm kvm, u64 spte)	1049	static void rmap_remove(struct kvm kvm, u64 spte)
1077	{	1050	{
1078	struct kvm_mmu_page *sp;	1051	struct kvm_mmu_page *sp;
1079	gfn_t gfn;	1052	gfn_t gfn;
1080	unsigned long *rmapp;	1053	struct kvm_rmap_head *rmap_head;
1081		1054
1082	sp = page_header(__pa(spte));	1055	sp = page_header(__pa(spte));
1083	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);	1056	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1084	rmapp = gfn_to_rmap(kvm, gfn, sp);	1057	rmap_head = gfn_to_rmap(kvm, gfn, sp);
1085	pte_list_remove(spte, rmapp);	1058	pte_list_remove(spte, rmap_head);
1086	}	1059	}
1087		1060
1088	/*	1061	/*
@@ -1102,19 +1075,26 @@ struct rmap_iterator { · 13 unmodified lines
1102	*	1075	*
1103	* Returns sptep if found, NULL otherwise.	1076	* Returns sptep if found, NULL otherwise.
1104	*/	1077	*/
1105	static u64 rmap_get_first(unsigned long rmap, struct rmap_iterator iter)	1078	static u64 rmap_get_first(struct kvm_rmap_head rmap_head,
		1079	struct rmap_iterator *iter)
1106	{	1080	{
1107	if (!rmap)	1081	u64 *sptep;
		1082
		1083	if (!rmap_head->val)
1108	return NULL;	1084	return NULL;
1109		1085
1110	if (!(rmap & 1)) {	1086	if (!(rmap_head->val & 1)) {
1111	iter->desc = NULL;	1087	iter->desc = NULL;
1112	return (u64 *)rmap;	1088	sptep = (u64 *)rmap_head->val;
		1089	goto out;
1113	}	1090	}
1114		1091
1115	iter->desc = (struct pte_list_desc *)(rmap & ~1ul);	1092	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1116	iter->pos = 0;	1093	iter->pos = 0;
1117	return iter->desc->sptes[iter->pos];	1094	sptep = iter->desc->sptes[iter->pos];
		1095	out:
		1096	BUG_ON(!is_shadow_present_pte(*sptep));
		1097	return sptep;
1118	}	1098	}
1119		1099
1120	/*	1100	/*
@@ -1124,14 +1104,14 @@ static u64 rmap_get_first(unsigned long rmap, struct rmap_iterator iter) · 3 unmodified lines
1124	*/	1104	*/
1125	static u64 rmap_get_next(struct rmap_iterator iter)	1105	static u64 rmap_get_next(struct rmap_iterator iter)
1126	{	1106	{
		1107	u64 *sptep;
		1108
1127	if (iter->desc) {	1109	if (iter->desc) {
1128	if (iter->pos < PTE_LIST_EXT - 1) {	1110	if (iter->pos < PTE_LIST_EXT - 1) {
1129	u64 *sptep;
1130
1131	++iter->pos;	1111	++iter->pos;
1132	sptep = iter->desc->sptes[iter->pos];	1112	sptep = iter->desc->sptes[iter->pos];
1133	if (sptep)	1113	if (sptep)
1134	return sptep;	1114	goto out;
1135	}	1115	}
1136		1116
1137	iter->desc = iter->desc->more;	1117	iter->desc = iter->desc->more;
@@ -1139,17 +1119,20 @@ static u64 rmap_get_next(struct rmap_iterator iter) · 1 unmodified line
1139	if (iter->desc) {	1119	if (iter->desc) {
1140	iter->pos = 0;	1120	iter->pos = 0;
1141	/* desc->sptes[0] cannot be NULL */	1121	/* desc->sptes[0] cannot be NULL */
1142	return iter->desc->sptes[iter->pos];	1122	sptep = iter->desc->sptes[iter->pos];
		1123	goto out;
1143	}	1124	}
1144	}	1125	}
1145		1126
1146	return NULL;	1127	return NULL;
		1128	out:
		1129	BUG_ON(!is_shadow_present_pte(*sptep));
		1130	return sptep;
1147	}	1131	}
1148		1132
1149	#define for_each_rmap_spte(_rmap_, _iter_, _spte_) \	1133	#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_) \
1150	for (_spte_ = rmap_get_first(*_rmap_, _iter_); \	1134	for (_spte_ = rmap_get_first(_rmap_head_, _iter_); \
1151	_spte_ && ({BUG_ON(!is_shadow_present_pte(*_spte_)); 1;}); \	1135	_spte_; _spte_ = rmap_get_next(_iter_))
1152	_spte_ = rmap_get_next(_iter_))
1153		1136
1154	static void drop_spte(struct kvm kvm, u64 sptep)	1137	static void drop_spte(struct kvm kvm, u64 sptep)
1155	{	1138	{
@@ -1207,14 +1190,15 @@ static bool spte_write_protect(struct kvm kvm, u64 sptep, bool pt_protect) · 51 unmodified lines
1207	return mmu_spte_update(sptep, spte);	1190	return mmu_spte_update(sptep, spte);
1208	}	1191	}
1209		1192
1210	static bool __rmap_write_protect(struct kvm kvm, unsigned long rmapp,	1193	static bool __rmap_write_protect(struct kvm *kvm,
		1194	struct kvm_rmap_head *rmap_head,
1211	bool pt_protect)	1195	bool pt_protect)
1212	{	1196	{
1213	u64 *sptep;	1197	u64 *sptep;
1214	struct rmap_iterator iter;	1198	struct rmap_iterator iter;
1215	bool flush = false;	1199	bool flush = false;
1216		1200
1217	for_each_rmap_spte(rmapp, &iter, sptep)	1201	for_each_rmap_spte(rmap_head, &iter, sptep)
1218	flush \|= spte_write_protect(kvm, sptep, pt_protect);	1202	flush \|= spte_write_protect(kvm, sptep, pt_protect);
1219		1203
1220	return flush;	1204	return flush;
@@ -1231,13 +1215,13 @@ static bool spte_clear_dirty(struct kvm kvm, u64 sptep) · 10 unmodified lines
1231	return mmu_spte_update(sptep, spte);	1215	return mmu_spte_update(sptep, spte);
1232	}	1216	}
1233		1217
1234	static bool __rmap_clear_dirty(struct kvm kvm, unsigned long rmapp)	1218	static bool __rmap_clear_dirty(struct kvm kvm, struct kvm_rmap_head rmap_head)
1235	{	1219	{
1236	u64 *sptep;	1220	u64 *sptep;
1237	struct rmap_iterator iter;	1221	struct rmap_iterator iter;
1238	bool flush = false;	1222	bool flush = false;
1239		1223
1240	for_each_rmap_spte(rmapp, &iter, sptep)	1224	for_each_rmap_spte(rmap_head, &iter, sptep)
1241	flush \|= spte_clear_dirty(kvm, sptep);	1225	flush \|= spte_clear_dirty(kvm, sptep);
1242		1226
1243	return flush;	1227	return flush;
@@ -1254,13 +1238,13 @@ static bool spte_set_dirty(struct kvm kvm, u64 sptep) · 10 unmodified lines
1254	return mmu_spte_update(sptep, spte);	1238	return mmu_spte_update(sptep, spte);
1255	}	1239	}
1256		1240
1257	static bool __rmap_set_dirty(struct kvm kvm, unsigned long rmapp)	1241	static bool __rmap_set_dirty(struct kvm kvm, struct kvm_rmap_head rmap_head)
1258	{	1242	{
1259	u64 *sptep;	1243	u64 *sptep;
1260	struct rmap_iterator iter;	1244	struct rmap_iterator iter;
1261	bool flush = false;	1245	bool flush = false;
1262		1246
1263	for_each_rmap_spte(rmapp, &iter, sptep)	1247	for_each_rmap_spte(rmap_head, &iter, sptep)
1264	flush \|= spte_set_dirty(kvm, sptep);	1248	flush \|= spte_set_dirty(kvm, sptep);
1265		1249
1266	return flush;	1250	return flush;
@@ -1280,12 +1264,12 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, · 13 unmodified lines
1280	struct kvm_memory_slot *slot,	1264	struct kvm_memory_slot *slot,
1281	gfn_t gfn_offset, unsigned long mask)	1265	gfn_t gfn_offset, unsigned long mask)
1282	{	1266	{
1283	unsigned long *rmapp;	1267	struct kvm_rmap_head *rmap_head;
1284		1268
1285	while (mask) {	1269	while (mask) {
1286	rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),	1270	rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1287	PT_PAGE_TABLE_LEVEL, slot);	1271	PT_PAGE_TABLE_LEVEL, slot);
1288	__rmap_write_protect(kvm, rmapp, false);	1272	__rmap_write_protect(kvm, rmap_head, false);
1289		1273
1290	/* clear the first set bit */	1274	/* clear the first set bit */
1291	mask &= mask - 1;	1275	mask &= mask - 1;
@@ -1305,12 +1289,12 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, · 13 unmodified lines
1305	struct kvm_memory_slot *slot,	1289	struct kvm_memory_slot *slot,
1306	gfn_t gfn_offset, unsigned long mask)	1290	gfn_t gfn_offset, unsigned long mask)
1307	{	1291	{
1308	unsigned long *rmapp;	1292	struct kvm_rmap_head *rmap_head;
1309		1293
1310	while (mask) {	1294	while (mask) {
1311	rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),	1295	rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1312	PT_PAGE_TABLE_LEVEL, slot);	1296	PT_PAGE_TABLE_LEVEL, slot);
1313	__rmap_clear_dirty(kvm, rmapp);	1297	__rmap_clear_dirty(kvm, rmap_head);
1314		1298
1315	/* clear the first set bit */	1299	/* clear the first set bit */
1316	mask &= mask - 1;	1300	mask &= mask - 1;
@@ -1342,28 +1326,27 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, · 25 unmodified lines
1342	static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)	1326	static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
1343	{	1327	{
1344	struct kvm_memory_slot *slot;	1328	struct kvm_memory_slot *slot;
1345	unsigned long *rmapp;	1329	struct kvm_rmap_head *rmap_head;
1346	int i;	1330	int i;
1347	bool write_protected = false;	1331	bool write_protected = false;
1348		1332
1349	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);	1333	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1350		1334
1351	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {	1335	for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) {
1352	rmapp = __gfn_to_rmap(gfn, i, slot);	1336	rmap_head = __gfn_to_rmap(gfn, i, slot);
1353	write_protected \|= __rmap_write_protect(vcpu->kvm, rmapp, true);	1337	write_protected \|= __rmap_write_protect(vcpu->kvm, rmap_head, true);
1354	}	1338	}
1355		1339
1356	return write_protected;	1340	return write_protected;
1357	}	1341	}
1358		1342
1359	static bool kvm_zap_rmapp(struct kvm kvm, unsigned long rmapp)	1343	static bool kvm_zap_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head)
1360	{	1344	{
1361	u64 *sptep;	1345	u64 *sptep;
1362	struct rmap_iterator iter;	1346	struct rmap_iterator iter;
1363	bool flush = false;	1347	bool flush = false;
1364		1348
1365	while ((sptep = rmap_get_first(*rmapp, &iter))) {	1349	while ((sptep = rmap_get_first(rmap_head, &iter))) {
1366	BUG_ON(!(*sptep & PT_PRESENT_MASK));
1367	rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);	1350	rmap_printk("%s: spte %p %llx.\n", __func__, sptep, *sptep);
1368		1351
1369	drop_spte(kvm, sptep);	1352	drop_spte(kvm, sptep);
@@ -1373,14 +1356,14 @@ static bool kvm_zap_rmapp(struct kvm kvm, unsigned long rmapp) · 3 unmodified lines
1373	return flush;	1356	return flush;
1374	}	1357	}
1375		1358
1376	static int kvm_unmap_rmapp(struct kvm kvm, unsigned long rmapp,	1359	static int kvm_unmap_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head,
1377	struct kvm_memory_slot *slot, gfn_t gfn, int level,	1360	struct kvm_memory_slot *slot, gfn_t gfn, int level,
1378	unsigned long data)	1361	unsigned long data)
1379	{	1362	{
1380	return kvm_zap_rmapp(kvm, rmapp);	1363	return kvm_zap_rmapp(kvm, rmap_head);
1381	}	1364	}
1382		1365
1383	static int kvm_set_pte_rmapp(struct kvm kvm, unsigned long rmapp,	1366	static int kvm_set_pte_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head,
1384	struct kvm_memory_slot *slot, gfn_t gfn, int level,	1367	struct kvm_memory_slot *slot, gfn_t gfn, int level,
1385	unsigned long data)	1368	unsigned long data)
1386	{	1369	{
@@ -1395,7 +1378,7 @@ static int kvm_set_pte_rmapp(struct kvm kvm, unsigned long rmapp, · 8 unmodified lines
1395	new_pfn = pte_pfn(*ptep);	1378	new_pfn = pte_pfn(*ptep);
1396		1379
1397	restart:	1380	restart:
1398	for_each_rmap_spte(rmapp, &iter, sptep) {	1381	for_each_rmap_spte(rmap_head, &iter, sptep) {
1399	rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",	1382	rmap_printk("kvm_set_pte_rmapp: spte %p %llx gfn %llx (%d)\n",
1400	sptep, *sptep, gfn, level);	1383	sptep, *sptep, gfn, level);
1401		1384
@@ -1433,11 +1416,11 @@ struct slot_rmap_walk_iterator { · 31 unmodified lines
1433		1416
1434	/* output fields. */	1417	/* output fields. */
1435	gfn_t gfn;	1418	gfn_t gfn;
1436	unsigned long *rmap;	1419	struct kvm_rmap_head *rmap;
1437	int level;	1420	int level;
1438		1421
1439	/* private field. */	1422	/* private field. */
1440	unsigned long *end_rmap;	1423	struct kvm_rmap_head *end_rmap;
1441	};	1424	};
1442		1425
1443	static void	1426	static void
@@ -1496,7 +1479,7 @@ static int kvm_handle_hva_range(struct kvm *kvm, · 52 unmodified lines
1496	unsigned long end,	1479	unsigned long end,
1497	unsigned long data,	1480	unsigned long data,
1498	int (handler)(struct kvm kvm,	1481	int (handler)(struct kvm kvm,
1499	unsigned long *rmapp,	1482	struct kvm_rmap_head *rmap_head,
1500	struct kvm_memory_slot *slot,	1483	struct kvm_memory_slot *slot,
1501	gfn_t gfn,	1484	gfn_t gfn,
1502	int level,	1485	int level,
@@ -1540,7 +1523,8 @@ static int kvm_handle_hva_range(struct kvm *kvm, · 37 unmodified lines
1540		1523
1541	static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,	1524	static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1542	unsigned long data,	1525	unsigned long data,
1543	int (handler)(struct kvm kvm, unsigned long *rmapp,	1526	int (handler)(struct kvm kvm,
		1527	struct kvm_rmap_head *rmap_head,
1544	struct kvm_memory_slot *slot,	1528	struct kvm_memory_slot *slot,
1545	gfn_t gfn, int level,	1529	gfn_t gfn, int level,
1546	unsigned long data))	1530	unsigned long data))
@@ -1563,7 +1547,7 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) · 16 unmodified lines
1563	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);	1547	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1564	}	1548	}
1565		1549
1566	static int kvm_age_rmapp(struct kvm kvm, unsigned long rmapp,	1550	static int kvm_age_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head,
1567	struct kvm_memory_slot *slot, gfn_t gfn, int level,	1551	struct kvm_memory_slot *slot, gfn_t gfn, int level,
1568	unsigned long data)	1552	unsigned long data)
1569	{	1553	{
@@ -1573,18 +1557,19 @@ static int kvm_age_rmapp(struct kvm kvm, unsigned long rmapp, · 3 unmodified lines
1573		1557
1574	BUG_ON(!shadow_accessed_mask);	1558	BUG_ON(!shadow_accessed_mask);
1575		1559
1576	for_each_rmap_spte(rmapp, &iter, sptep)	1560	for_each_rmap_spte(rmap_head, &iter, sptep) {
1577	if (*sptep & shadow_accessed_mask) {	1561	if (*sptep & shadow_accessed_mask) {
1578	young = 1;	1562	young = 1;
1579	clear_bit((ffs(shadow_accessed_mask) - 1),	1563	clear_bit((ffs(shadow_accessed_mask) - 1),
1580	(unsigned long *)sptep);	1564	(unsigned long *)sptep);
1581	}	1565	}
		1566	}
1582		1567
1583	trace_kvm_age_page(gfn, level, slot, young);	1568	trace_kvm_age_page(gfn, level, slot, young);
1584	return young;	1569	return young;
1585	}	1570	}
1586		1571
1587	static int kvm_test_age_rmapp(struct kvm kvm, unsigned long rmapp,	1572	static int kvm_test_age_rmapp(struct kvm kvm, struct kvm_rmap_head rmap_head,
1588	struct kvm_memory_slot *slot, gfn_t gfn,	1573	struct kvm_memory_slot *slot, gfn_t gfn,
1589	int level, unsigned long data)	1574	int level, unsigned long data)
1590	{	1575	{
@@ -1600,11 +1585,12 @@ static int kvm_test_age_rmapp(struct kvm kvm, unsigned long rmapp, · 9 unmodified lines
1600	if (!shadow_accessed_mask)	1585	if (!shadow_accessed_mask)
1601	goto out;	1586	goto out;
1602		1587
1603	for_each_rmap_spte(rmapp, &iter, sptep)	1588	for_each_rmap_spte(rmap_head, &iter, sptep) {
1604	if (*sptep & shadow_accessed_mask) {	1589	if (*sptep & shadow_accessed_mask) {
1605	young = 1;	1590	young = 1;
1606	break;	1591	break;
1607	}	1592	}
		1593	}
1608	out:	1594	out:
1609	return young;	1595	return young;
1610	}	1596	}
@@ -1613,14 +1599,14 @@ out: · 2 unmodified lines
1613		1599
1614	static void rmap_recycle(struct kvm_vcpu vcpu, u64 spte, gfn_t gfn)	1600	static void rmap_recycle(struct kvm_vcpu vcpu, u64 spte, gfn_t gfn)
1615	{	1601	{
1616	unsigned long *rmapp;	1602	struct kvm_rmap_head *rmap_head;
1617	struct kvm_mmu_page *sp;	1603	struct kvm_mmu_page *sp;
1618		1604
1619	sp = page_header(__pa(spte));	1605	sp = page_header(__pa(spte));
1620		1606
1621	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp);	1607	rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
1622		1608
1623	kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, gfn, sp->role.level, 0);	1609	kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1624	kvm_flush_remote_tlbs(vcpu->kvm);	1610	kvm_flush_remote_tlbs(vcpu->kvm);
1625	}	1611	}
1626		1612
@@ -1720,8 +1706,7 @@ static void drop_parent_pte(struct kvm_mmu_page *sp, · 93 unmodified lines
1720	mmu_spte_clear_no_track(parent_pte);	1706	mmu_spte_clear_no_track(parent_pte);
1721	}	1707	}
1722		1708
1723	static struct kvm_mmu_page kvm_mmu_alloc_page(struct kvm_vcpu vcpu,	1709	static struct kvm_mmu_page kvm_mmu_alloc_page(struct kvm_vcpu vcpu, int direct)
1724	u64 *parent_pte, int direct)
1725	{	1710	{
1726	struct kvm_mmu_page *sp;	1711	struct kvm_mmu_page *sp;
1727		1712
@@ -1737,8 +1722,6 @@ static struct kvm_mmu_page kvm_mmu_alloc_page(struct kvm_vcpu vcpu, · 9 unmodified lines
1737	* this feature. See the comments in kvm_zap_obsolete_pages().	1722	* this feature. See the comments in kvm_zap_obsolete_pages().
1738	*/	1723	*/
1739	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);	1724	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
1740	sp->parent_ptes = 0;
1741	mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1742	kvm_mod_used_mmu_pages(vcpu->kvm, +1);	1725	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
1743	return sp;	1726	return sp;
1744	}	1727	}
@@ -1746,7 +1729,12 @@ static struct kvm_mmu_page kvm_mmu_alloc_page(struct kvm_vcpu vcpu, · 1 unmodified line
1746	static void mark_unsync(u64 *spte);	1729	static void mark_unsync(u64 *spte);
1747	static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)	1730	static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1748	{	1731	{
1749	pte_list_walk(&sp->parent_ptes, mark_unsync);	1732	u64 *sptep;
		1733	struct rmap_iterator iter;
		1734
		1735	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
		1736	mark_unsync(sptep);
		1737	}
1750	}	1738	}
1751		1739
1752	static void mark_unsync(u64 *spte)	1740	static void mark_unsync(u64 *spte)
@@ -1806,6 +1794,13 @@ static int mmu_pages_add(struct kvm_mmu_pages pvec, struct kvm_mmu_page sp, · 53 unmodified lines
1806	return (pvec->nr == KVM_PAGE_ARRAY_NR);	1794	return (pvec->nr == KVM_PAGE_ARRAY_NR);
1807	}	1795	}
1808		1796
		1797	static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
		1798	{
		1799	--sp->unsync_children;
		1800	WARN_ON((int)sp->unsync_children < 0);
		1801	__clear_bit(idx, sp->unsync_child_bitmap);
		1802	}
		1803
1809	static int __mmu_unsync_walk(struct kvm_mmu_page *sp,	1804	static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
1810	struct kvm_mmu_pages *pvec)	1805	struct kvm_mmu_pages *pvec)
1811	{	1806	{
@@ -1815,8 +1810,10 @@ static int __mmu_unsync_walk(struct kvm_mmu_page *sp, · 3 unmodified lines
1815	struct kvm_mmu_page *child;	1810	struct kvm_mmu_page *child;
1816	u64 ent = sp->spt[i];	1811	u64 ent = sp->spt[i];
1817		1812
1818	if (!is_shadow_present_pte(ent) \|\| is_large_pte(ent))	1813	if (!is_shadow_present_pte(ent) \|\| is_large_pte(ent)) {
1819	goto clear_child_bitmap;	1814	clear_unsync_child_bit(sp, i);
		1815	continue;
		1816	}
1820		1817
1821	child = page_header(ent & PT64_BASE_ADDR_MASK);	1818	child = page_header(ent & PT64_BASE_ADDR_MASK);
1822		1819
@@ -1825,28 +1822,21 @@ static int __mmu_unsync_walk(struct kvm_mmu_page *sp, · 2 unmodified lines
1825	return -ENOSPC;	1822	return -ENOSPC;
1826		1823
1827	ret = __mmu_unsync_walk(child, pvec);	1824	ret = __mmu_unsync_walk(child, pvec);
1828	if (!ret)	1825	if (!ret) {
1829	goto clear_child_bitmap;	1826	clear_unsync_child_bit(sp, i);
1830	else if (ret > 0)	1827	continue;
		1828	} else if (ret > 0) {
1831	nr_unsync_leaf += ret;	1829	nr_unsync_leaf += ret;
1832	else	1830	} else
1833	return ret;	1831	return ret;
1834	} else if (child->unsync) {	1832	} else if (child->unsync) {
1835	nr_unsync_leaf++;	1833	nr_unsync_leaf++;
1836	if (mmu_pages_add(pvec, child, i))	1834	if (mmu_pages_add(pvec, child, i))
1837	return -ENOSPC;	1835	return -ENOSPC;
1838	} else	1836	} else
1839	goto clear_child_bitmap;	1837	clear_unsync_child_bit(sp, i);
1840
1841	continue;
1842
1843	clear_child_bitmap:
1844	__clear_bit(i, sp->unsync_child_bitmap);
1845	sp->unsync_children--;
1846	WARN_ON((int)sp->unsync_children < 0);
1847	}	1838	}
1848		1839
1849
1850	return nr_unsync_leaf;	1840	return nr_unsync_leaf;
1851	}	1841	}
1852		1842
@@ -2009,9 +1999,7 @@ static void mmu_pages_clear_parents(struct mmu_page_path *parents) · 156 unmodified lines
2009	if (!sp)	1999	if (!sp)
2010	return;	2000	return;
2011		2001
2012	--sp->unsync_children;	2002	clear_unsync_child_bit(sp, idx);
2013	WARN_ON((int)sp->unsync_children < 0);
2014	__clear_bit(idx, sp->unsync_child_bitmap);
2015	level++;	2003	level++;
2016	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);	2004	} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
2017	}	2005	}
@@ -2053,14 +2041,6 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, · 35 unmodified lines
2053	}	2041	}
2054	}	2042	}
2055		2043
2056	static void init_shadow_page_table(struct kvm_mmu_page *sp)
2057	{
2058	int i;
2059
2060	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2061	sp->spt[i] = 0ull;
2062	}
2063
2064	static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)	2044	static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
2065	{	2045	{
2066	sp->write_flooding_count = 0;	2046	sp->write_flooding_count = 0;
@@ -2083,8 +2063,7 @@ static struct kvm_mmu_page kvm_mmu_get_page(struct kvm_vcpu vcpu, · 16 unmodified lines
2083	gva_t gaddr,	2063	gva_t gaddr,
2084	unsigned level,	2064	unsigned level,
2085	int direct,	2065	int direct,
2086	unsigned access,	2066	unsigned access)
2087	u64 *parent_pte)
2088	{	2067	{
2089	union kvm_mmu_page_role role;	2068	union kvm_mmu_page_role role;
2090	unsigned quadrant;	2069	unsigned quadrant;
@@ -2116,21 +2095,18 @@ static struct kvm_mmu_page kvm_mmu_get_page(struct kvm_vcpu vcpu, · 25 unmodified lines
2116	if (sp->unsync && kvm_sync_page_transient(vcpu, sp))	2095	if (sp->unsync && kvm_sync_page_transient(vcpu, sp))
2117	break;	2096	break;
2118		2097
2119	mmu_page_add_parent_pte(vcpu, sp, parent_pte);	2098	if (sp->unsync_children)
2120	if (sp->unsync_children) {
2121	kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);	2099	kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
2122	kvm_mmu_mark_parents_unsync(sp);
2123	} else if (sp->unsync)
2124	kvm_mmu_mark_parents_unsync(sp);
2125		2100
2126	__clear_sp_write_flooding_count(sp);	2101	__clear_sp_write_flooding_count(sp);
2127	trace_kvm_mmu_get_page(sp, false);	2102	trace_kvm_mmu_get_page(sp, false);
2128	return sp;	2103	return sp;
2129	}	2104	}
		2105
2130	++vcpu->kvm->stat.mmu_cache_miss;	2106	++vcpu->kvm->stat.mmu_cache_miss;
2131	sp = kvm_mmu_alloc_page(vcpu, parent_pte, direct);	2107
2132	if (!sp)	2108	sp = kvm_mmu_alloc_page(vcpu, direct);
2133	return sp;	2109
2134	sp->gfn = gfn;	2110	sp->gfn = gfn;
2135	sp->role = role;	2111	sp->role = role;
2136	hlist_add_head(&sp->hash_link,	2112	hlist_add_head(&sp->hash_link,
@@ -2144,7 +2120,7 @@ static struct kvm_mmu_page kvm_mmu_get_page(struct kvm_vcpu vcpu, · 7 unmodified lines
2144	account_shadowed(vcpu->kvm, sp);	2120	account_shadowed(vcpu->kvm, sp);
2145	}	2121	}
2146	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;	2122	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
2147	init_shadow_page_table(sp);	2123	clear_page(sp->spt);
2148	trace_kvm_mmu_get_page(sp, true);	2124	trace_kvm_mmu_get_page(sp, true);
2149	return sp;	2125	return sp;
2150	}	2126	}
@@ -2198,7 +2174,8 @@ static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator) · 47 unmodified lines
2198	return __shadow_walk_next(iterator, *iterator->sptep);	2174	return __shadow_walk_next(iterator, *iterator->sptep);
2199	}	2175	}
2200		2176
2201	static void link_shadow_page(u64 sptep, struct kvm_mmu_page sp, bool accessed)	2177	static void link_shadow_page(struct kvm_vcpu vcpu, u64 sptep,
		2178	struct kvm_mmu_page *sp)
2202	{	2179	{
2203	u64 spte;	2180	u64 spte;
2204		2181
@@ -2206,12 +2183,14 @@ static void link_shadow_page(u64 sptep, struct kvm_mmu_page sp, bool accessed) · 1 unmodified line
2206	VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);	2183	VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);
2207		2184
2208	spte = __pa(sp->spt) \| PT_PRESENT_MASK \| PT_WRITABLE_MASK \|	2185	spte = __pa(sp->spt) \| PT_PRESENT_MASK \| PT_WRITABLE_MASK \|
2209	shadow_user_mask \| shadow_x_mask;	2186	shadow_user_mask \| shadow_x_mask \| shadow_accessed_mask;
2210
2211	if (accessed)
2212	spte \|= shadow_accessed_mask;
2213		2187
2214	mmu_spte_set(sptep, spte);	2188	mmu_spte_set(sptep, spte);
		2189
		2190	mmu_page_add_parent_pte(vcpu, sp, sptep);
		2191
		2192	if (sp->unsync_children \|\| sp->unsync)
		2193	mark_unsync(sptep);
2215	}	2194	}
2216		2195
2217	static void validate_direct_spte(struct kvm_vcpu vcpu, u64 sptep,	2196	static void validate_direct_spte(struct kvm_vcpu vcpu, u64 sptep,
@@ -2270,17 +2249,12 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, · 52 unmodified lines
2270	mmu_page_zap_pte(kvm, sp, sp->spt + i);	2249	mmu_page_zap_pte(kvm, sp, sp->spt + i);
2271	}	2250	}
2272		2251
2273	static void kvm_mmu_put_page(struct kvm_mmu_page sp, u64 parent_pte)
2274	{
2275	mmu_page_remove_parent_pte(sp, parent_pte);
2276	}
2277
2278	static void kvm_mmu_unlink_parents(struct kvm kvm, struct kvm_mmu_page sp)	2252	static void kvm_mmu_unlink_parents(struct kvm kvm, struct kvm_mmu_page sp)
2279	{	2253	{
2280	u64 *sptep;	2254	u64 *sptep;
2281	struct rmap_iterator iter;	2255	struct rmap_iterator iter;
2282		2256
2283	while ((sptep = rmap_get_first(sp->parent_ptes, &iter)))	2257	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2284	drop_parent_pte(sp, sptep);	2258	drop_parent_pte(sp, sptep);
2285	}	2259	}
2286		2260
@@ -2564,18 +2538,18 @@ done: · 277 unmodified lines
2564	return ret;	2538	return ret;
2565	}	2539	}
2566		2540
2567	static void mmu_set_spte(struct kvm_vcpu vcpu, u64 sptep,	2541	static bool mmu_set_spte(struct kvm_vcpu vcpu, u64 sptep, unsigned pte_access,
2568	unsigned pte_access, int write_fault, int *emulate,	2542	int write_fault, int level, gfn_t gfn, pfn_t pfn,
2569	int level, gfn_t gfn, pfn_t pfn, bool speculative,	2543	bool speculative, bool host_writable)
2570	bool host_writable)
2571	{	2544	{
2572	int was_rmapped = 0;	2545	int was_rmapped = 0;
2573	int rmap_count;	2546	int rmap_count;
		2547	bool emulate = false;
2574		2548
2575	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,	2549	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
2576	*sptep, write_fault, gfn);	2550	*sptep, write_fault, gfn);
2577		2551
2578	if (is_rmap_spte(*sptep)) {	2552	if (is_shadow_present_pte(*sptep)) {
2579	/*	2553	/*
2580	* If we overwrite a PTE page pointer with a 2MB PMD, unlink	2554	* If we overwrite a PTE page pointer with a 2MB PMD, unlink
2581	* the parent of the now unreachable PTE.	2555	* the parent of the now unreachable PTE.
@@ -2600,12 +2574,12 @@ static void mmu_set_spte(struct kvm_vcpu vcpu, u64 sptep, · 18 unmodified lines
2600	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,	2574	if (set_spte(vcpu, sptep, pte_access, level, gfn, pfn, speculative,
2601	true, host_writable)) {	2575	true, host_writable)) {
2602	if (write_fault)	2576	if (write_fault)
2603	*emulate = 1;	2577	emulate = true;
2604	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);	2578	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2605	}	2579	}
2606		2580
2607	if (unlikely(is_mmio_spte(*sptep) && emulate))	2581	if (unlikely(is_mmio_spte(*sptep)))
2608	*emulate = 1;	2582	emulate = true;
2609		2583
2610	pgprintk("%s: setting spte %llx\n", __func__, *sptep);	2584	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2611	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",	2585	pgprintk("instantiating %s PTE (%s) at %llx (%llx) addr %p\n",
@@ -2624,6 +2598,8 @@ static void mmu_set_spte(struct kvm_vcpu vcpu, u64 sptep, · 12 unmodified lines
2624	}	2598	}
2625		2599
2626	kvm_release_pfn_clean(pfn);	2600	kvm_release_pfn_clean(pfn);
		2601
		2602	return emulate;
2627	}	2603	}
2628		2604
2629	static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,	2605	static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
@@ -2658,9 +2634,8 @@ static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, · 28 unmodified lines
2658	return -1;	2634	return -1;
2659		2635
2660	for (i = 0; i < ret; i++, gfn++, start++)	2636	for (i = 0; i < ret; i++, gfn++, start++)
2661	mmu_set_spte(vcpu, start, access, 0, NULL,	2637	mmu_set_spte(vcpu, start, access, 0, sp->role.level, gfn,
2662	sp->role.level, gfn, page_to_pfn(pages[i]),	2638	page_to_pfn(pages[i]), true, true);
2663	true, true);
2664		2639
2665	return 0;	2640	return 0;
2666	}	2641	}
@@ -2708,9 +2683,8 @@ static void direct_pte_prefetch(struct kvm_vcpu vcpu, u64 sptep) · 41 unmodified lines
2708	__direct_pte_prefetch(vcpu, sp, sptep);	2683	__direct_pte_prefetch(vcpu, sp, sptep);
2709	}	2684	}
2710		2685
2711	static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,	2686	static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
2712	int map_writable, int level, gfn_t gfn, pfn_t pfn,	2687	int level, gfn_t gfn, pfn_t pfn, bool prefault)
2713	bool prefault)
2714	{	2688	{
2715	struct kvm_shadow_walk_iterator iterator;	2689	struct kvm_shadow_walk_iterator iterator;
2716	struct kvm_mmu_page *sp;	2690	struct kvm_mmu_page *sp;
@@ -2722,9 +2696,9 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, · 5 unmodified lines
2722		2696
2723	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {	2697	for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) {
2724	if (iterator.level == level) {	2698	if (iterator.level == level) {
2725	mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,	2699	emulate = mmu_set_spte(vcpu, iterator.sptep, ACC_ALL,
2726	write, &emulate, level, gfn, pfn,	2700	write, level, gfn, pfn, prefault,
2727	prefault, map_writable);	2701	map_writable);
2728	direct_pte_prefetch(vcpu, iterator.sptep);	2702	direct_pte_prefetch(vcpu, iterator.sptep);
2729	++vcpu->stat.pf_fixed;	2703	++vcpu->stat.pf_fixed;
2730	break;	2704	break;
@@ -2737,10 +2711,9 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, · 6 unmodified lines
2737	base_addr &= PT64_LVL_ADDR_MASK(iterator.level);	2711	base_addr &= PT64_LVL_ADDR_MASK(iterator.level);
2738	pseudo_gfn = base_addr >> PAGE_SHIFT;	2712	pseudo_gfn = base_addr >> PAGE_SHIFT;
2739	sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,	2713	sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2740	iterator.level - 1,	2714	iterator.level - 1, 1, ACC_ALL);
2741	1, ACC_ALL, iterator.sptep);
2742		2715
2743	link_shadow_page(iterator.sptep, sp, true);	2716	link_shadow_page(vcpu, iterator.sptep, sp);
2744	}	2717	}
2745	}	2718	}
2746	return emulate;	2719	return emulate;
@@ -2919,7 +2892,7 @@ static bool fast_page_fault(struct kvm_vcpu *vcpu, gva_t gva, int level, · 172 unmodified lines
2919	* If the mapping has been changed, let the vcpu fault on the	2892	* If the mapping has been changed, let the vcpu fault on the
2920	* same address again.	2893	* same address again.
2921	*/	2894	*/
2922	if (!is_rmap_spte(spte)) {	2895	if (!is_shadow_present_pte(spte)) {
2923	ret = true;	2896	ret = true;
2924	goto exit;	2897	goto exit;
2925	}	2898	}
@@ -3018,11 +2991,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, u32 error_code, · 92 unmodified lines
3018	make_mmu_pages_available(vcpu);	2991	make_mmu_pages_available(vcpu);
3019	if (likely(!force_pt_level))	2992	if (likely(!force_pt_level))
3020	transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);	2993	transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3021	r = __direct_map(vcpu, v, write, map_writable, level, gfn, pfn,	2994	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3022	prefault);
3023	spin_unlock(&vcpu->kvm->mmu_lock);	2995	spin_unlock(&vcpu->kvm->mmu_lock);
3024		2996
3025
3026	return r;	2997	return r;
3027		2998
3028	out_unlock:	2999	out_unlock:
@@ -3097,8 +3068,7 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) · 68 unmodified lines
3097	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {	3068	if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
3098	spin_lock(&vcpu->kvm->mmu_lock);	3069	spin_lock(&vcpu->kvm->mmu_lock);
3099	make_mmu_pages_available(vcpu);	3070	make_mmu_pages_available(vcpu);
3100	sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL,	3071	sp = kvm_mmu_get_page(vcpu, 0, 0, PT64_ROOT_LEVEL, 1, ACC_ALL);
3101	1, ACC_ALL, NULL);
3102	++sp->root_count;	3072	++sp->root_count;
3103	spin_unlock(&vcpu->kvm->mmu_lock);	3073	spin_unlock(&vcpu->kvm->mmu_lock);
3104	vcpu->arch.mmu.root_hpa = __pa(sp->spt);	3074	vcpu->arch.mmu.root_hpa = __pa(sp->spt);
@@ -3110,9 +3080,7 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) · 5 unmodified lines
3110	spin_lock(&vcpu->kvm->mmu_lock);	3080	spin_lock(&vcpu->kvm->mmu_lock);
3111	make_mmu_pages_available(vcpu);	3081	make_mmu_pages_available(vcpu);
3112	sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),	3082	sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT),
3113	i << 30,	3083	i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL);
3114	PT32_ROOT_LEVEL, 1, ACC_ALL,
3115	NULL);
3116	root = __pa(sp->spt);	3084	root = __pa(sp->spt);
3117	++sp->root_count;	3085	++sp->root_count;
3118	spin_unlock(&vcpu->kvm->mmu_lock);	3086	spin_unlock(&vcpu->kvm->mmu_lock);
@@ -3149,7 +3117,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) · 30 unmodified lines
3149	spin_lock(&vcpu->kvm->mmu_lock);	3117	spin_lock(&vcpu->kvm->mmu_lock);
3150	make_mmu_pages_available(vcpu);	3118	make_mmu_pages_available(vcpu);
3151	sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,	3119	sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL,
3152	0, ACC_ALL, NULL);	3120	0, ACC_ALL);
3153	root = __pa(sp->spt);	3121	root = __pa(sp->spt);
3154	++sp->root_count;	3122	++sp->root_count;
3155	spin_unlock(&vcpu->kvm->mmu_lock);	3123	spin_unlock(&vcpu->kvm->mmu_lock);
@@ -3182,9 +3150,8 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) · 26 unmodified lines
3182	}	3150	}
3183	spin_lock(&vcpu->kvm->mmu_lock);	3151	spin_lock(&vcpu->kvm->mmu_lock);
3184	make_mmu_pages_available(vcpu);	3152	make_mmu_pages_available(vcpu);
3185	sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,	3153	sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL,
3186	PT32_ROOT_LEVEL, 0,	3154	0, ACC_ALL);
3187	ACC_ALL, NULL);
3188	root = __pa(sp->spt);	3155	root = __pa(sp->spt);
3189	++sp->root_count;	3156	++sp->root_count;
3190	spin_unlock(&vcpu->kvm->mmu_lock);	3157	spin_unlock(&vcpu->kvm->mmu_lock);
@@ -3531,8 +3498,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, u32 error_code, · 340 unmodified lines
3531	make_mmu_pages_available(vcpu);	3498	make_mmu_pages_available(vcpu);
3532	if (likely(!force_pt_level))	3499	if (likely(!force_pt_level))
3533	transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);	3500	transparent_hugepage_adjust(vcpu, &gfn, &pfn, &level);
3534	r = __direct_map(vcpu, gpa, write, map_writable,	3501	r = __direct_map(vcpu, write, map_writable, level, gfn, pfn, prefault);
3535	level, gfn, pfn, prefault);
3536	spin_unlock(&vcpu->kvm->mmu_lock);	3502	spin_unlock(&vcpu->kvm->mmu_lock);
3537		3503
3538	return r;	3504	return r;
@@ -4058,10 +4024,12 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) · 519 unmodified lines
4058	g_context->inject_page_fault = kvm_inject_page_fault;	4024	g_context->inject_page_fault = kvm_inject_page_fault;
4059		4025
4060	/*	4026	/*
4061	* Note that arch.mmu.gva_to_gpa translates l2_gva to l1_gpa. The	4027	* Note that arch.mmu.gva_to_gpa translates l2_gpa to l1_gpa using
4062	* translation of l2_gpa to l1_gpa addresses is done using the	4028	* L1's nested page tables (e.g. EPT12). The nested translation
4063	* arch.nested_mmu.gva_to_gpa function. Basically the gva_to_gpa	4029	* of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
4064	* functions between mmu and nested_mmu are swapped.	4030	* L2's page tables as the first level of translation and L1's
		4031	* nested page tables as the second level of translation. Basically
		4032	* the gva_to_gpa functions between mmu and nested_mmu are swapped.
4065	*/	4033	*/
4066	if (!is_paging(vcpu)) {	4034	if (!is_paging(vcpu)) {
4067	g_context->nx = false;	4035	g_context->nx = false;
@@ -4495,7 +4463,7 @@ void kvm_mmu_setup(struct kvm_vcpu *vcpu) · 427 unmodified lines
4495	}	4463	}
4496		4464
4497	/* The return value indicates if tlb flush on all vcpus is needed. */	4465	/* The return value indicates if tlb flush on all vcpus is needed. */
4498	typedef bool (slot_level_handler) (struct kvm kvm, unsigned long *rmap);	4466	typedef bool (slot_level_handler) (struct kvm kvm, struct kvm_rmap_head *rmap_head);
4499		4467
4500	/* The caller should hold mmu-lock before calling this function. */	4468	/* The caller should hold mmu-lock before calling this function. */
4501	static bool	4469	static bool
@@ -4589,9 +4557,10 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 87 unmodified lines
4589	spin_unlock(&kvm->mmu_lock);	4557	spin_unlock(&kvm->mmu_lock);
4590	}	4558	}
4591		4559
4592	static bool slot_rmap_write_protect(struct kvm kvm, unsigned long rmapp)	4560	static bool slot_rmap_write_protect(struct kvm *kvm,
		4561	struct kvm_rmap_head *rmap_head)
4593	{	4562	{
4594	return __rmap_write_protect(kvm, rmapp, false);	4563	return __rmap_write_protect(kvm, rmap_head, false);
4595	}	4564	}
4596		4565
4597	void kvm_mmu_slot_remove_write_access(struct kvm *kvm,	4566	void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
@@ -4627,7 +4596,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, · 29 unmodified lines
4627	}	4596	}
4628		4597
4629	static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,	4598	static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
4630	unsigned long *rmapp)	4599	struct kvm_rmap_head *rmap_head)
4631	{	4600	{
4632	u64 *sptep;	4601	u64 *sptep;
4633	struct rmap_iterator iter;	4602	struct rmap_iterator iter;
@@ -4636,7 +4605,7 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, · 2 unmodified lines
4636	struct kvm_mmu_page *sp;	4605	struct kvm_mmu_page *sp;
4637		4606
4638	restart:	4607	restart:
4639	for_each_rmap_spte(rmapp, &iter, sptep) {	4608	for_each_rmap_spte(rmap_head, &iter, sptep) {
4640	sp = page_header(__pa(sptep));	4609	sp = page_header(__pa(sptep));
4641	pfn = spte_to_pfn(*sptep);	4610	pfn = spte_to_pfn(*sptep);
4642		4611

arch/x86/kvm/mmu_audit.c+8−7

 static void inspect_spte_has_rmap(struct kvm *kvm, u64 *sptep)
 {
 	static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
-	unsigned long *rmapp;
+	struct kvm_rmap_head *rmap_head;
 	struct kvm_mmu_page *rev_sp;
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *slot;
 		return;
 	}
-	rmapp = __gfn_to_rmap(gfn, rev_sp->role.level, slot);
+	rmap_head = __gfn_to_rmap(gfn, rev_sp->role.level, slot);
-	if (!*rmapp) {
+	if (!rmap_head->val) {
 		if (!__ratelimit(&ratelimit_state))
 			return;
 		audit_printk(kvm, "no rmap for writable spte %llx\n",
 		return;
 	for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
-		if (!is_rmap_spte(sp->spt[i]))
+		if (!is_shadow_present_pte(sp->spt[i]))
 			continue;
 		inspect_spte_has_rmap(kvm, sp->spt + i);
 static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
-	unsigned long *rmapp;
+	struct kvm_rmap_head *rmap_head;
 	u64 *sptep;
 	struct rmap_iterator iter;
 	struct kvm_memslots *slots;
 	slots = kvm_memslots_for_spte_role(kvm, sp->role);
 	slot = __gfn_to_memslot(slots, sp->gfn);
-	rmapp = __gfn_to_rmap(sp->gfn, PT_PAGE_TABLE_LEVEL, slot);
+	rmap_head = __gfn_to_rmap(sp->gfn, PT_PAGE_TABLE_LEVEL, slot);
-	for_each_rmap_spte(rmapp, &iter, sptep)
+	for_each_rmap_spte(rmap_head, &iter, sptep) {
 		if (is_writable_pte(*sptep))
 			audit_printk(kvm, "shadow page has writable "
 				     "mappings: gfn %llx role %x\n",
 				     sp->gfn, sp->role.word);
+	}
 }
 static void audit_sp(struct kvm *kvm, struct kvm_mmu_page *sp)

arch/x86/kvm/paging_tmpl.h+9−11

 	 * we call mmu_set_spte() with host_writable = true because
 	 * pte_prefetch_gfn_to_pfn always gets a writable pfn.
 	 */
-	mmu_set_spte(vcpu, spte, pte_access, 0, NULL, PT_PAGE_TABLE_LEVEL,
+	mmu_set_spte(vcpu, spte, pte_access, 0, PT_PAGE_TABLE_LEVEL, gfn, pfn,
-		     gfn, pfn, true, true);
+		     true, true);
 	return true;
 }
 	struct kvm_mmu_page *sp = NULL;
 	struct kvm_shadow_walk_iterator it;
 	unsigned direct_access, access = gw->pt_access;
-	int top_level, emulate = 0;
+	int top_level, emulate;
 	direct_access = gw->pte_access;
 		if (!is_shadow_present_pte(*it.sptep)) {
 			table_gfn = gw->table_gfn[it.level - 2];
 			sp = kvm_mmu_get_page(vcpu, table_gfn, addr, it.level-1,
-					      false, access, it.sptep);
+					      false, access);
 		}
 		/*
 			goto out_gpte_changed;
 		if (sp)
-			link_shadow_page(it.sptep, sp, PT_GUEST_ACCESSED_MASK);
+			link_shadow_page(vcpu, it.sptep, sp);
 	}
 	for (;
 		direct_gfn = gw->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
 		sp = kvm_mmu_get_page(vcpu, direct_gfn, addr, it.level-1,
-				      true, direct_access, it.sptep);
+				      true, direct_access);
-		link_shadow_page(it.sptep, sp, PT_GUEST_ACCESSED_MASK);
+		link_shadow_page(vcpu, it.sptep, sp);
 	}
 	clear_sp_write_flooding_count(it.sptep);
-	mmu_set_spte(vcpu, it.sptep, gw->pte_access, write_fault, &emulate,
+	emulate = mmu_set_spte(vcpu, it.sptep, gw->pte_access, write_fault,
-		     it.level, gw->gfn, pfn, prefault, map_writable);
+			       it.level, gw->gfn, pfn, prefault, map_writable);
 	FNAME(pte_prefetch)(vcpu, gw, it.sptep);
 	return emulate;
 out_gpte_changed:
-	if (sp)
-		kvm_mmu_put_page(sp, it.sptep);
 	kvm_release_pfn_clean(pfn);
 	return 0;
 }

arch/x86/kvm/svm.c+32−5

 	MSR_FS_BASE,
 #endif
 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
+	MSR_TSC_AUX,
 };
 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 	uint64_t asid_generation;
 	uint64_t sysenter_esp;
 	uint64_t sysenter_eip;
+	uint64_t tsc_aux;
 	u64 next_rip;
 			wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
 		}
 	}
+	/* This assumes that the kernel never uses MSR_TSC_AUX */
+	if (static_cpu_has(X86_FEATURE_RDTSCP))
+		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
 }
 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
 	case MSR_IA32_SYSENTER_ESP:
 		msr_info->data = svm->sysenter_esp;
 		break;
+	case MSR_TSC_AUX:
+		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
+			return 1;
+		msr_info->data = svm->tsc_aux;
+		break;
 	/*
 	 * Nobody will change the following 5 values in the VMCB so we can
 	 * safely return them on rdmsr. They will always be 0 until LBRV is
 		svm->sysenter_esp = data;
 		svm->vmcb->save.sysenter_esp = data;
 		break;
+	case MSR_TSC_AUX:
+		if (!boot_cpu_has(X86_FEATURE_RDTSCP))
+			return 1;
+		/*
+		 * This is rare, so we update the MSR here instead of using
+		 * direct_access_msrs.  Doing that would require a rdmsr in
+		 * svm_vcpu_put.
+		 */
+		svm->tsc_aux = data;
+		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
+		break;
 	case MSR_IA32_DEBUGCTLMSR:
 		if (!boot_cpu_has(X86_FEATURE_LBRV)) {
 			vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
 	return;
 }
-static int svm_cpu_uses_apicv(struct kvm_vcpu *vcpu)
+static bool svm_get_enable_apicv(void)
+{
+	return false;
+}
+static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
 {
-	return 0;
 }
-static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu)
+static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
 {
 	return;
 }
 static bool svm_rdtscp_supported(void)
 {
-	return false;
+	return boot_cpu_has(X86_FEATURE_RDTSCP);
 }
 static bool svm_invpcid_supported(void)
 	.enable_irq_window = enable_irq_window,
 	.update_cr8_intercept = update_cr8_intercept,
 	.set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
-	.cpu_uses_apicv = svm_cpu_uses_apicv,
+	.get_enable_apicv = svm_get_enable_apicv,
+	.refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
 	.load_eoi_exitmap = svm_load_eoi_exitmap,
 	.sync_pir_to_irr = svm_sync_pir_to_irr,

arch/x86/kvm/trace.h+264−1

@@ -268,7 +268,7 @@ TRACE_EVENT(kvm_inj_virq, · 267 unmodified lines
268	#define kvm_trace_sym_exc \	268	#define kvm_trace_sym_exc \
269	EXS(DE), EXS(DB), EXS(BP), EXS(OF), EXS(BR), EXS(UD), EXS(NM), \	269	EXS(DE), EXS(DB), EXS(BP), EXS(OF), EXS(BR), EXS(UD), EXS(NM), \
270	EXS(DF), EXS(TS), EXS(NP), EXS(SS), EXS(GP), EXS(PF), \	270	EXS(DF), EXS(TS), EXS(NP), EXS(SS), EXS(GP), EXS(PF), \
271	EXS(MF), EXS(MC)	271	EXS(MF), EXS(AC), EXS(MC)
272		272
273	/*	273	/*
274	* Tracepoint for kvm interrupt injection:	274	* Tracepoint for kvm interrupt injection:
@@ -1025,6 +1025,269 @@ TRACE_EVENT(kvm_pi_irte_update, · 750 unmodified lines
1025	__entry->pi_desc_addr)	1025	__entry->pi_desc_addr)
1026	);	1026	);
1027		1027
		1028	/*
		1029	* Tracepoint for kvm_hv_notify_acked_sint.
		1030	*/
		1031	TRACE_EVENT(kvm_hv_notify_acked_sint,
		1032	TP_PROTO(int vcpu_id, u32 sint),
		1033	TP_ARGS(vcpu_id, sint),
		1034
		1035	TP_STRUCT__entry(
		1036	__field(int, vcpu_id)
		1037	__field(u32, sint)
		1038	),
		1039
		1040	TP_fast_assign(
		1041	__entry->vcpu_id = vcpu_id;
		1042	__entry->sint = sint;
		1043	),
		1044
		1045	TP_printk("vcpu_id %d sint %u", __entry->vcpu_id, __entry->sint)
		1046	);
		1047
		1048	/*
		1049	* Tracepoint for synic_set_irq.
		1050	*/
		1051	TRACE_EVENT(kvm_hv_synic_set_irq,
		1052	TP_PROTO(int vcpu_id, u32 sint, int vector, int ret),
		1053	TP_ARGS(vcpu_id, sint, vector, ret),
		1054
		1055	TP_STRUCT__entry(
		1056	__field(int, vcpu_id)
		1057	__field(u32, sint)
		1058	__field(int, vector)
		1059	__field(int, ret)
		1060	),
		1061
		1062	TP_fast_assign(
		1063	__entry->vcpu_id = vcpu_id;
		1064	__entry->sint = sint;
		1065	__entry->vector = vector;
		1066	__entry->ret = ret;
		1067	),
		1068
		1069	TP_printk("vcpu_id %d sint %u vector %d ret %d",
		1070	__entry->vcpu_id, __entry->sint, __entry->vector,
		1071	__entry->ret)
		1072	);
		1073
		1074	/*
		1075	* Tracepoint for kvm_hv_synic_send_eoi.
		1076	*/
		1077	TRACE_EVENT(kvm_hv_synic_send_eoi,
		1078	TP_PROTO(int vcpu_id, int vector),
		1079	TP_ARGS(vcpu_id, vector),
		1080
		1081	TP_STRUCT__entry(
		1082	__field(int, vcpu_id)
		1083	__field(u32, sint)
		1084	__field(int, vector)
		1085	__field(int, ret)
		1086	),
		1087
		1088	TP_fast_assign(
		1089	__entry->vcpu_id = vcpu_id;
		1090	__entry->vector = vector;
		1091	),
		1092
		1093	TP_printk("vcpu_id %d vector %d", __entry->vcpu_id, __entry->vector)
		1094	);
		1095
		1096	/*
		1097	* Tracepoint for synic_set_msr.
		1098	*/
		1099	TRACE_EVENT(kvm_hv_synic_set_msr,
		1100	TP_PROTO(int vcpu_id, u32 msr, u64 data, bool host),
		1101	TP_ARGS(vcpu_id, msr, data, host),
		1102
		1103	TP_STRUCT__entry(
		1104	__field(int, vcpu_id)
		1105	__field(u32, msr)
		1106	__field(u64, data)
		1107	__field(bool, host)
		1108	),
		1109
		1110	TP_fast_assign(
		1111	__entry->vcpu_id = vcpu_id;
		1112	__entry->msr = msr;
		1113	__entry->data = data;
		1114	__entry->host = host
		1115	),
		1116
		1117	TP_printk("vcpu_id %d msr 0x%x data 0x%llx host %d",
		1118	__entry->vcpu_id, __entry->msr, __entry->data, __entry->host)
		1119	);
		1120
		1121	/*
		1122	* Tracepoint for stimer_set_config.
		1123	*/
		1124	TRACE_EVENT(kvm_hv_stimer_set_config,
		1125	TP_PROTO(int vcpu_id, int timer_index, u64 config, bool host),
		1126	TP_ARGS(vcpu_id, timer_index, config, host),
		1127
		1128	TP_STRUCT__entry(
		1129	__field(int, vcpu_id)
		1130	__field(int, timer_index)
		1131	__field(u64, config)
		1132	__field(bool, host)
		1133	),
		1134
		1135	TP_fast_assign(
		1136	__entry->vcpu_id = vcpu_id;
		1137	__entry->timer_index = timer_index;
		1138	__entry->config = config;
		1139	__entry->host = host;
		1140	),
		1141
		1142	TP_printk("vcpu_id %d timer %d config 0x%llx host %d",
		1143	__entry->vcpu_id, __entry->timer_index, __entry->config,
		1144	__entry->host)
		1145	);
		1146
		1147	/*
		1148	* Tracepoint for stimer_set_count.
		1149	*/
		1150	TRACE_EVENT(kvm_hv_stimer_set_count,
		1151	TP_PROTO(int vcpu_id, int timer_index, u64 count, bool host),
		1152	TP_ARGS(vcpu_id, timer_index, count, host),
		1153
		1154	TP_STRUCT__entry(
		1155	__field(int, vcpu_id)
		1156	__field(int, timer_index)
		1157	__field(u64, count)
		1158	__field(bool, host)
		1159	),
		1160
		1161	TP_fast_assign(
		1162	__entry->vcpu_id = vcpu_id;
		1163	__entry->timer_index = timer_index;
		1164	__entry->count = count;
		1165	__entry->host = host;
		1166	),
		1167
		1168	TP_printk("vcpu_id %d timer %d count %llu host %d",
		1169	__entry->vcpu_id, __entry->timer_index, __entry->count,
		1170	__entry->host)
		1171	);
		1172
		1173	/*
		1174	* Tracepoint for stimer_start(periodic timer case).
		1175	*/
		1176	TRACE_EVENT(kvm_hv_stimer_start_periodic,
		1177	TP_PROTO(int vcpu_id, int timer_index, u64 time_now, u64 exp_time),
		1178	TP_ARGS(vcpu_id, timer_index, time_now, exp_time),
		1179
		1180	TP_STRUCT__entry(
		1181	__field(int, vcpu_id)
		1182	__field(int, timer_index)
		1183	__field(u64, time_now)
		1184	__field(u64, exp_time)
		1185	),
		1186
		1187	TP_fast_assign(
		1188	__entry->vcpu_id = vcpu_id;
		1189	__entry->timer_index = timer_index;
		1190	__entry->time_now = time_now;
		1191	__entry->exp_time = exp_time;
		1192	),
		1193
		1194	TP_printk("vcpu_id %d timer %d time_now %llu exp_time %llu",
		1195	__entry->vcpu_id, __entry->timer_index, __entry->time_now,
		1196	__entry->exp_time)
		1197	);
		1198
		1199	/*
		1200	* Tracepoint for stimer_start(one-shot timer case).
		1201	*/
		1202	TRACE_EVENT(kvm_hv_stimer_start_one_shot,
		1203	TP_PROTO(int vcpu_id, int timer_index, u64 time_now, u64 count),
		1204	TP_ARGS(vcpu_id, timer_index, time_now, count),
		1205
		1206	TP_STRUCT__entry(
		1207	__field(int, vcpu_id)
		1208	__field(int, timer_index)
		1209	__field(u64, time_now)
		1210	__field(u64, count)
		1211	),
		1212
		1213	TP_fast_assign(
		1214	__entry->vcpu_id = vcpu_id;
		1215	__entry->timer_index = timer_index;
		1216	__entry->time_now = time_now;
		1217	__entry->count = count;
		1218	),
		1219
		1220	TP_printk("vcpu_id %d timer %d time_now %llu count %llu",
		1221	__entry->vcpu_id, __entry->timer_index, __entry->time_now,
		1222	__entry->count)
		1223	);
		1224
		1225	/*
		1226	* Tracepoint for stimer_timer_callback.
		1227	*/
		1228	TRACE_EVENT(kvm_hv_stimer_callback,
		1229	TP_PROTO(int vcpu_id, int timer_index),
		1230	TP_ARGS(vcpu_id, timer_index),
		1231
		1232	TP_STRUCT__entry(
		1233	__field(int, vcpu_id)
		1234	__field(int, timer_index)
		1235	),
		1236
		1237	TP_fast_assign(
		1238	__entry->vcpu_id = vcpu_id;
		1239	__entry->timer_index = timer_index;
		1240	),
		1241
		1242	TP_printk("vcpu_id %d timer %d",
		1243	__entry->vcpu_id, __entry->timer_index)
		1244	);
		1245
		1246	/*
		1247	* Tracepoint for stimer_expiration.
		1248	*/
		1249	TRACE_EVENT(kvm_hv_stimer_expiration,
		1250	TP_PROTO(int vcpu_id, int timer_index, int msg_send_result),
		1251	TP_ARGS(vcpu_id, timer_index, msg_send_result),
		1252
		1253	TP_STRUCT__entry(
		1254	__field(int, vcpu_id)
		1255	__field(int, timer_index)
		1256	__field(int, msg_send_result)
		1257	),
		1258
		1259	TP_fast_assign(
		1260	__entry->vcpu_id = vcpu_id;
		1261	__entry->timer_index = timer_index;
		1262	__entry->msg_send_result = msg_send_result;
		1263	),
		1264
		1265	TP_printk("vcpu_id %d timer %d msg send result %d",
		1266	__entry->vcpu_id, __entry->timer_index,
		1267	__entry->msg_send_result)
		1268	);
		1269
		1270	/*
		1271	* Tracepoint for stimer_cleanup.
		1272	*/
		1273	TRACE_EVENT(kvm_hv_stimer_cleanup,
		1274	TP_PROTO(int vcpu_id, int timer_index),
		1275	TP_ARGS(vcpu_id, timer_index),
		1276
		1277	TP_STRUCT__entry(
		1278	__field(int, vcpu_id)
		1279	__field(int, timer_index)
		1280	),
		1281
		1282	TP_fast_assign(
		1283	__entry->vcpu_id = vcpu_id;
		1284	__entry->timer_index = timer_index;
		1285	),
		1286
		1287	TP_printk("vcpu_id %d timer %d",
		1288	__entry->vcpu_id, __entry->timer_index)
		1289	);
		1290
1028	#endif /* _TRACE_KVM_H */	1291	#endif /* _TRACE_KVM_H */
1029		1292
1030	#undef TRACE_INCLUDE_PATH	1293	#undef TRACE_INCLUDE_PATH

arch/x86/kvm/vmx.c+131−71

 #include "irq.h"
 #include "mmu.h"
 #include "cpuid.h"
+#include "lapic.h"
 #include <linux/kvm_host.h>
 #include <linux/module.h>
 static void kvm_cpu_vmxoff(void);
 static bool vmx_mpx_supported(void);
 static bool vmx_xsaves_supported(void);
-static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu);
 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
 static void vmx_set_segment(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg);
 			    struct kvm_segment *var, int seg);
 static bool guest_state_valid(struct kvm_vcpu *vcpu);
 static u32 vmx_segment_access_rights(struct kvm_segment *var);
-static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu);
 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx);
 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
 static int alloc_identity_pagetable(struct kvm *kvm);
 	}
 }
-static __always_inline unsigned long vmcs_readl(unsigned long field)
+static __always_inline void vmcs_check16(unsigned long field)
+{
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
+			 "16-bit accessor invalid for 64-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
+			 "16-bit accessor invalid for 64-bit high field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
+			 "16-bit accessor invalid for 32-bit high field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
+			 "16-bit accessor invalid for natural width field");
+}
+static __always_inline void vmcs_check32(unsigned long field)
+{
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
+			 "32-bit accessor invalid for 16-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
+			 "32-bit accessor invalid for natural width field");
+}
+static __always_inline void vmcs_check64(unsigned long field)
+{
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
+			 "64-bit accessor invalid for 16-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
+			 "64-bit accessor invalid for 64-bit high field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
+			 "64-bit accessor invalid for 32-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
+			 "64-bit accessor invalid for natural width field");
+}
+static __always_inline void vmcs_checkl(unsigned long field)
+{
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
+			 "Natural width accessor invalid for 16-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
+			 "Natural width accessor invalid for 64-bit field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
+			 "Natural width accessor invalid for 64-bit high field");
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
+			 "Natural width accessor invalid for 32-bit field");
+}
+static __always_inline unsigned long __vmcs_readl(unsigned long field)
 {
 	unsigned long value;
 static __always_inline u16 vmcs_read16(unsigned long field)
 {
-	return vmcs_readl(field);
+	vmcs_check16(field);
+	return __vmcs_readl(field);
 }
 static __always_inline u32 vmcs_read32(unsigned long field)
 {
-	return vmcs_readl(field);
+	vmcs_check32(field);
+	return __vmcs_readl(field);
 }
 static __always_inline u64 vmcs_read64(unsigned long field)
 {
+	vmcs_check64(field);
 #ifdef CONFIG_X86_64
-	return vmcs_readl(field);
+	return __vmcs_readl(field);
 #else
-	return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
+	return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
 #endif
 }
+static __always_inline unsigned long vmcs_readl(unsigned long field)
+{
+	vmcs_checkl(field);
+	return __vmcs_readl(field);
+}
 static noinline void vmwrite_error(unsigned long field, unsigned long value)
 {
 	printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
 	dump_stack();
 }
-static void vmcs_writel(unsigned long field, unsigned long value)
+static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
 {
 	u8 error;
 		vmwrite_error(field, value);
 }
-static void vmcs_write16(unsigned long field, u16 value)
+static __always_inline void vmcs_write16(unsigned long field, u16 value)
 {
-	vmcs_writel(field, value);
+	vmcs_check16(field);
+	__vmcs_writel(field, value);
 }
-static void vmcs_write32(unsigned long field, u32 value)
+static __always_inline void vmcs_write32(unsigned long field, u32 value)
 {
-	vmcs_writel(field, value);
+	vmcs_check32(field);
+	__vmcs_writel(field, value);
 }
-static void vmcs_write64(unsigned long field, u64 value)
+static __always_inline void vmcs_write64(unsigned long field, u64 value)
 {
-	vmcs_writel(field, value);
+	vmcs_check64(field);
+	__vmcs_writel(field, value);
 #ifndef CONFIG_X86_64
 	asm volatile ("");
-	vmcs_writel(field+1, value >> 32);
+	__vmcs_writel(field+1, value >> 32);
 #endif
 }
-static void vmcs_clear_bits(unsigned long field, u32 mask)
+static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
 {
-	vmcs_writel(field, vmcs_readl(field) & ~mask);
+	vmcs_checkl(field);
+	__vmcs_writel(field, value);
 }
-static void vmcs_set_bits(unsigned long field, u32 mask)
+static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
 {
-	vmcs_writel(field, vmcs_readl(field) | mask);
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
+			 "vmcs_clear_bits does not support 64-bit fields");
+	__vmcs_writel(field, __vmcs_readl(field) & ~mask);
+}
+static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
+{
+        BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
+			 "vmcs_set_bits does not support 64-bit fields");
+	__vmcs_writel(field, __vmcs_readl(field) | mask);
 }
 static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
 	vmx->nested.nested_vmx_pinbased_ctls_high |=
 		PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
 		PIN_BASED_VMX_PREEMPTION_TIMER;
-	if (vmx_cpu_uses_apicv(&vmx->vcpu))
+	if (kvm_vcpu_apicv_active(&vmx->vcpu))
 		vmx->nested.nested_vmx_pinbased_ctls_high |=
 			PIN_BASED_POSTED_INTR;
 			msr, MSR_TYPE_W);
 }
-static int vmx_cpu_uses_apicv(struct kvm_vcpu *vcpu)
+static bool vmx_get_enable_apicv(void)
 {
-	return enable_apicv && lapic_in_kernel(vcpu);
+	return enable_apicv;
 }
 static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
 	kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
 }
-static void vmx_sync_pir_to_irr_dummy(struct kvm_vcpu *vcpu)
-{
-	return;
-}
 /*
  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
  * will not change in the lifetime of the guest.
 {
 	u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
-	if (!vmx_cpu_uses_apicv(&vmx->vcpu))
+	if (!kvm_vcpu_apicv_active(&vmx->vcpu))
 		pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
 	return pin_based_exec_ctrl;
 }
+static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
+}
 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
 {
 	u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
 		exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
 	if (!ple_gap)
 		exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
-	if (!vmx_cpu_uses_apicv(&vmx->vcpu))
+	if (!kvm_vcpu_apicv_active(&vmx->vcpu))
 		exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
 				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
 	exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
 		vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
 				vmx_secondary_exec_control(vmx));
-	if (vmx_cpu_uses_apicv(&vmx->vcpu)) {
+	if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
 		vmcs_write64(EOI_EXIT_BITMAP0, 0);
 		vmcs_write64(EOI_EXIT_BITMAP1, 0);
 		vmcs_write64(EOI_EXIT_BITMAP2, 0);
 		vmcs_write16(GUEST_INTR_STATUS, 0);
-		vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
+		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
 		vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
 	}
 	seg_setup(VCPU_SREG_CS);
 	vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
-	vmcs_write32(GUEST_CS_BASE, 0xffff0000);
+	vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
 	seg_setup(VCPU_SREG_DS);
 	seg_setup(VCPU_SREG_ES);
 	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
 	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
-	vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
+	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
 	setup_msrs(vmx);
 	kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
-	if (vmx_cpu_uses_apicv(vcpu))
+	if (kvm_vcpu_apicv_active(vcpu))
 		memset(&vmx->pi_desc, 0, sizeof(struct pi_desc));
 	if (vmx->vpid != 0)
 		kvm_tsc_scaling_ratio_frac_bits = 48;
 	}
-	if (enable_apicv)
-		kvm_x86_ops->update_cr8_intercept = NULL;
-	else {
-		kvm_x86_ops->hwapic_irr_update = NULL;
-		kvm_x86_ops->hwapic_isr_update = NULL;
-		kvm_x86_ops->deliver_posted_interrupt = NULL;
-		kvm_x86_ops->sync_pir_to_irr = vmx_sync_pir_to_irr_dummy;
-	}
 	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
 	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
 	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
 	u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
 	u32 secondary_exec_control = 0;
 	unsigned long cr4 = vmcs_readl(GUEST_CR4);
-	u64 efer = vmcs_readl(GUEST_IA32_EFER);
+	u64 efer = vmcs_read64(GUEST_IA32_EFER);
 	int i, n;
 	if (cpu_has_secondary_exec_ctrls())
 	if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
 	    (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
 	{
-		pr_err("PDPTR0 = 0x%016lx  PDPTR1 = 0x%016lx\n",
+		pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
-		       vmcs_readl(GUEST_PDPTR0), vmcs_readl(GUEST_PDPTR1));
+		       vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
-		pr_err("PDPTR2 = 0x%016lx  PDPTR3 = 0x%016lx\n",
+		pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
-		       vmcs_readl(GUEST_PDPTR2), vmcs_readl(GUEST_PDPTR3));
+		       vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
 	}
 	pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
 	       vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
 	vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
 	if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
 	    (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
-		pr_err("EFER =     0x%016llx  PAT = 0x%016lx\n",
+		pr_err("EFER =     0x%016llx  PAT = 0x%016llx\n",
-		       efer, vmcs_readl(GUEST_IA32_PAT));
+		       efer, vmcs_read64(GUEST_IA32_PAT));
-	pr_err("DebugCtl = 0x%016lx  DebugExceptions = 0x%016lx\n",
+	pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
-	       vmcs_readl(GUEST_IA32_DEBUGCTL),
+	       vmcs_read64(GUEST_IA32_DEBUGCTL),
 	       vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
 	if (vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
-		pr_err("PerfGlobCtl = 0x%016lx\n",
+		pr_err("PerfGlobCtl = 0x%016llx\n",
-		       vmcs_readl(GUEST_IA32_PERF_GLOBAL_CTRL));
+		       vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
 	if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
-		pr_err("BndCfgS = 0x%016lx\n", vmcs_readl(GUEST_BNDCFGS));
+		pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
 	pr_err("Interruptibility = %08x  ActivityState = %08x\n",
 	       vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
 	       vmcs_read32(GUEST_ACTIVITY_STATE));
 	       vmcs_read32(HOST_IA32_SYSENTER_CS),
 	       vmcs_readl(HOST_IA32_SYSENTER_EIP));
 	if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
-		pr_err("EFER = 0x%016lx  PAT = 0x%016lx\n",
+		pr_err("EFER = 0x%016llx  PAT = 0x%016llx\n",
-		       vmcs_readl(HOST_IA32_EFER), vmcs_readl(HOST_IA32_PAT));
+		       vmcs_read64(HOST_IA32_EFER),
+		       vmcs_read64(HOST_IA32_PAT));
 	if (vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
-		pr_err("PerfGlobCtl = 0x%016lx\n",
+		pr_err("PerfGlobCtl = 0x%016llx\n",
-		       vmcs_readl(HOST_IA32_PERF_GLOBAL_CTRL));
+		       vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
 	pr_err("*** Control State ***\n");
 	pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
 	pr_err("IDTVectoring: info=%08x errcode=%08x\n",
 	       vmcs_read32(IDT_VECTORING_INFO_FIELD),
 	       vmcs_read32(IDT_VECTORING_ERROR_CODE));
-	pr_err("TSC Offset = 0x%016lx\n", vmcs_readl(TSC_OFFSET));
+	pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
 	if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
-		pr_err("TSC Multiplier = 0x%016lx\n",
+		pr_err("TSC Multiplier = 0x%016llx\n",
-		       vmcs_readl(TSC_MULTIPLIER));
+		       vmcs_read64(TSC_MULTIPLIER));
 	if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
 		pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
 	if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
 		pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
 	if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
-		pr_err("EPT pointer = 0x%016lx\n", vmcs_readl(EPT_POINTER));
+		pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
 	n = vmcs_read32(CR3_TARGET_COUNT);
 	for (i = 0; i + 1 < n; i += 4)
 		pr_err("CR3 target%u=%016lx target%u=%016lx\n",
 	 * apicv
 	 */
 	if (!cpu_has_vmx_virtualize_x2apic_mode() ||
-				!vmx_cpu_uses_apicv(vcpu))
+				!kvm_vcpu_apicv_active(vcpu))
 		return;
 	if (!cpu_need_tpr_shadow(vcpu))
 	}
 }
-static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu)
+static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
 {
-	u64 *eoi_exit_bitmap = vcpu->arch.eoi_exit_bitmap;
+	if (!kvm_vcpu_apicv_active(vcpu))
-	if (!vmx_cpu_uses_apicv(vcpu))
 		return;
 	vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
 			best->ebx &= ~bit(X86_FEATURE_INVPCID);
 	}
-	vmcs_set_secondary_exec_control(secondary_exec_ctl);
+	if (cpu_has_secondary_exec_ctrls())
+		vmcs_set_secondary_exec_control(secondary_exec_ctl);
 	if (static_cpu_has(X86_FEATURE_PCOMMIT) && nested) {
 		if (guest_cpuid_has_pcommit(vcpu))
 		 */
 		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
 		vmx->nested.pi_pending = false;
-		vmcs_write64(POSTED_INTR_NV, POSTED_INTR_VECTOR);
+		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
 		vmcs_write64(POSTED_INTR_DESC_ADDR,
 			page_to_phys(vmx->nested.pi_desc_page) +
 			(unsigned long)(vmcs12->posted_intr_desc_addr &
 	 * Additionally, restore L2's PDPTR to vmcs12.
 	 */
 	if (enable_ept) {
-		vmcs12->guest_cr3 = vmcs_read64(GUEST_CR3);
+		vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
 		vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
 		vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
 		vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
 	.update_cr8_intercept = update_cr8_intercept,
 	.set_virtual_x2apic_mode = vmx_set_virtual_x2apic_mode,
 	.set_apic_access_page_addr = vmx_set_apic_access_page_addr,
-	.cpu_uses_apicv = vmx_cpu_uses_apicv,
+	.get_enable_apicv = vmx_get_enable_apicv,
+	.refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
 	.load_eoi_exitmap = vmx_load_eoi_exitmap,
 	.hwapic_irr_update = vmx_hwapic_irr_update,
 	.hwapic_isr_update = vmx_hwapic_isr_update,

arch/x86/kvm/x86.c+96−14

 	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 #endif
 	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
-	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS
+	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
 };
 static unsigned num_msrs_to_save;
 	HV_X64_MSR_RESET,
 	HV_X64_MSR_VP_INDEX,
 	HV_X64_MSR_VP_RUNTIME,
+	HV_X64_MSR_SCONTROL,
+	HV_X64_MSR_STIMER0_CONFIG,
 	HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
 	MSR_KVM_PV_EOI_EN,
 	++version;
-	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
+	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
+		return;
 	/*
 	 * The guest calculates current wall clock time by adding
 #endif
 }
+void kvm_make_mclock_inprogress_request(struct kvm *kvm)
+{
+	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
+}
 static void kvm_gen_update_masterclock(struct kvm *kvm)
 {
 #ifdef CONFIG_X86_64
 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
 	case HV_X64_MSR_CRASH_CTL:
+	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
 		return kvm_hv_set_msr_common(vcpu, msr, data,
 					     msr_info->host_initiated);
 	case MSR_IA32_BBL_CR_CTL3:
 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
 	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
 	case HV_X64_MSR_CRASH_CTL:
+	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
 		return kvm_hv_get_msr_common(vcpu,
 					     msr_info->index, &msr_info->data);
 		break;
 	case KVM_CAP_HYPERV:
 	case KVM_CAP_HYPERV_VAPIC:
 	case KVM_CAP_HYPERV_SPIN:
+	case KVM_CAP_HYPERV_SYNIC:
 	case KVM_CAP_PCI_SEGMENT:
 	case KVM_CAP_DEBUGREGS:
 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
 	return kvm_arch_has_noncoherent_dma(vcpu->kvm);
 }
+static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
+{
+	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
+}
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	/* Address WBINVD may be executed by guest */
 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
-	kvm_x86_ops->sync_pir_to_irr(vcpu);
+	if (vcpu->arch.apicv_active)
+		kvm_x86_ops->sync_pir_to_irr(vcpu);
 	memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
 	return 0;
 	return 0;
 }
+static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
+				     struct kvm_enable_cap *cap)
+{
+	if (cap->flags)
+		return -EINVAL;
+	switch (cap->cap) {
+	case KVM_CAP_HYPERV_SYNIC:
+		return kvm_hv_activate_synic(vcpu);
+	default:
+		return -EINVAL;
+	}
+}
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 		r = kvm_set_guest_paused(vcpu);
 		goto out;
 	}
+	case KVM_ENABLE_CAP: {
+		struct kvm_enable_cap cap;
+		r = -EFAULT;
+		if (copy_from_user(&cap, argp, sizeof(cap)))
+			goto out;
+		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
+		break;
+	}
 	default:
 		r = -EINVAL;
 	}
 		/*
 		 * Even MSRs that are valid in the host may not be exposed
-		 * to the guests in some cases.  We could work around this
+		 * to the guests in some cases.
-		 * in VMX with the generic MSR save/load machinery, but it
-		 * is not really worthwhile since it will really only
-		 * happen with nested virtualization.
 		 */
 		switch (msrs_to_save[i]) {
 		case MSR_IA32_BNDCFGS:
 			if (!kvm_x86_ops->mpx_supported())
 				continue;
 			break;
+		case MSR_TSC_AUX:
+			if (!kvm_x86_ops->rdtscp_supported())
+				continue;
+			break;
 		default:
 			break;
 		}
 	kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
 }
+void kvm_vcpu_deactivate_apicv(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.apicv_active = false;
+	kvm_x86_ops->refresh_apicv_exec_ctrl(vcpu);
+}
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
 	if (!vcpu->arch.apic)
 		return;
+	if (vcpu->arch.apicv_active)
+		return;
 	if (!vcpu->arch.apic->vapic_addr)
 		max_irr = kvm_lapic_find_highest_irr(vcpu);
 	else
 	kvm_mmu_reset_context(vcpu);
 }
+void kvm_make_scan_ioapic_request(struct kvm *kvm)
+{
+	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
+}
 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
 {
+	u64 eoi_exit_bitmap[4];
 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
 		return;
-	memset(vcpu->arch.eoi_exit_bitmap, 0, 256 / 8);
+	bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
 	if (irqchip_split(vcpu->kvm))
-		kvm_scan_ioapic_routes(vcpu, vcpu->arch.eoi_exit_bitmap);
+		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
 	else {
-		kvm_x86_ops->sync_pir_to_irr(vcpu);
+		if (vcpu->arch.apicv_active)
-		kvm_ioapic_scan_entry(vcpu, vcpu->arch.eoi_exit_bitmap);
+			kvm_x86_ops->sync_pir_to_irr(vcpu);
+		kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
 	}
-	kvm_x86_ops->load_eoi_exitmap(vcpu);
+	bitmap_or((ulong *)eoi_exit_bitmap, vcpu->arch.ioapic_handled_vectors,
+		  vcpu_to_synic(vcpu)->vec_bitmap, 256);
+	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
 }
 static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
 			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
 			if (test_bit(vcpu->arch.pending_ioapic_eoi,
-				     (void *) vcpu->arch.eoi_exit_bitmap)) {
+				     vcpu->arch.ioapic_handled_vectors)) {
 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
 				vcpu->run->eoi.vector =
 						vcpu->arch.pending_ioapic_eoi;
 			r = 0;
 			goto out;
 		}
+		if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
+			vcpu->run->exit_reason = KVM_EXIT_HYPERV;
+			vcpu->run->hyperv = vcpu->arch.hyperv.exit;
+			r = 0;
+			goto out;
+		}
+		/*
+		 * KVM_REQ_HV_STIMER has to be processed after
+		 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
+		 * depend on the guest clock being up-to-date
+		 */
+		if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
+			kvm_hv_process_stimers(vcpu);
 	}
 	/*
 		 * Update architecture specific hints for APIC
 		 * virtual interrupt delivery.
 		 */
-		if (kvm_x86_ops->hwapic_irr_update)
+		if (vcpu->arch.apicv_active)
 			kvm_x86_ops->hwapic_irr_update(vcpu,
 				kvm_lapic_find_highest_irr(vcpu));
 	}
 	BUG_ON(vcpu->kvm == NULL);
 	kvm = vcpu->kvm;
+	vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv();
 	vcpu->arch.pv.pv_unhalted = false;
 	vcpu->arch.emulate_ctxt.ops = &emulate_ops;
 	if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_reset_bsp(vcpu))
 	vcpu->arch.pending_external_vector = -1;
+	kvm_hv_vcpu_init(vcpu);
 	return 0;
 fail_free_mce_banks:
 {
 	int idx;
+	kvm_hv_vcpu_uninit(vcpu);
 	kvm_pmu_destroy(vcpu);
 	kfree(vcpu->arch.mce_banks);
 	kvm_free_lapic(vcpu);
 	    kvm_cpu_has_interrupt(vcpu))
 		return true;
+	if (kvm_hv_has_stimer_pending(vcpu))
+		return true;
 	return false;
 }

drivers/hv/hyperv_vmbus.h+1−87

@@ -63,10 +63,6 @@ enum hv_cpuid_function { · 62 unmodified lines
63	/* Define version of the synthetic interrupt controller. */	63	/* Define version of the synthetic interrupt controller. */
64	#define HV_SYNIC_VERSION (1)	64	#define HV_SYNIC_VERSION (1)
65		65
66	/* Define synthetic interrupt controller message constants. */
67	#define HV_MESSAGE_SIZE (256)
68	#define HV_MESSAGE_PAYLOAD_BYTE_COUNT (240)
69	#define HV_MESSAGE_PAYLOAD_QWORD_COUNT (30)
70	#define HV_ANY_VP (0xFFFFFFFF)	66	#define HV_ANY_VP (0xFFFFFFFF)
71		67
72	/* Define synthetic interrupt controller flag constants. */	68	/* Define synthetic interrupt controller flag constants. */
@@ -74,48 +70,9 @@ enum hv_cpuid_function { · 1 unmodified line
74	#define HV_EVENT_FLAGS_BYTE_COUNT (256)	70	#define HV_EVENT_FLAGS_BYTE_COUNT (256)
75	#define HV_EVENT_FLAGS_DWORD_COUNT (256 / sizeof(u32))	71	#define HV_EVENT_FLAGS_DWORD_COUNT (256 / sizeof(u32))
76		72
77	/* Define hypervisor message types. */
78	enum hv_message_type {
79	HVMSG_NONE = 0x00000000,
80
81	/* Memory access messages. */
82	HVMSG_UNMAPPED_GPA = 0x80000000,
83	HVMSG_GPA_INTERCEPT = 0x80000001,
84
85	/* Timer notification messages. */
86	HVMSG_TIMER_EXPIRED = 0x80000010,
87
88	/* Error messages. */
89	HVMSG_INVALID_VP_REGISTER_VALUE = 0x80000020,
90	HVMSG_UNRECOVERABLE_EXCEPTION = 0x80000021,
91	HVMSG_UNSUPPORTED_FEATURE = 0x80000022,
92
93	/* Trace buffer complete messages. */
94	HVMSG_EVENTLOG_BUFFERCOMPLETE = 0x80000040,
95
96	/* Platform-specific processor intercept messages. */
97	HVMSG_X64_IOPORT_INTERCEPT = 0x80010000,
98	HVMSG_X64_MSR_INTERCEPT = 0x80010001,
99	HVMSG_X64_CPUID_INTERCEPT = 0x80010002,
100	HVMSG_X64_EXCEPTION_INTERCEPT = 0x80010003,
101	HVMSG_X64_APIC_EOI = 0x80010004,
102	HVMSG_X64_LEGACY_FP_ERROR = 0x80010005
103	};
104
105	#define HV_SYNIC_STIMER_COUNT (4)
106
107	/* Define invalid partition identifier. */	73	/* Define invalid partition identifier. */
108	#define HV_PARTITION_ID_INVALID ((u64)0x0)	74	#define HV_PARTITION_ID_INVALID ((u64)0x0)
109		75
110	/* Define port identifier type. */
111	union hv_port_id {
112	u32 asu32;
113	struct {
114	u32 id:24;
115	u32 reserved:8;
116	} u ;
117	};
118
119	/* Define port type. */	76	/* Define port type. */
120	enum hv_port_type {	77	enum hv_port_type {
121	HVPORT_MSG = 1,	78	HVPORT_MSG = 1,
@@ -163,27 +120,6 @@ struct hv_connection_info { · 41 unmodified lines
163	};	120	};
164	};	121	};
165		122
166	/* Define synthetic interrupt controller message flags. */
167	union hv_message_flags {
168	u8 asu8;
169	struct {
170	u8 msg_pending:1;
171	u8 reserved:7;
172	};
173	};
174
175	/* Define synthetic interrupt controller message header. */
176	struct hv_message_header {
177	enum hv_message_type message_type;
178	u8 payload_size;
179	union hv_message_flags message_flags;
180	u8 reserved[2];
181	union {
182	u64 sender;
183	union hv_port_id port;
184	};
185	};
186
187	/*	123	/*
188	* Timer configuration register.	124	* Timer configuration register.
189	*/	125	*/
@@ -200,31 +136,9 @@ union hv_timer_config { · 10 unmodified lines
200	};	136	};
201	};	137	};
202		138
203
204	/* Define timer message payload structure. */
205	struct hv_timer_message_payload {
206	u32 timer_index;
207	u32 reserved;
208	u64 expiration_time; /* When the timer expired */
209	u64 delivery_time; /* When the message was delivered */
210	};
211
212	/* Define synthetic interrupt controller message format. */
213	struct hv_message {
214	struct hv_message_header header;
215	union {
216	u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
217	} u ;
218	};
219
220	/* Define the number of message buffers associated with each port. */	139	/* Define the number of message buffers associated with each port. */
221	#define HV_PORT_MESSAGE_BUFFER_COUNT (16)	140	#define HV_PORT_MESSAGE_BUFFER_COUNT (16)
222		141
223	/* Define the synthetic interrupt message page layout. */
224	struct hv_message_page {
225	struct hv_message sint_message[HV_SYNIC_SINT_COUNT];
226	};
227
228	/* Define the synthetic interrupt controller event flags format. */	142	/* Define the synthetic interrupt controller event flags format. */
229	union hv_synic_event_flags {	143	union hv_synic_event_flags {
230	u8 flags8[HV_EVENT_FLAGS_BYTE_COUNT];	144	u8 flags8[HV_EVENT_FLAGS_BYTE_COUNT];
@@ -347,7 +261,7 @@ enum hv_call_code { · 116 unmodified lines
347	struct hv_input_post_message {	261	struct hv_input_post_message {
348	union hv_connection_id connectionid;	262	union hv_connection_id connectionid;
349	u32 reserved;	263	u32 reserved;
350	enum hv_message_type message_type;	264	u32 message_type;
351	u32 payload_size;	265	u32 payload_size;
352	u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];	266	u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
353	};	267	};

drivers/s390/char/sclp_early.c+14−2

 	u8	fac85;			/* 85 */
 	u8	_pad_86[91 - 86];	/* 86-90 */
 	u8	flags;			/* 91 */
-	u8	_pad_92[100 - 92];	/* 92-99 */
+	u8	_pad_92[99 - 92];	/* 92-98 */
+	u8	hamaxpow;		/* 99 */
 	u32	rnsize2;		/* 100-103 */
 	u64	rnmax2;			/* 104-111 */
-	u8	_pad_112[120 - 112];	/* 112-119 */
+	u8	_pad_112[116 - 112];	/* 112-115 */
+	u8	fac116;			/* 116 */
+	u8	_pad_117[119 - 117];	/* 117-118 */
+	u8	fac119;			/* 119 */
 	u16	hcpua;			/* 120-121 */
 	u8	_pad_122[4096 - 122];	/* 122-4095 */
 } __packed __aligned(PAGE_SIZE);
 	sclp.facilities = sccb->facilities;
 	sclp.has_sprp = !!(sccb->fac84 & 0x02);
 	sclp.has_core_type = !!(sccb->fac84 & 0x01);
+	sclp.has_esca = !!(sccb->fac116 & 0x08);
+	sclp.has_hvs = !!(sccb->fac119 & 0x80);
 	if (sccb->fac85 & 0x02)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP;
 	sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2;
 	sclp.rzm <<= 20;
 	sclp.ibc = sccb->ibc;
+	if (sccb->hamaxpow && sccb->hamaxpow < 64)
+		sclp.hamax = (1UL << sccb->hamaxpow) - 1;
+	else
+		sclp.hamax = U64_MAX;
 	if (!sccb->hcpua) {
 		if (MACHINE_IS_VM)
 			sclp.max_cores = 64;
 			continue;
 		sclp.has_siif = cpue->siif;
 		sclp.has_sigpif = cpue->sigpif;
+		sclp.has_sief2 = cpue->sief2;
 		break;
 	}

include/clocksource/arm_arch_timer.h+6−0

 #define ARCH_TIMER_CTRL_IT_MASK		(1 << 1)
 #define ARCH_TIMER_CTRL_IT_STAT		(1 << 2)
+#define CNTHCTL_EL1PCTEN		(1 << 0)
+#define CNTHCTL_EL1PCEN			(1 << 1)
+#define CNTHCTL_EVNTEN			(1 << 2)
+#define CNTHCTL_EVNTDIR			(1 << 3)
+#define CNTHCTL_EVNTI			(0xF << 4)
 enum arch_timer_reg {
 	ARCH_TIMER_REG_CTRL,
 	ARCH_TIMER_REG_TVAL,

include/kvm/arm_vgic.h+6−0

 	u32		vgic_lr[VGIC_V2_MAX_LRS];
 };
+/*
+ * LRs are stored in reverse order in memory. make sure we index them
+ * correctly.
+ */
+#define VGIC_V3_LR_INDEX(lr)		(VGIC_V3_MAX_LRS - 1 - lr)
 struct vgic_v3_cpu_if {
 #ifdef CONFIG_KVM_ARM_VGIC_V3
 	u32		vgic_hcr;

include/linux/kvm_host.h+24−42

 }
 /*
- * vcpu->requests bit members
+ * Architecture-independent vcpu->requests bit members
+ * Bits 4-7 are reserved for more arch-independent bits.
  */
 #define KVM_REQ_TLB_FLUSH          0
-#define KVM_REQ_MIGRATE_TIMER      1
+#define KVM_REQ_MMU_RELOAD         1
-#define KVM_REQ_REPORT_TPR_ACCESS  2
+#define KVM_REQ_PENDING_TIMER      2
-#define KVM_REQ_MMU_RELOAD         3
+#define KVM_REQ_UNHALT             3
-#define KVM_REQ_TRIPLE_FAULT       4
-#define KVM_REQ_PENDING_TIMER      5
-#define KVM_REQ_UNHALT             6
-#define KVM_REQ_MMU_SYNC           7
-#define KVM_REQ_CLOCK_UPDATE       8
-#define KVM_REQ_KICK               9
-#define KVM_REQ_DEACTIVATE_FPU    10
-#define KVM_REQ_EVENT             11
-#define KVM_REQ_APF_HALT          12
-#define KVM_REQ_STEAL_UPDATE      13
-#define KVM_REQ_NMI               14
-#define KVM_REQ_PMU               15
-#define KVM_REQ_PMI               16
-#define KVM_REQ_WATCHDOG          17
-#define KVM_REQ_MASTERCLOCK_UPDATE 18
-#define KVM_REQ_MCLOCK_INPROGRESS 19
-#define KVM_REQ_EPR_EXIT          20
-#define KVM_REQ_SCAN_IOAPIC       21
-#define KVM_REQ_GLOBAL_CLOCK_UPDATE 22
-#define KVM_REQ_ENABLE_IBS        23
-#define KVM_REQ_DISABLE_IBS       24
-#define KVM_REQ_APIC_PAGE_RELOAD  25
-#define KVM_REQ_SMI               26
-#define KVM_REQ_HV_CRASH          27
-#define KVM_REQ_IOAPIC_EOI_EXIT   28
-#define KVM_REQ_HV_RESET          29
 #define KVM_USERSPACE_IRQ_SOURCE_ID		0
 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1
 	u32 adapter_id;
 };
+struct kvm_hv_sint {
+	u32 vcpu;
+	u32 sint;
+};
 struct kvm_kernel_irq_routing_entry {
 	u32 gsi;
 	u32 type;
 		} irqchip;
 		struct msi_msg msi;
 		struct kvm_s390_adapter_int adapter;
+		struct kvm_hv_sint hv_sint;
 	};
 	struct hlist_node link;
 };
 /* The guest did something we don't support. */
 #define vcpu_unimpl(vcpu, fmt, ...)					\
-	kvm_pr_unimpl("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
+	kvm_pr_unimpl("vcpu%i, guest rIP: 0x%lx " fmt,			\
+			(vcpu)->vcpu_id, kvm_rip_read(vcpu), ## __VA_ARGS__)
 #define vcpu_debug(vcpu, fmt, ...)					\
 	kvm_debug("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
+#define vcpu_err(vcpu, fmt, ...)					\
+	kvm_err("vcpu%i " fmt, (vcpu)->vcpu_id, ## __VA_ARGS__)
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
 {
 	struct kvm_vcpu *vcpu;
 	int i;
+	if (id < 0 || id >= KVM_MAX_VCPUS)
+		return NULL;
+	vcpu = kvm_get_vcpu(kvm, id);
+	if (vcpu && vcpu->vcpu_id == id)
+		return vcpu;
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		if (vcpu->vcpu_id == id)
 			return vcpu;
 #ifdef __KVM_HAVE_IOAPIC
 void kvm_vcpu_request_scan_ioapic(struct kvm *kvm);
-void kvm_arch_irq_routing_update(struct kvm *kvm);
+void kvm_arch_post_irq_routing_update(struct kvm *kvm);
 #else
 static inline void kvm_vcpu_request_scan_ioapic(struct kvm *kvm)
 {
 }
-static inline void kvm_arch_irq_routing_update(struct kvm *kvm)
+static inline void kvm_arch_post_irq_routing_update(struct kvm *kvm)
 {
 }
 #endif
 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
-int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
+bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn);
 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
 void kvm_flush_remote_tlbs(struct kvm *kvm);
 void kvm_reload_remote_mmus(struct kvm *kvm);
-void kvm_make_mclock_inprogress_request(struct kvm *kvm);
-void kvm_make_scan_ioapic_request(struct kvm *kvm);
 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req);
 long kvm_arch_dev_ioctl(struct file *filp,
 	return kvm_is_error_hva(hva);
 }
-static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
-{
-	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
-}
 enum kvm_stat_kind {
 	KVM_STAT_VM,
 	KVM_STAT_VCPU,
 	const char *name;
 	int offset;
 	enum kvm_stat_kind kind;
-	struct dentry *dentry;
 };
 extern struct kvm_stats_debugfs_item debugfs_entries[];
 extern struct dentry *kvm_debugfs_dir;
 {
 }
 #endif
+void kvm_arch_irq_routing_update(struct kvm *kvm);
 static inline int kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
 {

include/linux/kvm_para.h+2−4

 #include <uapi/linux/kvm_para.h>
-static inline int kvm_para_has_feature(unsigned int feature)
+static inline bool kvm_para_has_feature(unsigned int feature)
 {
-	if (kvm_arch_para_features() & (1UL << feature))
+	return !!(kvm_arch_para_features() & (1UL << feature));
-		return 1;
-	return 0;
 }
 #endif /* __LINUX_KVM_PARA_H */

include/uapi/linux/kvm.h+26−0

 	__u32 flags;
 	__u32 reserved[9];
 };
+struct kvm_hyperv_exit {
+#define KVM_EXIT_HYPERV_SYNIC          1
+	__u32 type;
+	union {
+		struct {
+			__u32 msr;
+			__u64 control;
+			__u64 evt_page;
+			__u64 msg_page;
+		} synic;
+	} u;
+};
 #define KVM_S390_GET_SKEYS_NONE   1
 #define KVM_S390_SKEYS_MAX        1048576
 #define KVM_EXIT_SYSTEM_EVENT     24
 #define KVM_EXIT_S390_STSI        25
 #define KVM_EXIT_IOAPIC_EOI       26
+#define KVM_EXIT_HYPERV           27
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
 		struct {
 			__u8 vector;
 		} eoi;
+		/* KVM_EXIT_HYPERV */
+		struct kvm_hyperv_exit hyperv;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
 #define KVM_CAP_GUEST_DEBUG_HW_WPS 120
 #define KVM_CAP_SPLIT_IRQCHIP 121
 #define KVM_CAP_IOEVENTFD_ANY_LENGTH 122
+#define KVM_CAP_HYPERV_SYNIC 123
+#define KVM_CAP_S390_RI 124
 #ifdef KVM_CAP_IRQ_ROUTING
 	__u32 adapter_id;
 };
+struct kvm_irq_routing_hv_sint {
+	__u32 vcpu;
+	__u32 sint;
+};
 /* gsi routing entry types */
 #define KVM_IRQ_ROUTING_IRQCHIP 1
 #define KVM_IRQ_ROUTING_MSI 2
 #define KVM_IRQ_ROUTING_S390_ADAPTER 3
+#define KVM_IRQ_ROUTING_HV_SINT 4
 struct kvm_irq_routing_entry {
 	__u32 gsi;
 		struct kvm_irq_routing_irqchip irqchip;
 		struct kvm_irq_routing_msi msi;
 		struct kvm_irq_routing_s390_adapter adapter;
+		struct kvm_irq_routing_hv_sint hv_sint;
 		__u32 pad[8];
 	} u;
 };

virt/kvm/arm/vgic-v3.c+3−8

 #include <asm/kvm_emulate.h>
 #include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
 #include <asm/kvm_mmu.h>
 /* These are for GICv2 emulation only */
 #define GICH_LR_PHYSID_CPUID		(7UL << GICH_LR_PHYSID_CPUID_SHIFT)
 #define ICH_LR_VIRTUALID_MASK		(BIT_ULL(32) - 1)
-/*
- * LRs are stored in reverse order in memory. make sure we index them
- * correctly.
- */
-#define LR_INDEX(lr)			(VGIC_V3_MAX_LRS - 1 - lr)
 static u32 ich_vtr_el2;
 static struct vgic_lr vgic_v3_get_lr(const struct kvm_vcpu *vcpu, int lr)
 {
 	struct vgic_lr lr_desc;
-	u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[LR_INDEX(lr)];
+	u64 val = vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[VGIC_V3_LR_INDEX(lr)];
 	if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
 		lr_desc.irq = val & ICH_LR_VIRTUALID_MASK;
 		lr_val |= ((u64)lr_desc.hwirq) << ICH_LR_PHYS_ID_SHIFT;
 	}
-	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[LR_INDEX(lr)] = lr_val;
+	vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[VGIC_V3_LR_INDEX(lr)] = lr_val;
 	if (!(lr_desc.state & LR_STATE_MASK))
 		vcpu->arch.vgic_cpu.vgic_v3.vgic_elrsr |= (1U << lr);

virt/kvm/arm/vgic.c+1−1

@@ -878,7 +878,7 @@ static int vgic_handle_mmio_write(struct kvm_vcpu *vcpu, · 877 unmodified lines
878	true);	878	true);
879	}	879	}
880		880
881	struct kvm_io_device_ops vgic_io_ops = {	881	static struct kvm_io_device_ops vgic_io_ops = {
882	.read = vgic_handle_mmio_read,	882	.read = vgic_handle_mmio_read,
883	.write = vgic_handle_mmio_write,	883	.write = vgic_handle_mmio_write,
884	};	884	};

virt/kvm/async_pf.c+1−2

@@ -57,8 +57,7 @@ int kvm_async_pf_init(void) · 56 unmodified lines
57		57
58	void kvm_async_pf_deinit(void)	58	void kvm_async_pf_deinit(void)
59	{	59	{
60	if (async_pf_cache)	60	kmem_cache_destroy(async_pf_cache);
61	kmem_cache_destroy(async_pf_cache);
62	async_pf_cache = NULL;	61	async_pf_cache = NULL;
63	}	62	}
64		63

virt/kvm/irqchip.c+6−1

 	return r;
 }
+void __attribute__((weak)) kvm_arch_irq_routing_update(struct kvm *kvm)
+{
+}
 int kvm_set_irq_routing(struct kvm *kvm,
 			const struct kvm_irq_routing_entry *ue,
 			unsigned nr,
 	old = kvm->irq_routing;
 	rcu_assign_pointer(kvm->irq_routing, new);
 	kvm_irq_routing_update(kvm);
+	kvm_arch_irq_routing_update(kvm);
 	mutex_unlock(&kvm->irq_lock);
-	kvm_arch_irq_routing_update(kvm);
+	kvm_arch_post_irq_routing_update(kvm);
 	synchronize_srcu_expedited(&kvm->irq_srcu);

virt/kvm/kvm_main.c+12−34

@@ -206,16 +206,6 @@ void kvm_reload_remote_mmus(struct kvm *kvm) · 205 unmodified lines
206	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);	206	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
207	}	207	}
208		208
209	void kvm_make_mclock_inprogress_request(struct kvm *kvm)
210	{
211	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
212	}
213
214	void kvm_make_scan_ioapic_request(struct kvm *kvm)
215	{
216	kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
217	}
218
219	int kvm_vcpu_init(struct kvm_vcpu vcpu, struct kvm kvm, unsigned id)	209	int kvm_vcpu_init(struct kvm_vcpu vcpu, struct kvm kvm, unsigned id)
220	{	210	{
221	struct page *page;	211	struct page *page;
@@ -1164,15 +1154,15 @@ struct kvm_memory_slot kvm_vcpu_gfn_to_memslot(struct kvm_vcpu vcpu, gfn_t gfn · 942 unmodified lines
1164	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);	1154	return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1165	}	1155	}
1166		1156
1167	int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)	1157	bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1168	{	1158	{
1169	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);	1159	struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1170		1160
1171	if (!memslot \|\| memslot->id >= KVM_USER_MEM_SLOTS \|\|	1161	if (!memslot \|\| memslot->id >= KVM_USER_MEM_SLOTS \|\|
1172	memslot->flags & KVM_MEMSLOT_INVALID)	1162	memslot->flags & KVM_MEMSLOT_INVALID)
1173	return 0;	1163	return false;
1174		1164
1175	return 1;	1165	return true;
1176	}	1166	}
1177	EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);	1167	EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1178		1168
@@ -2257,7 +2247,7 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) · 1078 unmodified lines
2257	static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)	2247	static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2258	{	2248	{
2259	int r;	2249	int r;
2260	struct kvm_vcpu vcpu, v;	2250	struct kvm_vcpu *vcpu;
2261		2251
2262	if (id >= KVM_MAX_VCPUS)	2252	if (id >= KVM_MAX_VCPUS)
2263	return -EINVAL;	2253	return -EINVAL;
@@ -2281,12 +2271,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) · 17 unmodified lines
2281	r = -EINVAL;	2271	r = -EINVAL;
2282	goto unlock_vcpu_destroy;	2272	goto unlock_vcpu_destroy;
2283	}	2273	}
2284		2274	if (kvm_get_vcpu_by_id(kvm, id)) {
2285	kvm_for_each_vcpu(r, v, kvm)	2275	r = -EEXIST;
2286	if (v->vcpu_id == id) {	2276	goto unlock_vcpu_destroy;
2287	r = -EEXIST;	2277	}
2288	goto unlock_vcpu_destroy;
2289	}
2290		2278
2291	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);	2279	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2292		2280
@@ -3449,10 +3437,9 @@ static int kvm_init_debug(void) · 1156 unmodified lines
3449	goto out;	3437	goto out;
3450		3438
3451	for (p = debugfs_entries; p->name; ++p) {	3439	for (p = debugfs_entries; p->name; ++p) {
3452	p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,	3440	if (!debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
3453	(void *)(long)p->offset,	3441	(void *)(long)p->offset,
3454	stat_fops[p->kind]);	3442	stat_fops[p->kind]))
3455	if (p->dentry == NULL)
3456	goto out_dir;	3443	goto out_dir;
3457	}	3444	}
3458		3445
@@ -3464,15 +3451,6 @@ out: · 5 unmodified lines
3464	return r;	3451	return r;
3465	}	3452	}
3466		3453
3467	static void kvm_exit_debug(void)
3468	{
3469	struct kvm_stats_debugfs_item *p;
3470
3471	for (p = debugfs_entries; p->name; ++p)
3472	debugfs_remove(p->dentry);
3473	debugfs_remove(kvm_debugfs_dir);
3474	}
3475
3476	static int kvm_suspend(void)	3454	static int kvm_suspend(void)
3477	{	3455	{
3478	if (kvm_usage_count)	3456	if (kvm_usage_count)
@@ -3630,7 +3608,7 @@ EXPORT_SYMBOL_GPL(kvm_init); · 151 unmodified lines
3630		3608
3631	void kvm_exit(void)	3609	void kvm_exit(void)
3632	{	3610	{
3633	kvm_exit_debug();	3611	debugfs_remove_recursive(kvm_debugfs_dir);
3634	misc_deregister(&kvm_dev);	3612	misc_deregister(&kvm_dev);
3635	kmem_cache_destroy(kvm_vcpu_cache);	3613	kmem_cache_destroy(kvm_vcpu_cache);
3636	kvm_async_pf_deinit();	3614	kvm_async_pf_deinit();

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code