History

Merge branch 'for-rmk/virt/kvm/core' of git://git.kernel.org/pub/scm/linux/kernel/git/will/linux into next/virt

* 'for-rmk/virt/kvm/core' of git://git.kernel.org/pub/scm/linux/kernel/git/will/linux: KVM: ARM: Add maintainer entry for KVM/ARM KVM: ARM: Power State Coordination Interface implementation KVM: ARM: Handle I/O aborts KVM: ARM: Handle guest faults in KVM KVM: ARM: VFP userspace interface KVM: ARM: Demux CCSIDR in the userspace API KVM: ARM: User space API for getting/setting co-proc registers KVM: ARM: Emulation framework and CP15 emulation KVM: ARM: World-switch implementation KVM: ARM: Inject IRQs and FIQs from userspace KVM: ARM: Memory virtualization setup KVM: ARM: Hypervisor initialization KVM: ARM: Initial skeleton to compile KVM support ARM: Section based HYP idmap ARM: Add page table and page defines needed by KVM

Olof Johansson committed 13 years ago

Commit 655e1942 parents 1e044f0 da141b638 files · +6546 −20Browse files

Unified · Split

38 files
  1. Documentation/virtual/kvm/api.txt +91 −8
  2. MAINTAINERS +9 −0
  3. arch/arm/Kconfig +2 −0
  4. arch/arm/Makefile +1 −0
  5. arch/arm/include/asm/idmap.h +1 −0
  6. arch/arm/include/asm/kvm_arm.h +214 −0
  7. arch/arm/include/asm/kvm_asm.h +82 −0
  8. arch/arm/include/asm/kvm_coproc.h +47 −0
  9. arch/arm/include/asm/kvm_emulate.h +72 −0
  10. arch/arm/include/asm/kvm_host.h +161 −0
  11. arch/arm/include/asm/kvm_mmio.h +56 −0
  12. arch/arm/include/asm/kvm_mmu.h +50 −0
  13. arch/arm/include/asm/kvm_psci.h +23 −0
  14. arch/arm/include/asm/pgtable-3level-hwdef.h +5 −0
  15. arch/arm/include/asm/pgtable-3level.h +18 −0
  16. arch/arm/include/asm/pgtable.h +7 −0
  17. arch/arm/include/uapi/asm/kvm.h +164 −0
  18. arch/arm/kernel/asm-offsets.c +25 −0
  19. arch/arm/kernel/vmlinux.lds.S +5 −1
  20. arch/arm/kvm/Kconfig +56 −0
  21. arch/arm/kvm/Makefile +21 −0
  22. arch/arm/kvm/arm.c +1015 −0
  23. arch/arm/kvm/coproc.c +1046 −0
  24. arch/arm/kvm/coproc.h +153 −0
  25. arch/arm/kvm/coproc_a15.c +162 −0
  26. arch/arm/kvm/emulate.c +373 −0
  27. arch/arm/kvm/guest.c +222 −0
  28. arch/arm/kvm/init.S +114 −0
  29. arch/arm/kvm/interrupts.S +478 −0
  30. arch/arm/kvm/interrupts_head.S +441 −0
  31. arch/arm/kvm/mmio.c +153 −0
  32. arch/arm/kvm/mmu.c +787 −0
  33. arch/arm/kvm/psci.c +108 −0
  34. arch/arm/kvm/reset.c +74 −0
  35. arch/arm/kvm/trace.h +235 −0
  36. arch/arm/mm/idmap.c +44 −11
  37. arch/arm/mm/mmu.c +22 −0
  38. include/uapi/linux/kvm.h +9 −0
Documentation/virtual/kvm/api.txt+91−8
@@ -293,7 +293,7 @@ kvm_run' (see below). · 292 unmodified lines
293293 4.11 KVM_GET_REGS
294294
295295 Capability: basic
296−Architectures: all
296+Architectures: all except ARM
297297 Type: vcpu ioctl
298298 Parameters: struct kvm_regs (out)
299299 Returns: 0 on success, -1 on error
@@ -314,7 +314,7 @@ struct kvm_regs { · 14 unmodified lines
314314 4.12 KVM_SET_REGS
315315
316316 Capability: basic
317−Architectures: all
317+Architectures: all except ARM
318318 Type: vcpu ioctl
319319 Parameters: struct kvm_regs (in)
320320 Returns: 0 on success, -1 on error
@@ -600,7 +600,7 @@ struct kvm_fpu { · 279 unmodified lines
600600 4.24 KVM_CREATE_IRQCHIP
601601
602602 Capability: KVM_CAP_IRQCHIP
603−Architectures: x86, ia64
603+Architectures: x86, ia64, ARM
604604 Type: vm ioctl
605605 Parameters: none
606606 Returns: 0 on success, -1 on error
@@ -608,21 +608,39 @@ Returns: 0 on success, -1 on error · 1 unmodified line
608608 Creates an interrupt controller model in the kernel. On x86, creates a virtual
609609 ioapic, a virtual PIC (two PICs, nested), and sets up future vcpus to have a
610610 local APIC. IRQ routing for GSIs 0-15 is set to both PIC and IOAPIC; GSI 16-23
611−only go to the IOAPIC. On ia64, a IOSAPIC is created.
611+only go to the IOAPIC. On ia64, a IOSAPIC is created. On ARM, a GIC is
612+created.
612613
613614
614615 4.25 KVM_IRQ_LINE
615616
616617 Capability: KVM_CAP_IRQCHIP
617−Architectures: x86, ia64
618+Architectures: x86, ia64, arm
618619 Type: vm ioctl
619620 Parameters: struct kvm_irq_level
620621 Returns: 0 on success, -1 on error
621622
622623 Sets the level of a GSI input to the interrupt controller model in the kernel.
623−Requires that an interrupt controller model has been previously created with
624−KVM_CREATE_IRQCHIP. Note that edge-triggered interrupts require the level
625−to be set to 1 and then back to 0.
624+On some architectures it is required that an interrupt controller model has
625+been previously created with KVM_CREATE_IRQCHIP. Note that edge-triggered
626+interrupts require the level to be set to 1 and then back to 0.
627+
628+ARM can signal an interrupt either at the CPU level, or at the in-kernel irqchip
629+(GIC), and for in-kernel irqchip can tell the GIC to use PPIs designated for
630+specific cpus. The irq field is interpreted like this:
631+
632+  bits: | 31 ... 24 | 23 ... 16 | 15 ... 0 |
633+ field: | irq_type | vcpu_index | irq_id |
634+
635+The irq_type field has the following values:
636+- irq_type[0]: out-of-kernel GIC: irq_id 0 is IRQ, irq_id 1 is FIQ
637+- irq_type[1]: in-kernel GIC: SPI, irq_id between 32 and 1019 (incl.)
638+ (the vcpu_index field is ignored)
639+- irq_type[2]: in-kernel GIC: PPI, irq_id between 16 and 31 (incl.)
640+
641+(The irq_id field thus corresponds nicely to the IRQ ID in the ARM GIC specs)
642+
643+In both cases, level is used to raise/lower the line.
626644
627645 struct kvm_irq_level {
628646 union {
@@ -1775,6 +1793,27 @@ registers, find a list below: · 1146 unmodified lines
17751793 PPC | KVM_REG_PPC_VPA_DTL | 128
17761794 PPC | KVM_REG_PPC_EPCR | 32
17771795
1796+ARM registers are mapped using the lower 32 bits. The upper 16 of that
1797+is the register group type, or coprocessor number:
1798+
1799+ARM core registers have the following id bit patterns:
1800+ 0x4002 0000 0010 <index into the kvm_regs struct:16>
1801+
1802+ARM 32-bit CP15 registers have the following id bit patterns:
1803+ 0x4002 0000 000F <zero:1> <crn:4> <crm:4> <opc1:4> <opc2:3>
1804+
1805+ARM 64-bit CP15 registers have the following id bit patterns:
1806+ 0x4003 0000 000F <zero:1> <zero:4> <crm:4> <opc1:4> <zero:3>
1807+
1808+ARM CCSIDR registers are demultiplexed by CSSELR value:
1809+ 0x4002 0000 0011 00 <csselr:8>
1810+
1811+ARM 32-bit VFP control registers have the following id bit patterns:
1812+ 0x4002 0000 0012 1 <regno:12>
1813+
1814+ARM 64-bit FP registers have the following id bit patterns:
1815+ 0x4002 0000 0012 0 <regno:12>
1816+
17781817 4.69 KVM_GET_ONE_REG
17791818
17801819 Capability: KVM_CAP_ONE_REG
@@ -2127,6 +2166,50 @@ written, then `n_invalid' invalid entries, invalidating any previously · 346 unmodified lines
21272166 valid entries found.
21282167
21292168
2169+4.77 KVM_ARM_VCPU_INIT
2170+
2171+Capability: basic
2172+Architectures: arm
2173+Type: vcpu ioctl
2174+Parameters: struct struct kvm_vcpu_init (in)
2175+Returns: 0 on success; -1 on error
2176+Errors:
2177+  EINVAL:    the target is unknown, or the combination of features is invalid.
2178+  ENOENT:    a features bit specified is unknown.
2179+
2180+This tells KVM what type of CPU to present to the guest, and what
2181+optional features it should have.  This will cause a reset of the cpu
2182+registers to their initial values.  If this is not called, KVM_RUN will
2183+return ENOEXEC for that vcpu.
2184+
2185+Note that because some registers reflect machine topology, all vcpus
2186+should be created before this ioctl is invoked.
2187+
2188+Possible features:
2189+ - KVM_ARM_VCPU_POWER_OFF: Starts the CPU in a power-off state.
2190+ Depends on KVM_CAP_ARM_PSCI.
2191+
2192+
2193+4.78 KVM_GET_REG_LIST
2194+
2195+Capability: basic
2196+Architectures: arm
2197+Type: vcpu ioctl
2198+Parameters: struct kvm_reg_list (in/out)
2199+Returns: 0 on success; -1 on error
2200+Errors:
2201+  E2BIG:     the reg index list is too big to fit in the array specified by
2202+             the user (the number required will be written into n).
2203+
2204+struct kvm_reg_list {
2205+ __u64 n; /* number of registers in reg[] */
2206+ __u64 reg[0];
2207+};
2208+
2209+This ioctl returns the guest registers that are supported for the
2210+KVM_GET_ONE_REG/KVM_SET_ONE_REG calls.
2211+
2212+
21302213 5. The kvm_run structure
21312214 ------------------------
21322215
MAINTAINERS+9−0
@@ -4481,6 +4481,15 @@ F: arch/s390/include/asm/kvm* · 4480 unmodified lines
44814481 F: arch/s390/kvm/
44824482 F: drivers/s390/kvm/
44834483
4484+KERNEL VIRTUAL MACHINE (KVM) FOR ARM
4485+M: Christoffer Dall <cdall@cs.columbia.edu>
4486+L: kvmarm@lists.cs.columbia.edu
4487+W: http://systems.cs.columbia.edu/projects/kvm-arm
4488+S: Maintained
4489+F: arch/arm/include/uapi/asm/kvm*
4490+F: arch/arm/include/asm/kvm*
4491+F: arch/arm/kvm/
4492+
44844493 KEXEC
44854494 M: Eric Biederman <ebiederm@xmission.com>
44864495 W: http://kernel.org/pub/linux/utils/kernel/kexec/
arch/arm/Kconfig+2−0
@@ -2322,3 +2322,5 @@ source "security/Kconfig" · 2321 unmodified lines
23222322 source "crypto/Kconfig"
23232323
23242324 source "lib/Kconfig"
2325+
2326+source "arch/arm/kvm/Kconfig"
arch/arm/Makefile+1−0
@@ -252,6 +252,7 @@ core-$(CONFIG_FPE_NWFPE) += arch/arm/nwfpe/ · 251 unmodified lines
252252 core-$(CONFIG_FPE_FASTFPE) += $(FASTFPE_OBJ)
253253 core-$(CONFIG_VFP) += arch/arm/vfp/
254254 core-$(CONFIG_XEN) += arch/arm/xen/
255+core-$(CONFIG_KVM_ARM_HOST) += arch/arm/kvm/
255256
256257 # If we have a machine-specific directory, then include it in the build.
257258 core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/
arch/arm/include/asm/idmap.h+1−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #define __idmap __section(.idmap.text) noinline notrace
99
1010 extern pgd_t *idmap_pgd;
11+extern pgd_t *hyp_pgd;
1112
1213 void setup_mm_for_reboot(void);
1314
arch/arm/include/asm/kvm_arm.h+214−0
@@ -0,0 +1,214 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_ARM_H__
20+#define __ARM_KVM_ARM_H__
21+
22+#include <linux/types.h>
23+
24+/* Hyp Configuration Register (HCR) bits */
25+#define HCR_TGE (1 << 27)
26+#define HCR_TVM (1 << 26)
27+#define HCR_TTLB (1 << 25)
28+#define HCR_TPU (1 << 24)
29+#define HCR_TPC (1 << 23)
30+#define HCR_TSW (1 << 22)
31+#define HCR_TAC (1 << 21)
32+#define HCR_TIDCP (1 << 20)
33+#define HCR_TSC (1 << 19)
34+#define HCR_TID3 (1 << 18)
35+#define HCR_TID2 (1 << 17)
36+#define HCR_TID1 (1 << 16)
37+#define HCR_TID0 (1 << 15)
38+#define HCR_TWE (1 << 14)
39+#define HCR_TWI (1 << 13)
40+#define HCR_DC (1 << 12)
41+#define HCR_BSU (3 << 10)
42+#define HCR_BSU_IS (1 << 10)
43+#define HCR_FB (1 << 9)
44+#define HCR_VA (1 << 8)
45+#define HCR_VI (1 << 7)
46+#define HCR_VF (1 << 6)
47+#define HCR_AMO (1 << 5)
48+#define HCR_IMO (1 << 4)
49+#define HCR_FMO (1 << 3)
50+#define HCR_PTW (1 << 2)
51+#define HCR_SWIO (1 << 1)
52+#define HCR_VM 1
53+
54+/*
55+ * The bits we set in HCR:
56+ * TAC: Trap ACTLR
57+ * TSC: Trap SMC
58+ * TSW: Trap cache operations by set/way
59+ * TWI: Trap WFI
60+ * TIDCP: Trap L2CTLR/L2ECTLR
61+ * BSU_IS: Upgrade barriers to the inner shareable domain
62+ * FB: Force broadcast of all maintainance operations
63+ * AMO: Override CPSR.A and enable signaling with VA
64+ * IMO: Override CPSR.I and enable signaling with VI
65+ * FMO: Override CPSR.F and enable signaling with VF
66+ * SWIO: Turn set/way invalidates into set/way clean+invalidate
67+ */
68+#define HCR_GUEST_MASK (HCR_TSC | HCR_TSW | HCR_TWI | HCR_VM | HCR_BSU_IS | \
69+ HCR_FB | HCR_TAC | HCR_AMO | HCR_IMO | HCR_FMO | \
70+ HCR_SWIO | HCR_TIDCP)
71+#define HCR_VIRT_EXCP_MASK (HCR_VA | HCR_VI | HCR_VF)
72+
73+/* System Control Register (SCTLR) bits */
74+#define SCTLR_TE (1 << 30)
75+#define SCTLR_EE (1 << 25)
76+#define SCTLR_V (1 << 13)
77+
78+/* Hyp System Control Register (HSCTLR) bits */
79+#define HSCTLR_TE (1 << 30)
80+#define HSCTLR_EE (1 << 25)
81+#define HSCTLR_FI (1 << 21)
82+#define HSCTLR_WXN (1 << 19)
83+#define HSCTLR_I (1 << 12)
84+#define HSCTLR_C (1 << 2)
85+#define HSCTLR_A (1 << 1)
86+#define HSCTLR_M 1
87+#define HSCTLR_MASK (HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I | \
88+ HSCTLR_WXN | HSCTLR_FI | HSCTLR_EE | HSCTLR_TE)
89+
90+/* TTBCR and HTCR Registers bits */
91+#define TTBCR_EAE (1 << 31)
92+#define TTBCR_IMP (1 << 30)
93+#define TTBCR_SH1 (3 << 28)
94+#define TTBCR_ORGN1 (3 << 26)
95+#define TTBCR_IRGN1 (3 << 24)
96+#define TTBCR_EPD1 (1 << 23)
97+#define TTBCR_A1 (1 << 22)
98+#define TTBCR_T1SZ (3 << 16)
99+#define TTBCR_SH0 (3 << 12)
100+#define TTBCR_ORGN0 (3 << 10)
101+#define TTBCR_IRGN0 (3 << 8)
102+#define TTBCR_EPD0 (1 << 7)
103+#define TTBCR_T0SZ 3
104+#define HTCR_MASK (TTBCR_T0SZ | TTBCR_IRGN0 | TTBCR_ORGN0 | TTBCR_SH0)
105+
106+/* Hyp System Trap Register */
107+#define HSTR_T(x) (1 << x)
108+#define HSTR_TTEE (1 << 16)
109+#define HSTR_TJDBX (1 << 17)
110+
111+/* Hyp Coprocessor Trap Register */
112+#define HCPTR_TCP(x) (1 << x)
113+#define HCPTR_TCP_MASK (0x3fff)
114+#define HCPTR_TASE (1 << 15)
115+#define HCPTR_TTA (1 << 20)
116+#define HCPTR_TCPAC (1 << 31)
117+
118+/* Hyp Debug Configuration Register bits */
119+#define HDCR_TDRA (1 << 11)
120+#define HDCR_TDOSA (1 << 10)
121+#define HDCR_TDA (1 << 9)
122+#define HDCR_TDE (1 << 8)
123+#define HDCR_HPME (1 << 7)
124+#define HDCR_TPM (1 << 6)
125+#define HDCR_TPMCR (1 << 5)
126+#define HDCR_HPMN_MASK (0x1F)
127+
128+/*
129+ * The architecture supports 40-bit IPA as input to the 2nd stage translations
130+ * and PTRS_PER_S2_PGD becomes 1024, because each entry covers 1GB of address
131+ * space.
132+ */
133+#define KVM_PHYS_SHIFT (40)
134+#define KVM_PHYS_SIZE (1ULL << KVM_PHYS_SHIFT)
135+#define KVM_PHYS_MASK (KVM_PHYS_SIZE - 1ULL)
136+#define PTRS_PER_S2_PGD (1ULL << (KVM_PHYS_SHIFT - 30))
137+#define S2_PGD_ORDER get_order(PTRS_PER_S2_PGD * sizeof(pgd_t))
138+#define S2_PGD_SIZE (1 << S2_PGD_ORDER)
139+
140+/* Virtualization Translation Control Register (VTCR) bits */
141+#define VTCR_SH0 (3 << 12)
142+#define VTCR_ORGN0 (3 << 10)
143+#define VTCR_IRGN0 (3 << 8)
144+#define VTCR_SL0 (3 << 6)
145+#define VTCR_S (1 << 4)
146+#define VTCR_T0SZ (0xf)
147+#define VTCR_MASK (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0 | VTCR_SL0 | \
148+ VTCR_S | VTCR_T0SZ)
149+#define VTCR_HTCR_SH (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0)
150+#define VTCR_SL_L2 (0 << 6) /* Starting-level: 2 */
151+#define VTCR_SL_L1 (1 << 6) /* Starting-level: 1 */
152+#define KVM_VTCR_SL0 VTCR_SL_L1
153+/* stage-2 input address range defined as 2^(32-T0SZ) */
154+#define KVM_T0SZ (32 - KVM_PHYS_SHIFT)
155+#define KVM_VTCR_T0SZ (KVM_T0SZ & VTCR_T0SZ)
156+#define KVM_VTCR_S ((KVM_VTCR_T0SZ << 1) & VTCR_S)
157+
158+/* Virtualization Translation Table Base Register (VTTBR) bits */
159+#if KVM_VTCR_SL0 == VTCR_SL_L2 /* see ARM DDI 0406C: B4-1720 */
160+#define VTTBR_X (14 - KVM_T0SZ)
161+#else
162+#define VTTBR_X (5 - KVM_T0SZ)
163+#endif
164+#define VTTBR_BADDR_SHIFT (VTTBR_X - 1)
165+#define VTTBR_BADDR_MASK (((1LLU << (40 - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT)
166+#define VTTBR_VMID_SHIFT (48LLU)
167+#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT)
168+
169+/* Hyp Syndrome Register (HSR) bits */
170+#define HSR_EC_SHIFT (26)
171+#define HSR_EC (0x3fU << HSR_EC_SHIFT)
172+#define HSR_IL (1U << 25)
173+#define HSR_ISS (HSR_IL - 1)
174+#define HSR_ISV_SHIFT (24)
175+#define HSR_ISV (1U << HSR_ISV_SHIFT)
176+#define HSR_SRT_SHIFT (16)
177+#define HSR_SRT_MASK (0xf << HSR_SRT_SHIFT)
178+#define HSR_FSC (0x3f)
179+#define HSR_FSC_TYPE (0x3c)
180+#define HSR_SSE (1 << 21)
181+#define HSR_WNR (1 << 6)
182+#define HSR_CV_SHIFT (24)
183+#define HSR_CV (1U << HSR_CV_SHIFT)
184+#define HSR_COND_SHIFT (20)
185+#define HSR_COND (0xfU << HSR_COND_SHIFT)
186+
187+#define FSC_FAULT (0x04)
188+#define FSC_PERM (0x0c)
189+
190+/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
191+#define HPFAR_MASK (~0xf)
192+
193+#define HSR_EC_UNKNOWN (0x00)
194+#define HSR_EC_WFI (0x01)
195+#define HSR_EC_CP15_32 (0x03)
196+#define HSR_EC_CP15_64 (0x04)
197+#define HSR_EC_CP14_MR (0x05)
198+#define HSR_EC_CP14_LS (0x06)
199+#define HSR_EC_CP_0_13 (0x07)
200+#define HSR_EC_CP10_ID (0x08)
201+#define HSR_EC_JAZELLE (0x09)
202+#define HSR_EC_BXJ (0x0A)
203+#define HSR_EC_CP14_64 (0x0C)
204+#define HSR_EC_SVC_HYP (0x11)
205+#define HSR_EC_HVC (0x12)
206+#define HSR_EC_SMC (0x13)
207+#define HSR_EC_IABT (0x20)
208+#define HSR_EC_IABT_HYP (0x21)
209+#define HSR_EC_DABT (0x24)
210+#define HSR_EC_DABT_HYP (0x25)
211+
212+#define HSR_HVC_IMM_MASK ((1UL << 16) - 1)
213+
214+#endif /* __ARM_KVM_ARM_H__ */
arch/arm/include/asm/kvm_asm.h+82−0
@@ -0,0 +1,82 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_ASM_H__
20+#define __ARM_KVM_ASM_H__
21+
22+/* 0 is reserved as an invalid value. */
23+#define c0_MPIDR 1 /* MultiProcessor ID Register */
24+#define c0_CSSELR 2 /* Cache Size Selection Register */
25+#define c1_SCTLR 3 /* System Control Register */
26+#define c1_ACTLR 4 /* Auxilliary Control Register */
27+#define c1_CPACR 5 /* Coprocessor Access Control */
28+#define c2_TTBR0 6 /* Translation Table Base Register 0 */
29+#define c2_TTBR0_high 7 /* TTBR0 top 32 bits */
30+#define c2_TTBR1 8 /* Translation Table Base Register 1 */
31+#define c2_TTBR1_high 9 /* TTBR1 top 32 bits */
32+#define c2_TTBCR 10 /* Translation Table Base Control R. */
33+#define c3_DACR 11 /* Domain Access Control Register */
34+#define c5_DFSR 12 /* Data Fault Status Register */
35+#define c5_IFSR 13 /* Instruction Fault Status Register */
36+#define c5_ADFSR 14 /* Auxilary Data Fault Status R */
37+#define c5_AIFSR 15 /* Auxilary Instrunction Fault Status R */
38+#define c6_DFAR 16 /* Data Fault Address Register */
39+#define c6_IFAR 17 /* Instruction Fault Address Register */
40+#define c9_L2CTLR 18 /* Cortex A15 L2 Control Register */
41+#define c10_PRRR 19 /* Primary Region Remap Register */
42+#define c10_NMRR 20 /* Normal Memory Remap Register */
43+#define c12_VBAR 21 /* Vector Base Address Register */
44+#define c13_CID 22 /* Context ID Register */
45+#define c13_TID_URW 23 /* Thread ID, User R/W */
46+#define c13_TID_URO 24 /* Thread ID, User R/O */
47+#define c13_TID_PRIV 25 /* Thread ID, Privileged */
48+#define NR_CP15_REGS 26 /* Number of regs (incl. invalid) */
49+
50+#define ARM_EXCEPTION_RESET 0
51+#define ARM_EXCEPTION_UNDEFINED 1
52+#define ARM_EXCEPTION_SOFTWARE 2
53+#define ARM_EXCEPTION_PREF_ABORT 3
54+#define ARM_EXCEPTION_DATA_ABORT 4
55+#define ARM_EXCEPTION_IRQ 5
56+#define ARM_EXCEPTION_FIQ 6
57+#define ARM_EXCEPTION_HVC 7
58+
59+#ifndef __ASSEMBLY__
60+struct kvm;
61+struct kvm_vcpu;
62+
63+extern char __kvm_hyp_init[];
64+extern char __kvm_hyp_init_end[];
65+
66+extern char __kvm_hyp_exit[];
67+extern char __kvm_hyp_exit_end[];
68+
69+extern char __kvm_hyp_vector[];
70+
71+extern char __kvm_hyp_code_start[];
72+extern char __kvm_hyp_code_end[];
73+
74+extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
75+
76+extern void __kvm_flush_vm_context(void);
77+extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
78+
79+extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
80+#endif
81+
82+#endif /* __ARM_KVM_ASM_H__ */
arch/arm/include/asm/kvm_coproc.h+47−0
@@ -0,0 +1,47 @@
1+/*
2+ * Copyright (C) 2012 Rusty Russell IBM Corporation
3+ *
4+ * This program is free software; you can redistribute it and/or modify
5+ * it under the terms of the GNU General Public License, version 2, as
6+ * published by the Free Software Foundation.
7+ *
8+ * This program is distributed in the hope that it will be useful,
9+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
10+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11+ * GNU General Public License for more details.
12+ *
13+ * You should have received a copy of the GNU General Public License
14+ * along with this program; if not, write to the Free Software
15+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16+ */
17+
18+#ifndef __ARM_KVM_COPROC_H__
19+#define __ARM_KVM_COPROC_H__
20+#include <linux/kvm_host.h>
21+
22+void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
23+
24+struct kvm_coproc_target_table {
25+ unsigned target;
26+ const struct coproc_reg *table;
27+ size_t num;
28+};
29+void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table);
30+
31+int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run);
32+int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run);
33+int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run);
34+int kvm_handle_cp14_access(struct kvm_vcpu *vcpu, struct kvm_run *run);
35+int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run);
36+int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run);
37+
38+unsigned long kvm_arm_num_guest_msrs(struct kvm_vcpu *vcpu);
39+int kvm_arm_copy_msrindices(struct kvm_vcpu *vcpu, u64 __user *uindices);
40+void kvm_coproc_table_init(void);
41+
42+struct kvm_one_reg;
43+int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
44+int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
45+int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
46+unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu);
47+#endif /* __ARM_KVM_COPROC_H__ */
arch/arm/include/asm/kvm_emulate.h+72−0
@@ -0,0 +1,72 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_EMULATE_H__
20+#define __ARM_KVM_EMULATE_H__
21+
22+#include <linux/kvm_host.h>
23+#include <asm/kvm_asm.h>
24+#include <asm/kvm_mmio.h>
25+
26+u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num);
27+u32 *vcpu_spsr(struct kvm_vcpu *vcpu);
28+
29+int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run);
30+void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr);
31+void kvm_inject_undefined(struct kvm_vcpu *vcpu);
32+void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
33+void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
34+
35+static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu)
36+{
37+ return 1;
38+}
39+
40+static inline u32 *vcpu_pc(struct kvm_vcpu *vcpu)
41+{
42+ return (u32 *)&vcpu->arch.regs.usr_regs.ARM_pc;
43+}
44+
45+static inline u32 *vcpu_cpsr(struct kvm_vcpu *vcpu)
46+{
47+ return (u32 *)&vcpu->arch.regs.usr_regs.ARM_cpsr;
48+}
49+
50+static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
51+{
52+ *vcpu_cpsr(vcpu) |= PSR_T_BIT;
53+}
54+
55+static inline bool mode_has_spsr(struct kvm_vcpu *vcpu)
56+{
57+ unsigned long cpsr_mode = vcpu->arch.regs.usr_regs.ARM_cpsr & MODE_MASK;
58+ return (cpsr_mode > USR_MODE && cpsr_mode < SYSTEM_MODE);
59+}
60+
61+static inline bool vcpu_mode_priv(struct kvm_vcpu *vcpu)
62+{
63+ unsigned long cpsr_mode = vcpu->arch.regs.usr_regs.ARM_cpsr & MODE_MASK;
64+ return cpsr_mode > USR_MODE;;
65+}
66+
67+static inline bool kvm_vcpu_reg_is_pc(struct kvm_vcpu *vcpu, int reg)
68+{
69+ return reg == 15;
70+}
71+
72+#endif /* __ARM_KVM_EMULATE_H__ */
arch/arm/include/asm/kvm_host.h+161−0
@@ -0,0 +1,161 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_HOST_H__
20+#define __ARM_KVM_HOST_H__
21+
22+#include <asm/kvm.h>
23+#include <asm/kvm_asm.h>
24+#include <asm/kvm_mmio.h>
25+#include <asm/fpstate.h>
26+
27+#define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
28+#define KVM_MEMORY_SLOTS 32
29+#define KVM_PRIVATE_MEM_SLOTS 4
30+#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
31+#define KVM_HAVE_ONE_REG
32+
33+#define KVM_VCPU_MAX_FEATURES 1
34+
35+/* We don't currently support large pages. */
36+#define KVM_HPAGE_GFN_SHIFT(x) 0
37+#define KVM_NR_PAGE_SIZES 1
38+#define KVM_PAGES_PER_HPAGE(x) (1UL<<31)
39+
40+struct kvm_vcpu;
41+u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode);
42+int kvm_target_cpu(void);
43+int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
44+void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
45+
46+struct kvm_arch {
47+ /* VTTBR value associated with below pgd and vmid */
48+ u64 vttbr;
49+
50+ /*
51+ * Anything that is not used directly from assembly code goes
52+ * here.
53+ */
54+
55+ /* The VMID generation used for the virt. memory system */
56+ u64 vmid_gen;
57+ u32 vmid;
58+
59+ /* Stage-2 page table */
60+ pgd_t *pgd;
61+};
62+
63+#define KVM_NR_MEM_OBJS 40
64+
65+/*
66+ * We don't want allocation failures within the mmu code, so we preallocate
67+ * enough memory for a single page fault in a cache.
68+ */
69+struct kvm_mmu_memory_cache {
70+ int nobjs;
71+ void *objects[KVM_NR_MEM_OBJS];
72+};
73+
74+struct kvm_vcpu_arch {
75+ struct kvm_regs regs;
76+
77+ int target; /* Processor target */
78+ DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
79+
80+ /* System control coprocessor (cp15) */
81+ u32 cp15[NR_CP15_REGS];
82+
83+ /* The CPU type we expose to the VM */
84+ u32 midr;
85+
86+ /* Exception Information */
87+ u32 hsr; /* Hyp Syndrome Register */
88+ u32 hxfar; /* Hyp Data/Inst Fault Address Register */
89+ u32 hpfar; /* Hyp IPA Fault Address Register */
90+
91+ /* Floating point registers (VFP and Advanced SIMD/NEON) */
92+ struct vfp_hard_struct vfp_guest;
93+ struct vfp_hard_struct *vfp_host;
94+
95+ /*
96+ * Anything that is not used directly from assembly code goes
97+ * here.
98+ */
99+ /* dcache set/way operation pending */
100+ int last_pcpu;
101+ cpumask_t require_dcache_flush;
102+
103+ /* Don't run the guest on this vcpu */
104+ bool pause;
105+
106+ /* IO related fields */
107+ struct kvm_decode mmio_decode;
108+
109+ /* Interrupt related fields */
110+ u32 irq_lines; /* IRQ and FIQ levels */
111+
112+ /* Hyp exception information */
113+ u32 hyp_pc; /* PC when exception was taken from Hyp mode */
114+
115+ /* Cache some mmu pages needed inside spinlock regions */
116+ struct kvm_mmu_memory_cache mmu_page_cache;
117+
118+ /* Detect first run of a vcpu */
119+ bool has_run_once;
120+};
121+
122+struct kvm_vm_stat {
123+ u32 remote_tlb_flush;
124+};
125+
126+struct kvm_vcpu_stat {
127+ u32 halt_wakeup;
128+};
129+
130+struct kvm_vcpu_init;
131+int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
132+ const struct kvm_vcpu_init *init);
133+unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
134+int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
135+struct kvm_one_reg;
136+int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
137+int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
138+u64 kvm_call_hyp(void *hypfn, ...);
139+void force_vm_exit(const cpumask_t *mask);
140+
141+#define KVM_ARCH_WANT_MMU_NOTIFIER
142+struct kvm;
143+int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
144+int kvm_unmap_hva_range(struct kvm *kvm,
145+ unsigned long start, unsigned long end);
146+void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
147+
148+unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
149+int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
150+
151+/* We do not have shadow page tables, hence the empty hooks */
152+static inline int kvm_age_hva(struct kvm *kvm, unsigned long hva)
153+{
154+ return 0;
155+}
156+
157+static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
158+{
159+ return 0;
160+}
161+#endif /* __ARM_KVM_HOST_H__ */
arch/arm/include/asm/kvm_mmio.h+56−0
@@ -0,0 +1,56 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_MMIO_H__
20+#define __ARM_KVM_MMIO_H__
21+
22+#include <linux/kvm_host.h>
23+#include <asm/kvm_asm.h>
24+#include <asm/kvm_arm.h>
25+
26+struct kvm_decode {
27+ unsigned long rt;
28+ bool sign_extend;
29+};
30+
31+/*
32+ * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
33+ * which is an anonymous type. Use our own type instead.
34+ */
35+struct kvm_exit_mmio {
36+ phys_addr_t phys_addr;
37+ u8 data[8];
38+ u32 len;
39+ bool is_write;
40+};
41+
42+static inline void kvm_prepare_mmio(struct kvm_run *run,
43+ struct kvm_exit_mmio *mmio)
44+{
45+ run->mmio.phys_addr = mmio->phys_addr;
46+ run->mmio.len = mmio->len;
47+ run->mmio.is_write = mmio->is_write;
48+ memcpy(run->mmio.data, mmio->data, mmio->len);
49+ run->exit_reason = KVM_EXIT_MMIO;
50+}
51+
52+int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
53+int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
54+ phys_addr_t fault_ipa);
55+
56+#endif /* __ARM_KVM_MMIO_H__ */
arch/arm/include/asm/kvm_mmu.h+50−0
@@ -0,0 +1,50 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_MMU_H__
20+#define __ARM_KVM_MMU_H__
21+
22+int create_hyp_mappings(void *from, void *to);
23+int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
24+void free_hyp_pmds(void);
25+
26+int kvm_alloc_stage2_pgd(struct kvm *kvm);
27+void kvm_free_stage2_pgd(struct kvm *kvm);
28+int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
29+ phys_addr_t pa, unsigned long size);
30+
31+int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
32+
33+void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
34+
35+phys_addr_t kvm_mmu_get_httbr(void);
36+int kvm_mmu_init(void);
37+void kvm_clear_hyp_idmap(void);
38+
39+static inline bool kvm_is_write_fault(unsigned long hsr)
40+{
41+ unsigned long hsr_ec = hsr >> HSR_EC_SHIFT;
42+ if (hsr_ec == HSR_EC_IABT)
43+ return false;
44+ else if ((hsr & HSR_ISV) && !(hsr & HSR_WNR))
45+ return false;
46+ else
47+ return true;
48+}
49+
50+#endif /* __ARM_KVM_MMU_H__ */
arch/arm/include/asm/kvm_psci.h+23−0
@@ -0,0 +1,23 @@
1+/*
2+ * Copyright (C) 2012 - ARM Ltd
3+ * Author: Marc Zyngier <marc.zyngier@arm.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License version 2 as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
16+ */
17+
18+#ifndef __ARM_KVM_PSCI_H__
19+#define __ARM_KVM_PSCI_H__
20+
21+bool kvm_psci_call(struct kvm_vcpu *vcpu);
22+
23+#endif /* __ARM_KVM_PSCI_H__ */
arch/arm/include/asm/pgtable-3level-hwdef.h+5−0
@@ -32,6 +32,9 @@ · 31 unmodified lines
3232 #define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
3333 #define PMD_BIT4 (_AT(pmdval_t, 0))
3434 #define PMD_DOMAIN(x) (_AT(pmdval_t, 0))
35+#define PMD_APTABLE_SHIFT (61)
36+#define PMD_APTABLE (_AT(pgdval_t, 3) << PGD_APTABLE_SHIFT)
37+#define PMD_PXNTABLE (_AT(pgdval_t, 1) << 59)
3538
3639 /*
3740 * - section
@@ -41,9 +44,11 @@ · 3 unmodified lines
4144 #define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
4245 #define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
4346 #define PMD_SECT_nG (_AT(pmdval_t, 1) << 11)
47+#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 53)
4448 #define PMD_SECT_XN (_AT(pmdval_t, 1) << 54)
4549 #define PMD_SECT_AP_WRITE (_AT(pmdval_t, 0))
4650 #define PMD_SECT_AP_READ (_AT(pmdval_t, 0))
51+#define PMD_SECT_AP1 (_AT(pmdval_t, 1) << 6)
4752 #define PMD_SECT_TEX(x) (_AT(pmdval_t, 0))
4853
4954 /*
arch/arm/include/asm/pgtable-3level.h+18−0
@@ -104,11 +104,29 @@ · 103 unmodified lines
104104 */
105105 #define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */
106106
107+/*
108+ * 2nd stage PTE definitions for LPAE.
109+ */
110+#define L_PTE_S2_MT_UNCACHED (_AT(pteval_t, 0x5) << 2) /* MemAttr[3:0] */
111+#define L_PTE_S2_MT_WRITETHROUGH (_AT(pteval_t, 0xa) << 2) /* MemAttr[3:0] */
112+#define L_PTE_S2_MT_WRITEBACK (_AT(pteval_t, 0xf) << 2) /* MemAttr[3:0] */
113+#define L_PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[1] */
114+#define L_PTE_S2_RDWR (_AT(pteval_t, 2) << 6) /* HAP[2:1] */
115+
116+/*
117+ * Hyp-mode PL2 PTE definitions for LPAE.
118+ */
119+#define L_PTE_HYP L_PTE_USER
120+
107121 #ifndef __ASSEMBLY__
108122
109123 #define pud_none(pud) (!pud_val(pud))
110124 #define pud_bad(pud) (!(pud_val(pud) & 2))
111125 #define pud_present(pud) (pud_val(pud))
126+#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
127+ PMD_TYPE_TABLE)
128+#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
129+ PMD_TYPE_SECT)
112130
113131 #define pud_clear(pudp) \
114132 do { \
arch/arm/include/asm/pgtable.h+7−0
@@ -70,6 +70,9 @@ extern void __pgd_error(const char *file, int line, pgd_t); · 69 unmodified lines
7070
7171 extern pgprot_t pgprot_user;
7272 extern pgprot_t pgprot_kernel;
73+extern pgprot_t pgprot_hyp_device;
74+extern pgprot_t pgprot_s2;
75+extern pgprot_t pgprot_s2_device;
7376
7477 #define _MOD_PROT(p, b) __pgprot(pgprot_val(p) | (b))
7578
@@ -82,6 +85,10 @@ extern pgprot_t pgprot_kernel; · 6 unmodified lines
8285 #define PAGE_READONLY_EXEC _MOD_PROT(pgprot_user, L_PTE_USER | L_PTE_RDONLY)
8386 #define PAGE_KERNEL _MOD_PROT(pgprot_kernel, L_PTE_XN)
8487 #define PAGE_KERNEL_EXEC pgprot_kernel
88+#define PAGE_HYP _MOD_PROT(pgprot_kernel, L_PTE_HYP)
89+#define PAGE_HYP_DEVICE _MOD_PROT(pgprot_hyp_device, L_PTE_HYP)
90+#define PAGE_S2 _MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY)
91+#define PAGE_S2_DEVICE _MOD_PROT(pgprot_s2_device, L_PTE_USER | L_PTE_S2_RDONLY)
8592
8693 #define __PAGE_NONE __pgprot(_L_PTE_DEFAULT | L_PTE_RDONLY | L_PTE_XN | L_PTE_NONE)
8794 #define __PAGE_SHARED __pgprot(_L_PTE_DEFAULT | L_PTE_USER | L_PTE_XN)
arch/arm/include/uapi/asm/kvm.h+164−0
@@ -0,0 +1,164 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_H__
20+#define __ARM_KVM_H__
21+
22+#include <linux/types.h>
23+#include <asm/ptrace.h>
24+
25+#define __KVM_HAVE_GUEST_DEBUG
26+#define __KVM_HAVE_IRQ_LINE
27+
28+#define KVM_REG_SIZE(id) \
29+ (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
30+
31+/* Valid for svc_regs, abt_regs, und_regs, irq_regs in struct kvm_regs */
32+#define KVM_ARM_SVC_sp svc_regs[0]
33+#define KVM_ARM_SVC_lr svc_regs[1]
34+#define KVM_ARM_SVC_spsr svc_regs[2]
35+#define KVM_ARM_ABT_sp abt_regs[0]
36+#define KVM_ARM_ABT_lr abt_regs[1]
37+#define KVM_ARM_ABT_spsr abt_regs[2]
38+#define KVM_ARM_UND_sp und_regs[0]
39+#define KVM_ARM_UND_lr und_regs[1]
40+#define KVM_ARM_UND_spsr und_regs[2]
41+#define KVM_ARM_IRQ_sp irq_regs[0]
42+#define KVM_ARM_IRQ_lr irq_regs[1]
43+#define KVM_ARM_IRQ_spsr irq_regs[2]
44+
45+/* Valid only for fiq_regs in struct kvm_regs */
46+#define KVM_ARM_FIQ_r8 fiq_regs[0]
47+#define KVM_ARM_FIQ_r9 fiq_regs[1]
48+#define KVM_ARM_FIQ_r10 fiq_regs[2]
49+#define KVM_ARM_FIQ_fp fiq_regs[3]
50+#define KVM_ARM_FIQ_ip fiq_regs[4]
51+#define KVM_ARM_FIQ_sp fiq_regs[5]
52+#define KVM_ARM_FIQ_lr fiq_regs[6]
53+#define KVM_ARM_FIQ_spsr fiq_regs[7]
54+
55+struct kvm_regs {
56+ struct pt_regs usr_regs;/* R0_usr - R14_usr, PC, CPSR */
57+ __u32 svc_regs[3]; /* SP_svc, LR_svc, SPSR_svc */
58+ __u32 abt_regs[3]; /* SP_abt, LR_abt, SPSR_abt */
59+ __u32 und_regs[3]; /* SP_und, LR_und, SPSR_und */
60+ __u32 irq_regs[3]; /* SP_irq, LR_irq, SPSR_irq */
61+ __u32 fiq_regs[8]; /* R8_fiq - R14_fiq, SPSR_fiq */
62+};
63+
64+/* Supported Processor Types */
65+#define KVM_ARM_TARGET_CORTEX_A15 0
66+#define KVM_ARM_NUM_TARGETS 1
67+
68+#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */
69+
70+struct kvm_vcpu_init {
71+ __u32 target;
72+ __u32 features[7];
73+};
74+
75+struct kvm_sregs {
76+};
77+
78+struct kvm_fpu {
79+};
80+
81+struct kvm_guest_debug_arch {
82+};
83+
84+struct kvm_debug_exit_arch {
85+};
86+
87+struct kvm_sync_regs {
88+};
89+
90+struct kvm_arch_memory_slot {
91+};
92+
93+/* If you need to interpret the index values, here is the key: */
94+#define KVM_REG_ARM_COPROC_MASK 0x000000000FFF0000
95+#define KVM_REG_ARM_COPROC_SHIFT 16
96+#define KVM_REG_ARM_32_OPC2_MASK 0x0000000000000007
97+#define KVM_REG_ARM_32_OPC2_SHIFT 0
98+#define KVM_REG_ARM_OPC1_MASK 0x0000000000000078
99+#define KVM_REG_ARM_OPC1_SHIFT 3
100+#define KVM_REG_ARM_CRM_MASK 0x0000000000000780
101+#define KVM_REG_ARM_CRM_SHIFT 7
102+#define KVM_REG_ARM_32_CRN_MASK 0x0000000000007800
103+#define KVM_REG_ARM_32_CRN_SHIFT 11
104+
105+/* Normal registers are mapped as coprocessor 16. */
106+#define KVM_REG_ARM_CORE (0x0010 << KVM_REG_ARM_COPROC_SHIFT)
107+#define KVM_REG_ARM_CORE_REG(name) (offsetof(struct kvm_regs, name) / 4)
108+
109+/* Some registers need more space to represent values. */
110+#define KVM_REG_ARM_DEMUX (0x0011 << KVM_REG_ARM_COPROC_SHIFT)
111+#define KVM_REG_ARM_DEMUX_ID_MASK 0x000000000000FF00
112+#define KVM_REG_ARM_DEMUX_ID_SHIFT 8
113+#define KVM_REG_ARM_DEMUX_ID_CCSIDR (0x00 << KVM_REG_ARM_DEMUX_ID_SHIFT)
114+#define KVM_REG_ARM_DEMUX_VAL_MASK 0x00000000000000FF
115+#define KVM_REG_ARM_DEMUX_VAL_SHIFT 0
116+
117+/* VFP registers: we could overload CP10 like ARM does, but that's ugly. */
118+#define KVM_REG_ARM_VFP (0x0012 << KVM_REG_ARM_COPROC_SHIFT)
119+#define KVM_REG_ARM_VFP_MASK 0x000000000000FFFF
120+#define KVM_REG_ARM_VFP_BASE_REG 0x0
121+#define KVM_REG_ARM_VFP_FPSID 0x1000
122+#define KVM_REG_ARM_VFP_FPSCR 0x1001
123+#define KVM_REG_ARM_VFP_MVFR1 0x1006
124+#define KVM_REG_ARM_VFP_MVFR0 0x1007
125+#define KVM_REG_ARM_VFP_FPEXC 0x1008
126+#define KVM_REG_ARM_VFP_FPINST 0x1009
127+#define KVM_REG_ARM_VFP_FPINST2 0x100A
128+
129+
130+/* KVM_IRQ_LINE irq field index values */
131+#define KVM_ARM_IRQ_TYPE_SHIFT 24
132+#define KVM_ARM_IRQ_TYPE_MASK 0xff
133+#define KVM_ARM_IRQ_VCPU_SHIFT 16
134+#define KVM_ARM_IRQ_VCPU_MASK 0xff
135+#define KVM_ARM_IRQ_NUM_SHIFT 0
136+#define KVM_ARM_IRQ_NUM_MASK 0xffff
137+
138+/* irq_type field */
139+#define KVM_ARM_IRQ_TYPE_CPU 0
140+#define KVM_ARM_IRQ_TYPE_SPI 1
141+#define KVM_ARM_IRQ_TYPE_PPI 2
142+
143+/* out-of-kernel GIC cpu interrupt injection irq_number field */
144+#define KVM_ARM_IRQ_CPU_IRQ 0
145+#define KVM_ARM_IRQ_CPU_FIQ 1
146+
147+/* Highest supported SPI, from VGIC_NR_IRQS */
148+#define KVM_ARM_IRQ_GIC_MAX 127
149+
150+/* PSCI interface */
151+#define KVM_PSCI_FN_BASE 0x95c1ba5e
152+#define KVM_PSCI_FN(n) (KVM_PSCI_FN_BASE + (n))
153+
154+#define KVM_PSCI_FN_CPU_SUSPEND KVM_PSCI_FN(0)
155+#define KVM_PSCI_FN_CPU_OFF KVM_PSCI_FN(1)
156+#define KVM_PSCI_FN_CPU_ON KVM_PSCI_FN(2)
157+#define KVM_PSCI_FN_MIGRATE KVM_PSCI_FN(3)
158+
159+#define KVM_PSCI_RET_SUCCESS 0
160+#define KVM_PSCI_RET_NI ((unsigned long)-1)
161+#define KVM_PSCI_RET_INVAL ((unsigned long)-2)
162+#define KVM_PSCI_RET_DENIED ((unsigned long)-3)
163+
164+#endif /* __ARM_KVM_H__ */
arch/arm/kernel/asm-offsets.c+25−0
@@ -13,6 +13,9 @@ · 12 unmodified lines
1313 #include <linux/sched.h>
1414 #include <linux/mm.h>
1515 #include <linux/dma-mapping.h>
16+#ifdef CONFIG_KVM_ARM_HOST
17+#include <linux/kvm_host.h>
18+#endif
1619 #include <asm/cacheflush.h>
1720 #include <asm/glue-df.h>
1821 #include <asm/glue-pf.h>
@@ -146,5 +149,27 @@ int main(void) · 127 unmodified lines
146149 DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL);
147150 DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE);
148151 DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
152+#ifdef CONFIG_KVM_ARM_HOST
153+ DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm));
154+ DEFINE(VCPU_MIDR, offsetof(struct kvm_vcpu, arch.midr));
155+ DEFINE(VCPU_CP15, offsetof(struct kvm_vcpu, arch.cp15));
156+ DEFINE(VCPU_VFP_GUEST, offsetof(struct kvm_vcpu, arch.vfp_guest));
157+ DEFINE(VCPU_VFP_HOST, offsetof(struct kvm_vcpu, arch.vfp_host));
158+ DEFINE(VCPU_REGS, offsetof(struct kvm_vcpu, arch.regs));
159+ DEFINE(VCPU_USR_REGS, offsetof(struct kvm_vcpu, arch.regs.usr_regs));
160+ DEFINE(VCPU_SVC_REGS, offsetof(struct kvm_vcpu, arch.regs.svc_regs));
161+ DEFINE(VCPU_ABT_REGS, offsetof(struct kvm_vcpu, arch.regs.abt_regs));
162+ DEFINE(VCPU_UND_REGS, offsetof(struct kvm_vcpu, arch.regs.und_regs));
163+ DEFINE(VCPU_IRQ_REGS, offsetof(struct kvm_vcpu, arch.regs.irq_regs));
164+ DEFINE(VCPU_FIQ_REGS, offsetof(struct kvm_vcpu, arch.regs.fiq_regs));
165+ DEFINE(VCPU_PC, offsetof(struct kvm_vcpu, arch.regs.usr_regs.ARM_pc));
166+ DEFINE(VCPU_CPSR, offsetof(struct kvm_vcpu, arch.regs.usr_regs.ARM_cpsr));
167+ DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines));
168+ DEFINE(VCPU_HSR, offsetof(struct kvm_vcpu, arch.hsr));
169+ DEFINE(VCPU_HxFAR, offsetof(struct kvm_vcpu, arch.hxfar));
170+ DEFINE(VCPU_HPFAR, offsetof(struct kvm_vcpu, arch.hpfar));
171+ DEFINE(VCPU_HYP_PC, offsetof(struct kvm_vcpu, arch.hyp_pc));
172+ DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
173+#endif
149174 return 0;
150175 }
arch/arm/kernel/vmlinux.lds.S+5−1
@@ -19,7 +19,11 @@ · 18 unmodified lines
1919 ALIGN_FUNCTION(); \
2020 VMLINUX_SYMBOL(__idmap_text_start) = .; \
2121 *(.idmap.text) \
22− VMLINUX_SYMBOL(__idmap_text_end) = .;
22+ VMLINUX_SYMBOL(__idmap_text_end) = .; \
23+ ALIGN_FUNCTION(); \
24+ VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \
25+ *(.hyp.idmap.text) \
26+ VMLINUX_SYMBOL(__hyp_idmap_text_end) = .;
2327
2428 #ifdef CONFIG_HOTPLUG_CPU
2529 #define ARM_CPU_DISCARD(x)
arch/arm/kvm/Kconfig+56−0
@@ -0,0 +1,56 @@
1+#
2+# KVM configuration
3+#
4+
5+source "virt/kvm/Kconfig"
6+
7+menuconfig VIRTUALIZATION
8+ bool "Virtualization"
9+ ---help---
10+ Say Y here to get to see options for using your Linux host to run
11+ other operating systems inside virtual machines (guests).
12+ This option alone does not add any kernel code.
13+
14+ If you say N, all options in this submenu will be skipped and
15+ disabled.
16+
17+if VIRTUALIZATION
18+
19+config KVM
20+ bool "Kernel-based Virtual Machine (KVM) support"
21+ select PREEMPT_NOTIFIERS
22+ select ANON_INODES
23+ select KVM_MMIO
24+ select KVM_ARM_HOST
25+ depends on ARM_VIRT_EXT && ARM_LPAE
26+ ---help---
27+ Support hosting virtualized guest machines. You will also
28+ need to select one or more of the processor modules below.
29+
30+ This module provides access to the hardware capabilities through
31+ a character device node named /dev/kvm.
32+
33+ If unsure, say N.
34+
35+config KVM_ARM_HOST
36+ bool "KVM host support for ARM cpus."
37+ depends on KVM
38+ depends on MMU
39+ select MMU_NOTIFIER
40+ ---help---
41+ Provides host support for ARM processors.
42+
43+config KVM_ARM_MAX_VCPUS
44+ int "Number maximum supported virtual CPUs per VM"
45+ depends on KVM_ARM_HOST
46+ default 4
47+ help
48+ Static number of max supported virtual CPUs per VM.
49+
50+ If you choose a high number, the vcpu structures will be quite
51+ large, so only choose a reasonable number that you expect to
52+ actually use.
53+
54+source drivers/virtio/Kconfig
55+
56+endif # VIRTUALIZATION
arch/arm/kvm/Makefile+21−0
@@ -0,0 +1,21 @@
1+#
2+# Makefile for Kernel-based Virtual Machine module
3+#
4+
5+plus_virt := $(call as-instr,.arch_extension virt,+virt)
6+ifeq ($(plus_virt),+virt)
7+ plus_virt_def := -DREQUIRES_VIRT=1
8+endif
9+
10+ccflags-y += -Ivirt/kvm -Iarch/arm/kvm
11+CFLAGS_arm.o := -I. $(plus_virt_def)
12+CFLAGS_mmu.o := -I.
13+
14+AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
15+AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
16+
17+kvm-arm-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
18+
19+obj-y += kvm-arm.o init.o interrupts.o
20+obj-y += arm.o guest.o mmu.o emulate.o reset.o
21+obj-y += coproc.o coproc_a15.o mmio.o psci.o
arch/arm/kvm/arm.c+1015−0
@@ -0,0 +1,1015 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/errno.h>
20+#include <linux/err.h>
21+#include <linux/kvm_host.h>
22+#include <linux/module.h>
23+#include <linux/vmalloc.h>
24+#include <linux/fs.h>
25+#include <linux/mman.h>
26+#include <linux/sched.h>
27+#include <linux/kvm.h>
28+#include <trace/events/kvm.h>
29+
30+#define CREATE_TRACE_POINTS
31+#include "trace.h"
32+
33+#include <asm/unified.h>
34+#include <asm/uaccess.h>
35+#include <asm/ptrace.h>
36+#include <asm/mman.h>
37+#include <asm/cputype.h>
38+#include <asm/tlbflush.h>
39+#include <asm/cacheflush.h>
40+#include <asm/virt.h>
41+#include <asm/kvm_arm.h>
42+#include <asm/kvm_asm.h>
43+#include <asm/kvm_mmu.h>
44+#include <asm/kvm_emulate.h>
45+#include <asm/kvm_coproc.h>
46+#include <asm/kvm_psci.h>
47+#include <asm/opcodes.h>
48+
49+#ifdef REQUIRES_VIRT
50+__asm__(".arch_extension virt");
51+#endif
52+
53+static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
54+static struct vfp_hard_struct __percpu *kvm_host_vfp_state;
55+static unsigned long hyp_default_vectors;
56+
57+/* The VMID used in the VTTBR */
58+static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
59+static u8 kvm_next_vmid;
60+static DEFINE_SPINLOCK(kvm_vmid_lock);
61+
62+int kvm_arch_hardware_enable(void *garbage)
63+{
64+ return 0;
65+}
66+
67+int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
68+{
69+ return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
70+}
71+
72+void kvm_arch_hardware_disable(void *garbage)
73+{
74+}
75+
76+int kvm_arch_hardware_setup(void)
77+{
78+ return 0;
79+}
80+
81+void kvm_arch_hardware_unsetup(void)
82+{
83+}
84+
85+void kvm_arch_check_processor_compat(void *rtn)
86+{
87+ *(int *)rtn = 0;
88+}
89+
90+void kvm_arch_sync_events(struct kvm *kvm)
91+{
92+}
93+
94+/**
95+ * kvm_arch_init_vm - initializes a VM data structure
96+ * @kvm: pointer to the KVM struct
97+ */
98+int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
99+{
100+ int ret = 0;
101+
102+ if (type)
103+ return -EINVAL;
104+
105+ ret = kvm_alloc_stage2_pgd(kvm);
106+ if (ret)
107+ goto out_fail_alloc;
108+
109+ ret = create_hyp_mappings(kvm, kvm + 1);
110+ if (ret)
111+ goto out_free_stage2_pgd;
112+
113+ /* Mark the initial VMID generation invalid */
114+ kvm->arch.vmid_gen = 0;
115+
116+ return ret;
117+out_free_stage2_pgd:
118+ kvm_free_stage2_pgd(kvm);
119+out_fail_alloc:
120+ return ret;
121+}
122+
123+int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
124+{
125+ return VM_FAULT_SIGBUS;
126+}
127+
128+void kvm_arch_free_memslot(struct kvm_memory_slot *free,
129+ struct kvm_memory_slot *dont)
130+{
131+}
132+
133+int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
134+{
135+ return 0;
136+}
137+
138+/**
139+ * kvm_arch_destroy_vm - destroy the VM data structure
140+ * @kvm: pointer to the KVM struct
141+ */
142+void kvm_arch_destroy_vm(struct kvm *kvm)
143+{
144+ int i;
145+
146+ kvm_free_stage2_pgd(kvm);
147+
148+ for (i = 0; i < KVM_MAX_VCPUS; ++i) {
149+ if (kvm->vcpus[i]) {
150+ kvm_arch_vcpu_free(kvm->vcpus[i]);
151+ kvm->vcpus[i] = NULL;
152+ }
153+ }
154+}
155+
156+int kvm_dev_ioctl_check_extension(long ext)
157+{
158+ int r;
159+ switch (ext) {
160+ case KVM_CAP_USER_MEMORY:
161+ case KVM_CAP_SYNC_MMU:
162+ case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
163+ case KVM_CAP_ONE_REG:
164+ case KVM_CAP_ARM_PSCI:
165+ r = 1;
166+ break;
167+ case KVM_CAP_COALESCED_MMIO:
168+ r = KVM_COALESCED_MMIO_PAGE_OFFSET;
169+ break;
170+ case KVM_CAP_NR_VCPUS:
171+ r = num_online_cpus();
172+ break;
173+ case KVM_CAP_MAX_VCPUS:
174+ r = KVM_MAX_VCPUS;
175+ break;
176+ default:
177+ r = 0;
178+ break;
179+ }
180+ return r;
181+}
182+
183+long kvm_arch_dev_ioctl(struct file *filp,
184+ unsigned int ioctl, unsigned long arg)
185+{
186+ return -EINVAL;
187+}
188+
189+int kvm_arch_set_memory_region(struct kvm *kvm,
190+ struct kvm_userspace_memory_region *mem,
191+ struct kvm_memory_slot old,
192+ int user_alloc)
193+{
194+ return 0;
195+}
196+
197+int kvm_arch_prepare_memory_region(struct kvm *kvm,
198+ struct kvm_memory_slot *memslot,
199+ struct kvm_memory_slot old,
200+ struct kvm_userspace_memory_region *mem,
201+ int user_alloc)
202+{
203+ return 0;
204+}
205+
206+void kvm_arch_commit_memory_region(struct kvm *kvm,
207+ struct kvm_userspace_memory_region *mem,
208+ struct kvm_memory_slot old,
209+ int user_alloc)
210+{
211+}
212+
213+void kvm_arch_flush_shadow_all(struct kvm *kvm)
214+{
215+}
216+
217+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
218+ struct kvm_memory_slot *slot)
219+{
220+}
221+
222+struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
223+{
224+ int err;
225+ struct kvm_vcpu *vcpu;
226+
227+ vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
228+ if (!vcpu) {
229+ err = -ENOMEM;
230+ goto out;
231+ }
232+
233+ err = kvm_vcpu_init(vcpu, kvm, id);
234+ if (err)
235+ goto free_vcpu;
236+
237+ err = create_hyp_mappings(vcpu, vcpu + 1);
238+ if (err)
239+ goto vcpu_uninit;
240+
241+ return vcpu;
242+vcpu_uninit:
243+ kvm_vcpu_uninit(vcpu);
244+free_vcpu:
245+ kmem_cache_free(kvm_vcpu_cache, vcpu);
246+out:
247+ return ERR_PTR(err);
248+}
249+
250+int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
251+{
252+ return 0;
253+}
254+
255+void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
256+{
257+ kvm_mmu_free_memory_caches(vcpu);
258+ kmem_cache_free(kvm_vcpu_cache, vcpu);
259+}
260+
261+void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
262+{
263+ kvm_arch_vcpu_free(vcpu);
264+}
265+
266+int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
267+{
268+ return 0;
269+}
270+
271+int __attribute_const__ kvm_target_cpu(void)
272+{
273+ unsigned long implementor = read_cpuid_implementor();
274+ unsigned long part_number = read_cpuid_part_number();
275+
276+ if (implementor != ARM_CPU_IMP_ARM)
277+ return -EINVAL;
278+
279+ switch (part_number) {
280+ case ARM_CPU_PART_CORTEX_A15:
281+ return KVM_ARM_TARGET_CORTEX_A15;
282+ default:
283+ return -EINVAL;
284+ }
285+}
286+
287+int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
288+{
289+ /* Force users to call KVM_ARM_VCPU_INIT */
290+ vcpu->arch.target = -1;
291+ return 0;
292+}
293+
294+void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
295+{
296+}
297+
298+void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
299+{
300+ vcpu->cpu = cpu;
301+ vcpu->arch.vfp_host = this_cpu_ptr(kvm_host_vfp_state);
302+
303+ /*
304+ * Check whether this vcpu requires the cache to be flushed on
305+ * this physical CPU. This is a consequence of doing dcache
306+ * operations by set/way on this vcpu. We do it here to be in
307+ * a non-preemptible section.
308+ */
309+ if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
310+ flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
311+}
312+
313+void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
314+{
315+}
316+
317+int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
318+ struct kvm_guest_debug *dbg)
319+{
320+ return -EINVAL;
321+}
322+
323+
324+int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
325+ struct kvm_mp_state *mp_state)
326+{
327+ return -EINVAL;
328+}
329+
330+int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
331+ struct kvm_mp_state *mp_state)
332+{
333+ return -EINVAL;
334+}
335+
336+/**
337+ * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
338+ * @v: The VCPU pointer
339+ *
340+ * If the guest CPU is not waiting for interrupts or an interrupt line is
341+ * asserted, the CPU is by definition runnable.
342+ */
343+int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
344+{
345+ return !!v->arch.irq_lines;
346+}
347+
348+/* Just ensure a guest exit from a particular CPU */
349+static void exit_vm_noop(void *info)
350+{
351+}
352+
353+void force_vm_exit(const cpumask_t *mask)
354+{
355+ smp_call_function_many(mask, exit_vm_noop, NULL, true);
356+}
357+
358+/**
359+ * need_new_vmid_gen - check that the VMID is still valid
360+ * @kvm: The VM's VMID to checkt
361+ *
362+ * return true if there is a new generation of VMIDs being used
363+ *
364+ * The hardware supports only 256 values with the value zero reserved for the
365+ * host, so we check if an assigned value belongs to a previous generation,
366+ * which which requires us to assign a new value. If we're the first to use a
367+ * VMID for the new generation, we must flush necessary caches and TLBs on all
368+ * CPUs.
369+ */
370+static bool need_new_vmid_gen(struct kvm *kvm)
371+{
372+ return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
373+}
374+
375+/**
376+ * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
377+ * @kvm The guest that we are about to run
378+ *
379+ * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
380+ * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
381+ * caches and TLBs.
382+ */
383+static void update_vttbr(struct kvm *kvm)
384+{
385+ phys_addr_t pgd_phys;
386+ u64 vmid;
387+
388+ if (!need_new_vmid_gen(kvm))
389+ return;
390+
391+ spin_lock(&kvm_vmid_lock);
392+
393+ /*
394+ * We need to re-check the vmid_gen here to ensure that if another vcpu
395+ * already allocated a valid vmid for this vm, then this vcpu should
396+ * use the same vmid.
397+ */
398+ if (!need_new_vmid_gen(kvm)) {
399+ spin_unlock(&kvm_vmid_lock);
400+ return;
401+ }
402+
403+ /* First user of a new VMID generation? */
404+ if (unlikely(kvm_next_vmid == 0)) {
405+ atomic64_inc(&kvm_vmid_gen);
406+ kvm_next_vmid = 1;
407+
408+ /*
409+ * On SMP we know no other CPUs can use this CPU's or each
410+ * other's VMID after force_vm_exit returns since the
411+ * kvm_vmid_lock blocks them from reentry to the guest.
412+ */
413+ force_vm_exit(cpu_all_mask);
414+ /*
415+ * Now broadcast TLB + ICACHE invalidation over the inner
416+ * shareable domain to make sure all data structures are
417+ * clean.
418+ */
419+ kvm_call_hyp(__kvm_flush_vm_context);
420+ }
421+
422+ kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
423+ kvm->arch.vmid = kvm_next_vmid;
424+ kvm_next_vmid++;
425+
426+ /* update vttbr to be used with the new vmid */
427+ pgd_phys = virt_to_phys(kvm->arch.pgd);
428+ vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
429+ kvm->arch.vttbr = pgd_phys & VTTBR_BADDR_MASK;
430+ kvm->arch.vttbr |= vmid;
431+
432+ spin_unlock(&kvm_vmid_lock);
433+}
434+
435+static int handle_svc_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
436+{
437+ /* SVC called from Hyp mode should never get here */
438+ kvm_debug("SVC called from Hyp mode shouldn't go here\n");
439+ BUG();
440+ return -EINVAL; /* Squash warning */
441+}
442+
443+static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
444+{
445+ trace_kvm_hvc(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0),
446+ vcpu->arch.hsr & HSR_HVC_IMM_MASK);
447+
448+ if (kvm_psci_call(vcpu))
449+ return 1;
450+
451+ kvm_inject_undefined(vcpu);
452+ return 1;
453+}
454+
455+static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
456+{
457+ if (kvm_psci_call(vcpu))
458+ return 1;
459+
460+ kvm_inject_undefined(vcpu);
461+ return 1;
462+}
463+
464+static int handle_pabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
465+{
466+ /* The hypervisor should never cause aborts */
467+ kvm_err("Prefetch Abort taken from Hyp mode at %#08x (HSR: %#08x)\n",
468+ vcpu->arch.hxfar, vcpu->arch.hsr);
469+ return -EFAULT;
470+}
471+
472+static int handle_dabt_hyp(struct kvm_vcpu *vcpu, struct kvm_run *run)
473+{
474+ /* This is either an error in the ws. code or an external abort */
475+ kvm_err("Data Abort taken from Hyp mode at %#08x (HSR: %#08x)\n",
476+ vcpu->arch.hxfar, vcpu->arch.hsr);
477+ return -EFAULT;
478+}
479+
480+typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *);
481+static exit_handle_fn arm_exit_handlers[] = {
482+ [HSR_EC_WFI] = kvm_handle_wfi,
483+ [HSR_EC_CP15_32] = kvm_handle_cp15_32,
484+ [HSR_EC_CP15_64] = kvm_handle_cp15_64,
485+ [HSR_EC_CP14_MR] = kvm_handle_cp14_access,
486+ [HSR_EC_CP14_LS] = kvm_handle_cp14_load_store,
487+ [HSR_EC_CP14_64] = kvm_handle_cp14_access,
488+ [HSR_EC_CP_0_13] = kvm_handle_cp_0_13_access,
489+ [HSR_EC_CP10_ID] = kvm_handle_cp10_id,
490+ [HSR_EC_SVC_HYP] = handle_svc_hyp,
491+ [HSR_EC_HVC] = handle_hvc,
492+ [HSR_EC_SMC] = handle_smc,
493+ [HSR_EC_IABT] = kvm_handle_guest_abort,
494+ [HSR_EC_IABT_HYP] = handle_pabt_hyp,
495+ [HSR_EC_DABT] = kvm_handle_guest_abort,
496+ [HSR_EC_DABT_HYP] = handle_dabt_hyp,
497+};
498+
499+/*
500+ * A conditional instruction is allowed to trap, even though it
501+ * wouldn't be executed. So let's re-implement the hardware, in
502+ * software!
503+ */
504+static bool kvm_condition_valid(struct kvm_vcpu *vcpu)
505+{
506+ unsigned long cpsr, cond, insn;
507+
508+ /*
509+ * Exception Code 0 can only happen if we set HCR.TGE to 1, to
510+ * catch undefined instructions, and then we won't get past
511+ * the arm_exit_handlers test anyway.
512+ */
513+ BUG_ON(((vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT) == 0);
514+
515+ /* Top two bits non-zero? Unconditional. */
516+ if (vcpu->arch.hsr >> 30)
517+ return true;
518+
519+ cpsr = *vcpu_cpsr(vcpu);
520+
521+ /* Is condition field valid? */
522+ if ((vcpu->arch.hsr & HSR_CV) >> HSR_CV_SHIFT)
523+ cond = (vcpu->arch.hsr & HSR_COND) >> HSR_COND_SHIFT;
524+ else {
525+ /* This can happen in Thumb mode: examine IT state. */
526+ unsigned long it;
527+
528+ it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3);
529+
530+ /* it == 0 => unconditional. */
531+ if (it == 0)
532+ return true;
533+
534+ /* The cond for this insn works out as the top 4 bits. */
535+ cond = (it >> 4);
536+ }
537+
538+ /* Shift makes it look like an ARM-mode instruction */
539+ insn = cond << 28;
540+ return arm_check_condition(insn, cpsr) != ARM_OPCODE_CONDTEST_FAIL;
541+}
542+
543+/*
544+ * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on
545+ * proper exit to QEMU.
546+ */
547+static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
548+ int exception_index)
549+{
550+ unsigned long hsr_ec;
551+
552+ switch (exception_index) {
553+ case ARM_EXCEPTION_IRQ:
554+ return 1;
555+ case ARM_EXCEPTION_UNDEFINED:
556+ kvm_err("Undefined exception in Hyp mode at: %#08x\n",
557+ vcpu->arch.hyp_pc);
558+ BUG();
559+ panic("KVM: Hypervisor undefined exception!\n");
560+ case ARM_EXCEPTION_DATA_ABORT:
561+ case ARM_EXCEPTION_PREF_ABORT:
562+ case ARM_EXCEPTION_HVC:
563+ hsr_ec = (vcpu->arch.hsr & HSR_EC) >> HSR_EC_SHIFT;
564+
565+ if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers)
566+ || !arm_exit_handlers[hsr_ec]) {
567+ kvm_err("Unkown exception class: %#08lx, "
568+ "hsr: %#08x\n", hsr_ec,
569+ (unsigned int)vcpu->arch.hsr);
570+ BUG();
571+ }
572+
573+ /*
574+ * See ARM ARM B1.14.1: "Hyp traps on instructions
575+ * that fail their condition code check"
576+ */
577+ if (!kvm_condition_valid(vcpu)) {
578+ bool is_wide = vcpu->arch.hsr & HSR_IL;
579+ kvm_skip_instr(vcpu, is_wide);
580+ return 1;
581+ }
582+
583+ return arm_exit_handlers[hsr_ec](vcpu, run);
584+ default:
585+ kvm_pr_unimpl("Unsupported exception type: %d",
586+ exception_index);
587+ run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
588+ return 0;
589+ }
590+}
591+
592+static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
593+{
594+ if (likely(vcpu->arch.has_run_once))
595+ return 0;
596+
597+ vcpu->arch.has_run_once = true;
598+
599+ /*
600+ * Handle the "start in power-off" case by calling into the
601+ * PSCI code.
602+ */
603+ if (test_and_clear_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) {
604+ *vcpu_reg(vcpu, 0) = KVM_PSCI_FN_CPU_OFF;
605+ kvm_psci_call(vcpu);
606+ }
607+
608+ return 0;
609+}
610+
611+static void vcpu_pause(struct kvm_vcpu *vcpu)
612+{
613+ wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
614+
615+ wait_event_interruptible(*wq, !vcpu->arch.pause);
616+}
617+
618+/**
619+ * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
620+ * @vcpu: The VCPU pointer
621+ * @run: The kvm_run structure pointer used for userspace state exchange
622+ *
623+ * This function is called through the VCPU_RUN ioctl called from user space. It
624+ * will execute VM code in a loop until the time slice for the process is used
625+ * or some emulation is needed from user space in which case the function will
626+ * return with return value 0 and with the kvm_run structure filled in with the
627+ * required data for the requested emulation.
628+ */
629+int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
630+{
631+ int ret;
632+ sigset_t sigsaved;
633+
634+ /* Make sure they initialize the vcpu with KVM_ARM_VCPU_INIT */
635+ if (unlikely(vcpu->arch.target < 0))
636+ return -ENOEXEC;
637+
638+ ret = kvm_vcpu_first_run_init(vcpu);
639+ if (ret)
640+ return ret;
641+
642+ if (run->exit_reason == KVM_EXIT_MMIO) {
643+ ret = kvm_handle_mmio_return(vcpu, vcpu->run);
644+ if (ret)
645+ return ret;
646+ }
647+
648+ if (vcpu->sigset_active)
649+ sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
650+
651+ ret = 1;
652+ run->exit_reason = KVM_EXIT_UNKNOWN;
653+ while (ret > 0) {
654+ /*
655+ * Check conditions before entering the guest
656+ */
657+ cond_resched();
658+
659+ update_vttbr(vcpu->kvm);
660+
661+ if (vcpu->arch.pause)
662+ vcpu_pause(vcpu);
663+
664+ local_irq_disable();
665+
666+ /*
667+ * Re-check atomic conditions
668+ */
669+ if (signal_pending(current)) {
670+ ret = -EINTR;
671+ run->exit_reason = KVM_EXIT_INTR;
672+ }
673+
674+ if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
675+ local_irq_enable();
676+ continue;
677+ }
678+
679+ /**************************************************************
680+ * Enter the guest
681+ */
682+ trace_kvm_entry(*vcpu_pc(vcpu));
683+ kvm_guest_enter();
684+ vcpu->mode = IN_GUEST_MODE;
685+
686+ ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
687+
688+ vcpu->mode = OUTSIDE_GUEST_MODE;
689+ vcpu->arch.last_pcpu = smp_processor_id();
690+ kvm_guest_exit();
691+ trace_kvm_exit(*vcpu_pc(vcpu));
692+ /*
693+ * We may have taken a host interrupt in HYP mode (ie
694+ * while executing the guest). This interrupt is still
695+ * pending, as we haven't serviced it yet!
696+ *
697+ * We're now back in SVC mode, with interrupts
698+ * disabled. Enabling the interrupts now will have
699+ * the effect of taking the interrupt again, in SVC
700+ * mode this time.
701+ */
702+ local_irq_enable();
703+
704+ /*
705+ * Back from guest
706+ *************************************************************/
707+
708+ ret = handle_exit(vcpu, run, ret);
709+ }
710+
711+ if (vcpu->sigset_active)
712+ sigprocmask(SIG_SETMASK, &sigsaved, NULL);
713+ return ret;
714+}
715+
716+static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
717+{
718+ int bit_index;
719+ bool set;
720+ unsigned long *ptr;
721+
722+ if (number == KVM_ARM_IRQ_CPU_IRQ)
723+ bit_index = __ffs(HCR_VI);
724+ else /* KVM_ARM_IRQ_CPU_FIQ */
725+ bit_index = __ffs(HCR_VF);
726+
727+ ptr = (unsigned long *)&vcpu->arch.irq_lines;
728+ if (level)
729+ set = test_and_set_bit(bit_index, ptr);
730+ else
731+ set = test_and_clear_bit(bit_index, ptr);
732+
733+ /*
734+ * If we didn't change anything, no need to wake up or kick other CPUs
735+ */
736+ if (set == level)
737+ return 0;
738+
739+ /*
740+ * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
741+ * trigger a world-switch round on the running physical CPU to set the
742+ * virtual IRQ/FIQ fields in the HCR appropriately.
743+ */
744+ kvm_vcpu_kick(vcpu);
745+
746+ return 0;
747+}
748+
749+int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level)
750+{
751+ u32 irq = irq_level->irq;
752+ unsigned int irq_type, vcpu_idx, irq_num;
753+ int nrcpus = atomic_read(&kvm->online_vcpus);
754+ struct kvm_vcpu *vcpu = NULL;
755+ bool level = irq_level->level;
756+
757+ irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
758+ vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
759+ irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
760+
761+ trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
762+
763+ if (irq_type != KVM_ARM_IRQ_TYPE_CPU)
764+ return -EINVAL;
765+
766+ if (vcpu_idx >= nrcpus)
767+ return -EINVAL;
768+
769+ vcpu = kvm_get_vcpu(kvm, vcpu_idx);
770+ if (!vcpu)
771+ return -EINVAL;
772+
773+ if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
774+ return -EINVAL;
775+
776+ return vcpu_interrupt_line(vcpu, irq_num, level);
777+}
778+
779+long kvm_arch_vcpu_ioctl(struct file *filp,
780+ unsigned int ioctl, unsigned long arg)
781+{
782+ struct kvm_vcpu *vcpu = filp->private_data;
783+ void __user *argp = (void __user *)arg;
784+
785+ switch (ioctl) {
786+ case KVM_ARM_VCPU_INIT: {
787+ struct kvm_vcpu_init init;
788+
789+ if (copy_from_user(&init, argp, sizeof(init)))
790+ return -EFAULT;
791+
792+ return kvm_vcpu_set_target(vcpu, &init);
793+
794+ }
795+ case KVM_SET_ONE_REG:
796+ case KVM_GET_ONE_REG: {
797+ struct kvm_one_reg reg;
798+ if (copy_from_user(&reg, argp, sizeof(reg)))
799+ return -EFAULT;
800+ if (ioctl == KVM_SET_ONE_REG)
801+ return kvm_arm_set_reg(vcpu, &reg);
802+ else
803+ return kvm_arm_get_reg(vcpu, &reg);
804+ }
805+ case KVM_GET_REG_LIST: {
806+ struct kvm_reg_list __user *user_list = argp;
807+ struct kvm_reg_list reg_list;
808+ unsigned n;
809+
810+ if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
811+ return -EFAULT;
812+ n = reg_list.n;
813+ reg_list.n = kvm_arm_num_regs(vcpu);
814+ if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
815+ return -EFAULT;
816+ if (n < reg_list.n)
817+ return -E2BIG;
818+ return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
819+ }
820+ default:
821+ return -EINVAL;
822+ }
823+}
824+
825+int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
826+{
827+ return -EINVAL;
828+}
829+
830+long kvm_arch_vm_ioctl(struct file *filp,
831+ unsigned int ioctl, unsigned long arg)
832+{
833+ return -EINVAL;
834+}
835+
836+static void cpu_init_hyp_mode(void *vector)
837+{
838+ unsigned long long pgd_ptr;
839+ unsigned long pgd_low, pgd_high;
840+ unsigned long hyp_stack_ptr;
841+ unsigned long stack_page;
842+ unsigned long vector_ptr;
843+
844+ /* Switch from the HYP stub to our own HYP init vector */
845+ __hyp_set_vectors((unsigned long)vector);
846+
847+ pgd_ptr = (unsigned long long)kvm_mmu_get_httbr();
848+ pgd_low = (pgd_ptr & ((1ULL << 32) - 1));
849+ pgd_high = (pgd_ptr >> 32ULL);
850+ stack_page = __get_cpu_var(kvm_arm_hyp_stack_page);
851+ hyp_stack_ptr = stack_page + PAGE_SIZE;
852+ vector_ptr = (unsigned long)__kvm_hyp_vector;
853+
854+ /*
855+ * Call initialization code, and switch to the full blown
856+ * HYP code. The init code doesn't need to preserve these registers as
857+ * r1-r3 and r12 are already callee save according to the AAPCS.
858+ * Note that we slightly misuse the prototype by casing the pgd_low to
859+ * a void *.
860+ */
861+ kvm_call_hyp((void *)pgd_low, pgd_high, hyp_stack_ptr, vector_ptr);
862+}
863+
864+/**
865+ * Inits Hyp-mode on all online CPUs
866+ */
867+static int init_hyp_mode(void)
868+{
869+ phys_addr_t init_phys_addr;
870+ int cpu;
871+ int err = 0;
872+
873+ /*
874+ * Allocate Hyp PGD and setup Hyp identity mapping
875+ */
876+ err = kvm_mmu_init();
877+ if (err)
878+ goto out_err;
879+
880+ /*
881+ * It is probably enough to obtain the default on one
882+ * CPU. It's unlikely to be different on the others.
883+ */
884+ hyp_default_vectors = __hyp_get_vectors();
885+
886+ /*
887+ * Allocate stack pages for Hypervisor-mode
888+ */
889+ for_each_possible_cpu(cpu) {
890+ unsigned long stack_page;
891+
892+ stack_page = __get_free_page(GFP_KERNEL);
893+ if (!stack_page) {
894+ err = -ENOMEM;
895+ goto out_free_stack_pages;
896+ }
897+
898+ per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
899+ }
900+
901+ /*
902+ * Execute the init code on each CPU.
903+ *
904+ * Note: The stack is not mapped yet, so don't do anything else than
905+ * initializing the hypervisor mode on each CPU using a local stack
906+ * space for temporary storage.
907+ */
908+ init_phys_addr = virt_to_phys(__kvm_hyp_init);
909+ for_each_online_cpu(cpu) {
910+ smp_call_function_single(cpu, cpu_init_hyp_mode,
911+ (void *)(long)init_phys_addr, 1);
912+ }
913+
914+ /*
915+ * Unmap the identity mapping
916+ */
917+ kvm_clear_hyp_idmap();
918+
919+ /*
920+ * Map the Hyp-code called directly from the host
921+ */
922+ err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
923+ if (err) {
924+ kvm_err("Cannot map world-switch code\n");
925+ goto out_free_mappings;
926+ }
927+
928+ /*
929+ * Map the Hyp stack pages
930+ */
931+ for_each_possible_cpu(cpu) {
932+ char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
933+ err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
934+
935+ if (err) {
936+ kvm_err("Cannot map hyp stack\n");
937+ goto out_free_mappings;
938+ }
939+ }
940+
941+ /*
942+ * Map the host VFP structures
943+ */
944+ kvm_host_vfp_state = alloc_percpu(struct vfp_hard_struct);
945+ if (!kvm_host_vfp_state) {
946+ err = -ENOMEM;
947+ kvm_err("Cannot allocate host VFP state\n");
948+ goto out_free_mappings;
949+ }
950+
951+ for_each_possible_cpu(cpu) {
952+ struct vfp_hard_struct *vfp;
953+
954+ vfp = per_cpu_ptr(kvm_host_vfp_state, cpu);
955+ err = create_hyp_mappings(vfp, vfp + 1);
956+
957+ if (err) {
958+ kvm_err("Cannot map host VFP state: %d\n", err);
959+ goto out_free_vfp;
960+ }
961+ }
962+
963+ kvm_info("Hyp mode initialized successfully\n");
964+ return 0;
965+out_free_vfp:
966+ free_percpu(kvm_host_vfp_state);
967+out_free_mappings:
968+ free_hyp_pmds();
969+out_free_stack_pages:
970+ for_each_possible_cpu(cpu)
971+ free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
972+out_err:
973+ kvm_err("error initializing Hyp mode: %d\n", err);
974+ return err;
975+}
976+
977+/**
978+ * Initialize Hyp-mode and memory mappings on all CPUs.
979+ */
980+int kvm_arch_init(void *opaque)
981+{
982+ int err;
983+
984+ if (!is_hyp_mode_available()) {
985+ kvm_err("HYP mode not available\n");
986+ return -ENODEV;
987+ }
988+
989+ if (kvm_target_cpu() < 0) {
990+ kvm_err("Target CPU not supported!\n");
991+ return -ENODEV;
992+ }
993+
994+ err = init_hyp_mode();
995+ if (err)
996+ goto out_err;
997+
998+ kvm_coproc_table_init();
999+ return 0;
1000+out_err:
1001+ return err;
1002+}
1003+
1004+/* NOP: Compiling as a module not supported */
1005+void kvm_arch_exit(void)
1006+{
1007+}
1008+
1009+static int arm_init(void)
1010+{
1011+ int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1012+ return rc;
1013+}
1014+
1015+module_init(arm_init);
arch/arm/kvm/coproc.c+1046−0
@@ -0,0 +1,1046 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Authors: Rusty Russell <rusty@rustcorp.com.au>
4+ * Christoffer Dall <c.dall@virtualopensystems.com>
5+ *
6+ * This program is free software; you can redistribute it and/or modify
7+ * it under the terms of the GNU General Public License, version 2, as
8+ * published by the Free Software Foundation.
9+ *
10+ * This program is distributed in the hope that it will be useful,
11+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
12+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+ * GNU General Public License for more details.
14+ *
15+ * You should have received a copy of the GNU General Public License
16+ * along with this program; if not, write to the Free Software
17+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18+ */
19+#include <linux/mm.h>
20+#include <linux/kvm_host.h>
21+#include <linux/uaccess.h>
22+#include <asm/kvm_arm.h>
23+#include <asm/kvm_host.h>
24+#include <asm/kvm_emulate.h>
25+#include <asm/kvm_coproc.h>
26+#include <asm/cacheflush.h>
27+#include <asm/cputype.h>
28+#include <trace/events/kvm.h>
29+#include <asm/vfp.h>
30+#include "../vfp/vfpinstr.h"
31+
32+#include "trace.h"
33+#include "coproc.h"
34+
35+
36+/******************************************************************************
37+ * Co-processor emulation
38+ *****************************************************************************/
39+
40+/* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
41+static u32 cache_levels;
42+
43+/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
44+#define CSSELR_MAX 12
45+
46+int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run)
47+{
48+ kvm_inject_undefined(vcpu);
49+ return 1;
50+}
51+
52+int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
53+{
54+ /*
55+ * We can get here, if the host has been built without VFPv3 support,
56+ * but the guest attempted a floating point operation.
57+ */
58+ kvm_inject_undefined(vcpu);
59+ return 1;
60+}
61+
62+int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
63+{
64+ kvm_inject_undefined(vcpu);
65+ return 1;
66+}
67+
68+int kvm_handle_cp14_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
69+{
70+ kvm_inject_undefined(vcpu);
71+ return 1;
72+}
73+
74+/* See note at ARM ARM B1.14.4 */
75+static bool access_dcsw(struct kvm_vcpu *vcpu,
76+ const struct coproc_params *p,
77+ const struct coproc_reg *r)
78+{
79+ u32 val;
80+ int cpu;
81+
82+ cpu = get_cpu();
83+
84+ if (!p->is_write)
85+ return read_from_write_only(vcpu, p);
86+
87+ cpumask_setall(&vcpu->arch.require_dcache_flush);
88+ cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
89+
90+ /* If we were already preempted, take the long way around */
91+ if (cpu != vcpu->arch.last_pcpu) {
92+ flush_cache_all();
93+ goto done;
94+ }
95+
96+ val = *vcpu_reg(vcpu, p->Rt1);
97+
98+ switch (p->CRm) {
99+ case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */
100+ case 14: /* DCCISW */
101+ asm volatile("mcr p15, 0, %0, c7, c14, 2" : : "r" (val));
102+ break;
103+
104+ case 10: /* DCCSW */
105+ asm volatile("mcr p15, 0, %0, c7, c10, 2" : : "r" (val));
106+ break;
107+ }
108+
109+done:
110+ put_cpu();
111+
112+ return true;
113+}
114+
115+/*
116+ * We could trap ID_DFR0 and tell the guest we don't support performance
117+ * monitoring. Unfortunately the patch to make the kernel check ID_DFR0 was
118+ * NAKed, so it will read the PMCR anyway.
119+ *
120+ * Therefore we tell the guest we have 0 counters. Unfortunately, we
121+ * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
122+ * all PM registers, which doesn't crash the guest kernel at least.
123+ */
124+static bool pm_fake(struct kvm_vcpu *vcpu,
125+ const struct coproc_params *p,
126+ const struct coproc_reg *r)
127+{
128+ if (p->is_write)
129+ return ignore_write(vcpu, p);
130+ else
131+ return read_zero(vcpu, p);
132+}
133+
134+#define access_pmcr pm_fake
135+#define access_pmcntenset pm_fake
136+#define access_pmcntenclr pm_fake
137+#define access_pmovsr pm_fake
138+#define access_pmselr pm_fake
139+#define access_pmceid0 pm_fake
140+#define access_pmceid1 pm_fake
141+#define access_pmccntr pm_fake
142+#define access_pmxevtyper pm_fake
143+#define access_pmxevcntr pm_fake
144+#define access_pmuserenr pm_fake
145+#define access_pmintenset pm_fake
146+#define access_pmintenclr pm_fake
147+
148+/* Architected CP15 registers.
149+ * Important: Must be sorted ascending by CRn, CRM, Op1, Op2
150+ */
151+static const struct coproc_reg cp15_regs[] = {
152+ /* CSSELR: swapped by interrupt.S. */
153+ { CRn( 0), CRm( 0), Op1( 2), Op2( 0), is32,
154+ NULL, reset_unknown, c0_CSSELR },
155+
156+ /* TTBR0/TTBR1: swapped by interrupt.S. */
157+ { CRm( 2), Op1( 0), is64, NULL, reset_unknown64, c2_TTBR0 },
158+ { CRm( 2), Op1( 1), is64, NULL, reset_unknown64, c2_TTBR1 },
159+
160+ /* TTBCR: swapped by interrupt.S. */
161+ { CRn( 2), CRm( 0), Op1( 0), Op2( 2), is32,
162+ NULL, reset_val, c2_TTBCR, 0x00000000 },
163+
164+ /* DACR: swapped by interrupt.S. */
165+ { CRn( 3), CRm( 0), Op1( 0), Op2( 0), is32,
166+ NULL, reset_unknown, c3_DACR },
167+
168+ /* DFSR/IFSR/ADFSR/AIFSR: swapped by interrupt.S. */
169+ { CRn( 5), CRm( 0), Op1( 0), Op2( 0), is32,
170+ NULL, reset_unknown, c5_DFSR },
171+ { CRn( 5), CRm( 0), Op1( 0), Op2( 1), is32,
172+ NULL, reset_unknown, c5_IFSR },
173+ { CRn( 5), CRm( 1), Op1( 0), Op2( 0), is32,
174+ NULL, reset_unknown, c5_ADFSR },
175+ { CRn( 5), CRm( 1), Op1( 0), Op2( 1), is32,
176+ NULL, reset_unknown, c5_AIFSR },
177+
178+ /* DFAR/IFAR: swapped by interrupt.S. */
179+ { CRn( 6), CRm( 0), Op1( 0), Op2( 0), is32,
180+ NULL, reset_unknown, c6_DFAR },
181+ { CRn( 6), CRm( 0), Op1( 0), Op2( 2), is32,
182+ NULL, reset_unknown, c6_IFAR },
183+ /*
184+ * DC{C,I,CI}SW operations:
185+ */
186+ { CRn( 7), CRm( 6), Op1( 0), Op2( 2), is32, access_dcsw},
187+ { CRn( 7), CRm(10), Op1( 0), Op2( 2), is32, access_dcsw},
188+ { CRn( 7), CRm(14), Op1( 0), Op2( 2), is32, access_dcsw},
189+ /*
190+ * Dummy performance monitor implementation.
191+ */
192+ { CRn( 9), CRm(12), Op1( 0), Op2( 0), is32, access_pmcr},
193+ { CRn( 9), CRm(12), Op1( 0), Op2( 1), is32, access_pmcntenset},
194+ { CRn( 9), CRm(12), Op1( 0), Op2( 2), is32, access_pmcntenclr},
195+ { CRn( 9), CRm(12), Op1( 0), Op2( 3), is32, access_pmovsr},
196+ { CRn( 9), CRm(12), Op1( 0), Op2( 5), is32, access_pmselr},
197+ { CRn( 9), CRm(12), Op1( 0), Op2( 6), is32, access_pmceid0},
198+ { CRn( 9), CRm(12), Op1( 0), Op2( 7), is32, access_pmceid1},
199+ { CRn( 9), CRm(13), Op1( 0), Op2( 0), is32, access_pmccntr},
200+ { CRn( 9), CRm(13), Op1( 0), Op2( 1), is32, access_pmxevtyper},
201+ { CRn( 9), CRm(13), Op1( 0), Op2( 2), is32, access_pmxevcntr},
202+ { CRn( 9), CRm(14), Op1( 0), Op2( 0), is32, access_pmuserenr},
203+ { CRn( 9), CRm(14), Op1( 0), Op2( 1), is32, access_pmintenset},
204+ { CRn( 9), CRm(14), Op1( 0), Op2( 2), is32, access_pmintenclr},
205+
206+ /* PRRR/NMRR (aka MAIR0/MAIR1): swapped by interrupt.S. */
207+ { CRn(10), CRm( 2), Op1( 0), Op2( 0), is32,
208+ NULL, reset_unknown, c10_PRRR},
209+ { CRn(10), CRm( 2), Op1( 0), Op2( 1), is32,
210+ NULL, reset_unknown, c10_NMRR},
211+
212+ /* VBAR: swapped by interrupt.S. */
213+ { CRn(12), CRm( 0), Op1( 0), Op2( 0), is32,
214+ NULL, reset_val, c12_VBAR, 0x00000000 },
215+
216+ /* CONTEXTIDR/TPIDRURW/TPIDRURO/TPIDRPRW: swapped by interrupt.S. */
217+ { CRn(13), CRm( 0), Op1( 0), Op2( 1), is32,
218+ NULL, reset_val, c13_CID, 0x00000000 },
219+ { CRn(13), CRm( 0), Op1( 0), Op2( 2), is32,
220+ NULL, reset_unknown, c13_TID_URW },
221+ { CRn(13), CRm( 0), Op1( 0), Op2( 3), is32,
222+ NULL, reset_unknown, c13_TID_URO },
223+ { CRn(13), CRm( 0), Op1( 0), Op2( 4), is32,
224+ NULL, reset_unknown, c13_TID_PRIV },
225+};
226+
227+/* Target specific emulation tables */
228+static struct kvm_coproc_target_table *target_tables[KVM_ARM_NUM_TARGETS];
229+
230+void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table)
231+{
232+ target_tables[table->target] = table;
233+}
234+
235+/* Get specific register table for this target. */
236+static const struct coproc_reg *get_target_table(unsigned target, size_t *num)
237+{
238+ struct kvm_coproc_target_table *table;
239+
240+ table = target_tables[target];
241+ *num = table->num;
242+ return table->table;
243+}
244+
245+static const struct coproc_reg *find_reg(const struct coproc_params *params,
246+ const struct coproc_reg table[],
247+ unsigned int num)
248+{
249+ unsigned int i;
250+
251+ for (i = 0; i < num; i++) {
252+ const struct coproc_reg *r = &table[i];
253+
254+ if (params->is_64bit != r->is_64)
255+ continue;
256+ if (params->CRn != r->CRn)
257+ continue;
258+ if (params->CRm != r->CRm)
259+ continue;
260+ if (params->Op1 != r->Op1)
261+ continue;
262+ if (params->Op2 != r->Op2)
263+ continue;
264+
265+ return r;
266+ }
267+ return NULL;
268+}
269+
270+static int emulate_cp15(struct kvm_vcpu *vcpu,
271+ const struct coproc_params *params)
272+{
273+ size_t num;
274+ const struct coproc_reg *table, *r;
275+
276+ trace_kvm_emulate_cp15_imp(params->Op1, params->Rt1, params->CRn,
277+ params->CRm, params->Op2, params->is_write);
278+
279+ table = get_target_table(vcpu->arch.target, &num);
280+
281+ /* Search target-specific then generic table. */
282+ r = find_reg(params, table, num);
283+ if (!r)
284+ r = find_reg(params, cp15_regs, ARRAY_SIZE(cp15_regs));
285+
286+ if (likely(r)) {
287+ /* If we don't have an accessor, we should never get here! */
288+ BUG_ON(!r->access);
289+
290+ if (likely(r->access(vcpu, params, r))) {
291+ /* Skip instruction, since it was emulated */
292+ kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1);
293+ return 1;
294+ }
295+ /* If access function fails, it should complain. */
296+ } else {
297+ kvm_err("Unsupported guest CP15 access at: %08x\n",
298+ *vcpu_pc(vcpu));
299+ print_cp_instr(params);
300+ }
301+ kvm_inject_undefined(vcpu);
302+ return 1;
303+}
304+
305+/**
306+ * kvm_handle_cp15_64 -- handles a mrrc/mcrr trap on a guest CP15 access
307+ * @vcpu: The VCPU pointer
308+ * @run: The kvm_run struct
309+ */
310+int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
311+{
312+ struct coproc_params params;
313+
314+ params.CRm = (vcpu->arch.hsr >> 1) & 0xf;
315+ params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf;
316+ params.is_write = ((vcpu->arch.hsr & 1) == 0);
317+ params.is_64bit = true;
318+
319+ params.Op1 = (vcpu->arch.hsr >> 16) & 0xf;
320+ params.Op2 = 0;
321+ params.Rt2 = (vcpu->arch.hsr >> 10) & 0xf;
322+ params.CRn = 0;
323+
324+ return emulate_cp15(vcpu, &params);
325+}
326+
327+static void reset_coproc_regs(struct kvm_vcpu *vcpu,
328+ const struct coproc_reg *table, size_t num)
329+{
330+ unsigned long i;
331+
332+ for (i = 0; i < num; i++)
333+ if (table[i].reset)
334+ table[i].reset(vcpu, &table[i]);
335+}
336+
337+/**
338+ * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access
339+ * @vcpu: The VCPU pointer
340+ * @run: The kvm_run struct
341+ */
342+int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
343+{
344+ struct coproc_params params;
345+
346+ params.CRm = (vcpu->arch.hsr >> 1) & 0xf;
347+ params.Rt1 = (vcpu->arch.hsr >> 5) & 0xf;
348+ params.is_write = ((vcpu->arch.hsr & 1) == 0);
349+ params.is_64bit = false;
350+
351+ params.CRn = (vcpu->arch.hsr >> 10) & 0xf;
352+ params.Op1 = (vcpu->arch.hsr >> 14) & 0x7;
353+ params.Op2 = (vcpu->arch.hsr >> 17) & 0x7;
354+ params.Rt2 = 0;
355+
356+ return emulate_cp15(vcpu, &params);
357+}
358+
359+/******************************************************************************
360+ * Userspace API
361+ *****************************************************************************/
362+
363+static bool index_to_params(u64 id, struct coproc_params *params)
364+{
365+ switch (id & KVM_REG_SIZE_MASK) {
366+ case KVM_REG_SIZE_U32:
367+ /* Any unused index bits means it's not valid. */
368+ if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
369+ | KVM_REG_ARM_COPROC_MASK
370+ | KVM_REG_ARM_32_CRN_MASK
371+ | KVM_REG_ARM_CRM_MASK
372+ | KVM_REG_ARM_OPC1_MASK
373+ | KVM_REG_ARM_32_OPC2_MASK))
374+ return false;
375+
376+ params->is_64bit = false;
377+ params->CRn = ((id & KVM_REG_ARM_32_CRN_MASK)
378+ >> KVM_REG_ARM_32_CRN_SHIFT);
379+ params->CRm = ((id & KVM_REG_ARM_CRM_MASK)
380+ >> KVM_REG_ARM_CRM_SHIFT);
381+ params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
382+ >> KVM_REG_ARM_OPC1_SHIFT);
383+ params->Op2 = ((id & KVM_REG_ARM_32_OPC2_MASK)
384+ >> KVM_REG_ARM_32_OPC2_SHIFT);
385+ return true;
386+ case KVM_REG_SIZE_U64:
387+ /* Any unused index bits means it's not valid. */
388+ if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
389+ | KVM_REG_ARM_COPROC_MASK
390+ | KVM_REG_ARM_CRM_MASK
391+ | KVM_REG_ARM_OPC1_MASK))
392+ return false;
393+ params->is_64bit = true;
394+ params->CRm = ((id & KVM_REG_ARM_CRM_MASK)
395+ >> KVM_REG_ARM_CRM_SHIFT);
396+ params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
397+ >> KVM_REG_ARM_OPC1_SHIFT);
398+ params->Op2 = 0;
399+ params->CRn = 0;
400+ return true;
401+ default:
402+ return false;
403+ }
404+}
405+
406+/* Decode an index value, and find the cp15 coproc_reg entry. */
407+static const struct coproc_reg *index_to_coproc_reg(struct kvm_vcpu *vcpu,
408+ u64 id)
409+{
410+ size_t num;
411+ const struct coproc_reg *table, *r;
412+ struct coproc_params params;
413+
414+ /* We only do cp15 for now. */
415+ if ((id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT != 15)
416+ return NULL;
417+
418+ if (!index_to_params(id, &params))
419+ return NULL;
420+
421+ table = get_target_table(vcpu->arch.target, &num);
422+ r = find_reg(&params, table, num);
423+ if (!r)
424+ r = find_reg(&params, cp15_regs, ARRAY_SIZE(cp15_regs));
425+
426+ /* Not saved in the cp15 array? */
427+ if (r && !r->reg)
428+ r = NULL;
429+
430+ return r;
431+}
432+
433+/*
434+ * These are the invariant cp15 registers: we let the guest see the host
435+ * versions of these, so they're part of the guest state.
436+ *
437+ * A future CPU may provide a mechanism to present different values to
438+ * the guest, or a future kvm may trap them.
439+ */
440+/* Unfortunately, there's no register-argument for mrc, so generate. */
441+#define FUNCTION_FOR32(crn, crm, op1, op2, name) \
442+ static void get_##name(struct kvm_vcpu *v, \
443+ const struct coproc_reg *r) \
444+ { \
445+ u32 val; \
446+ \
447+ asm volatile("mrc p15, " __stringify(op1) \
448+ ", %0, c" __stringify(crn) \
449+ ", c" __stringify(crm) \
450+ ", " __stringify(op2) "\n" : "=r" (val)); \
451+ ((struct coproc_reg *)r)->val = val; \
452+ }
453+
454+FUNCTION_FOR32(0, 0, 0, 0, MIDR)
455+FUNCTION_FOR32(0, 0, 0, 1, CTR)
456+FUNCTION_FOR32(0, 0, 0, 2, TCMTR)
457+FUNCTION_FOR32(0, 0, 0, 3, TLBTR)
458+FUNCTION_FOR32(0, 0, 0, 6, REVIDR)
459+FUNCTION_FOR32(0, 1, 0, 0, ID_PFR0)
460+FUNCTION_FOR32(0, 1, 0, 1, ID_PFR1)
461+FUNCTION_FOR32(0, 1, 0, 2, ID_DFR0)
462+FUNCTION_FOR32(0, 1, 0, 3, ID_AFR0)
463+FUNCTION_FOR32(0, 1, 0, 4, ID_MMFR0)
464+FUNCTION_FOR32(0, 1, 0, 5, ID_MMFR1)
465+FUNCTION_FOR32(0, 1, 0, 6, ID_MMFR2)
466+FUNCTION_FOR32(0, 1, 0, 7, ID_MMFR3)
467+FUNCTION_FOR32(0, 2, 0, 0, ID_ISAR0)
468+FUNCTION_FOR32(0, 2, 0, 1, ID_ISAR1)
469+FUNCTION_FOR32(0, 2, 0, 2, ID_ISAR2)
470+FUNCTION_FOR32(0, 2, 0, 3, ID_ISAR3)
471+FUNCTION_FOR32(0, 2, 0, 4, ID_ISAR4)
472+FUNCTION_FOR32(0, 2, 0, 5, ID_ISAR5)
473+FUNCTION_FOR32(0, 0, 1, 1, CLIDR)
474+FUNCTION_FOR32(0, 0, 1, 7, AIDR)
475+
476+/* ->val is filled in by kvm_invariant_coproc_table_init() */
477+static struct coproc_reg invariant_cp15[] = {
478+ { CRn( 0), CRm( 0), Op1( 0), Op2( 0), is32, NULL, get_MIDR },
479+ { CRn( 0), CRm( 0), Op1( 0), Op2( 1), is32, NULL, get_CTR },
480+ { CRn( 0), CRm( 0), Op1( 0), Op2( 2), is32, NULL, get_TCMTR },
481+ { CRn( 0), CRm( 0), Op1( 0), Op2( 3), is32, NULL, get_TLBTR },
482+ { CRn( 0), CRm( 0), Op1( 0), Op2( 6), is32, NULL, get_REVIDR },
483+
484+ { CRn( 0), CRm( 1), Op1( 0), Op2( 0), is32, NULL, get_ID_PFR0 },
485+ { CRn( 0), CRm( 1), Op1( 0), Op2( 1), is32, NULL, get_ID_PFR1 },
486+ { CRn( 0), CRm( 1), Op1( 0), Op2( 2), is32, NULL, get_ID_DFR0 },
487+ { CRn( 0), CRm( 1), Op1( 0), Op2( 3), is32, NULL, get_ID_AFR0 },
488+ { CRn( 0), CRm( 1), Op1( 0), Op2( 4), is32, NULL, get_ID_MMFR0 },
489+ { CRn( 0), CRm( 1), Op1( 0), Op2( 5), is32, NULL, get_ID_MMFR1 },
490+ { CRn( 0), CRm( 1), Op1( 0), Op2( 6), is32, NULL, get_ID_MMFR2 },
491+ { CRn( 0), CRm( 1), Op1( 0), Op2( 7), is32, NULL, get_ID_MMFR3 },
492+
493+ { CRn( 0), CRm( 2), Op1( 0), Op2( 0), is32, NULL, get_ID_ISAR0 },
494+ { CRn( 0), CRm( 2), Op1( 0), Op2( 1), is32, NULL, get_ID_ISAR1 },
495+ { CRn( 0), CRm( 2), Op1( 0), Op2( 2), is32, NULL, get_ID_ISAR2 },
496+ { CRn( 0), CRm( 2), Op1( 0), Op2( 3), is32, NULL, get_ID_ISAR3 },
497+ { CRn( 0), CRm( 2), Op1( 0), Op2( 4), is32, NULL, get_ID_ISAR4 },
498+ { CRn( 0), CRm( 2), Op1( 0), Op2( 5), is32, NULL, get_ID_ISAR5 },
499+
500+ { CRn( 0), CRm( 0), Op1( 1), Op2( 1), is32, NULL, get_CLIDR },
501+ { CRn( 0), CRm( 0), Op1( 1), Op2( 7), is32, NULL, get_AIDR },
502+};
503+
504+static int reg_from_user(void *val, const void __user *uaddr, u64 id)
505+{
506+ /* This Just Works because we are little endian. */
507+ if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
508+ return -EFAULT;
509+ return 0;
510+}
511+
512+static int reg_to_user(void __user *uaddr, const void *val, u64 id)
513+{
514+ /* This Just Works because we are little endian. */
515+ if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
516+ return -EFAULT;
517+ return 0;
518+}
519+
520+static int get_invariant_cp15(u64 id, void __user *uaddr)
521+{
522+ struct coproc_params params;
523+ const struct coproc_reg *r;
524+
525+ if (!index_to_params(id, &params))
526+ return -ENOENT;
527+
528+ r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
529+ if (!r)
530+ return -ENOENT;
531+
532+ return reg_to_user(uaddr, &r->val, id);
533+}
534+
535+static int set_invariant_cp15(u64 id, void __user *uaddr)
536+{
537+ struct coproc_params params;
538+ const struct coproc_reg *r;
539+ int err;
540+ u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
541+
542+ if (!index_to_params(id, &params))
543+ return -ENOENT;
544+ r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
545+ if (!r)
546+ return -ENOENT;
547+
548+ err = reg_from_user(&val, uaddr, id);
549+ if (err)
550+ return err;
551+
552+ /* This is what we mean by invariant: you can't change it. */
553+ if (r->val != val)
554+ return -EINVAL;
555+
556+ return 0;
557+}
558+
559+static bool is_valid_cache(u32 val)
560+{
561+ u32 level, ctype;
562+
563+ if (val >= CSSELR_MAX)
564+ return -ENOENT;
565+
566+ /* Bottom bit is Instruction or Data bit. Next 3 bits are level. */
567+ level = (val >> 1);
568+ ctype = (cache_levels >> (level * 3)) & 7;
569+
570+ switch (ctype) {
571+ case 0: /* No cache */
572+ return false;
573+ case 1: /* Instruction cache only */
574+ return (val & 1);
575+ case 2: /* Data cache only */
576+ case 4: /* Unified cache */
577+ return !(val & 1);
578+ case 3: /* Separate instruction and data caches */
579+ return true;
580+ default: /* Reserved: we can't know instruction or data. */
581+ return false;
582+ }
583+}
584+
585+/* Which cache CCSIDR represents depends on CSSELR value. */
586+static u32 get_ccsidr(u32 csselr)
587+{
588+ u32 ccsidr;
589+
590+ /* Make sure noone else changes CSSELR during this! */
591+ local_irq_disable();
592+ /* Put value into CSSELR */
593+ asm volatile("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr));
594+ isb();
595+ /* Read result out of CCSIDR */
596+ asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr));
597+ local_irq_enable();
598+
599+ return ccsidr;
600+}
601+
602+static int demux_c15_get(u64 id, void __user *uaddr)
603+{
604+ u32 val;
605+ u32 __user *uval = uaddr;
606+
607+ /* Fail if we have unknown bits set. */
608+ if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
609+ | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
610+ return -ENOENT;
611+
612+ switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
613+ case KVM_REG_ARM_DEMUX_ID_CCSIDR:
614+ if (KVM_REG_SIZE(id) != 4)
615+ return -ENOENT;
616+ val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
617+ >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
618+ if (!is_valid_cache(val))
619+ return -ENOENT;
620+
621+ return put_user(get_ccsidr(val), uval);
622+ default:
623+ return -ENOENT;
624+ }
625+}
626+
627+static int demux_c15_set(u64 id, void __user *uaddr)
628+{
629+ u32 val, newval;
630+ u32 __user *uval = uaddr;
631+
632+ /* Fail if we have unknown bits set. */
633+ if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
634+ | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
635+ return -ENOENT;
636+
637+ switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
638+ case KVM_REG_ARM_DEMUX_ID_CCSIDR:
639+ if (KVM_REG_SIZE(id) != 4)
640+ return -ENOENT;
641+ val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
642+ >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
643+ if (!is_valid_cache(val))
644+ return -ENOENT;
645+
646+ if (get_user(newval, uval))
647+ return -EFAULT;
648+
649+ /* This is also invariant: you can't change it. */
650+ if (newval != get_ccsidr(val))
651+ return -EINVAL;
652+ return 0;
653+ default:
654+ return -ENOENT;
655+ }
656+}
657+
658+#ifdef CONFIG_VFPv3
659+static const int vfp_sysregs[] = { KVM_REG_ARM_VFP_FPEXC,
660+ KVM_REG_ARM_VFP_FPSCR,
661+ KVM_REG_ARM_VFP_FPINST,
662+ KVM_REG_ARM_VFP_FPINST2,
663+ KVM_REG_ARM_VFP_MVFR0,
664+ KVM_REG_ARM_VFP_MVFR1,
665+ KVM_REG_ARM_VFP_FPSID };
666+
667+static unsigned int num_fp_regs(void)
668+{
669+ if (((fmrx(MVFR0) & MVFR0_A_SIMD_MASK) >> MVFR0_A_SIMD_BIT) == 2)
670+ return 32;
671+ else
672+ return 16;
673+}
674+
675+static unsigned int num_vfp_regs(void)
676+{
677+ /* Normal FP regs + control regs. */
678+ return num_fp_regs() + ARRAY_SIZE(vfp_sysregs);
679+}
680+
681+static int copy_vfp_regids(u64 __user *uindices)
682+{
683+ unsigned int i;
684+ const u64 u32reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP;
685+ const u64 u64reg = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
686+
687+ for (i = 0; i < num_fp_regs(); i++) {
688+ if (put_user((u64reg | KVM_REG_ARM_VFP_BASE_REG) + i,
689+ uindices))
690+ return -EFAULT;
691+ uindices++;
692+ }
693+
694+ for (i = 0; i < ARRAY_SIZE(vfp_sysregs); i++) {
695+ if (put_user(u32reg | vfp_sysregs[i], uindices))
696+ return -EFAULT;
697+ uindices++;
698+ }
699+
700+ return num_vfp_regs();
701+}
702+
703+static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
704+{
705+ u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
706+ u32 val;
707+
708+ /* Fail if we have unknown bits set. */
709+ if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
710+ | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
711+ return -ENOENT;
712+
713+ if (vfpid < num_fp_regs()) {
714+ if (KVM_REG_SIZE(id) != 8)
715+ return -ENOENT;
716+ return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpregs[vfpid],
717+ id);
718+ }
719+
720+ /* FP control registers are all 32 bit. */
721+ if (KVM_REG_SIZE(id) != 4)
722+ return -ENOENT;
723+
724+ switch (vfpid) {
725+ case KVM_REG_ARM_VFP_FPEXC:
726+ return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpexc, id);
727+ case KVM_REG_ARM_VFP_FPSCR:
728+ return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpscr, id);
729+ case KVM_REG_ARM_VFP_FPINST:
730+ return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst, id);
731+ case KVM_REG_ARM_VFP_FPINST2:
732+ return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst2, id);
733+ case KVM_REG_ARM_VFP_MVFR0:
734+ val = fmrx(MVFR0);
735+ return reg_to_user(uaddr, &val, id);
736+ case KVM_REG_ARM_VFP_MVFR1:
737+ val = fmrx(MVFR1);
738+ return reg_to_user(uaddr, &val, id);
739+ case KVM_REG_ARM_VFP_FPSID:
740+ val = fmrx(FPSID);
741+ return reg_to_user(uaddr, &val, id);
742+ default:
743+ return -ENOENT;
744+ }
745+}
746+
747+static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
748+{
749+ u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
750+ u32 val;
751+
752+ /* Fail if we have unknown bits set. */
753+ if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
754+ | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
755+ return -ENOENT;
756+
757+ if (vfpid < num_fp_regs()) {
758+ if (KVM_REG_SIZE(id) != 8)
759+ return -ENOENT;
760+ return reg_from_user(&vcpu->arch.vfp_guest.fpregs[vfpid],
761+ uaddr, id);
762+ }
763+
764+ /* FP control registers are all 32 bit. */
765+ if (KVM_REG_SIZE(id) != 4)
766+ return -ENOENT;
767+
768+ switch (vfpid) {
769+ case KVM_REG_ARM_VFP_FPEXC:
770+ return reg_from_user(&vcpu->arch.vfp_guest.fpexc, uaddr, id);
771+ case KVM_REG_ARM_VFP_FPSCR:
772+ return reg_from_user(&vcpu->arch.vfp_guest.fpscr, uaddr, id);
773+ case KVM_REG_ARM_VFP_FPINST:
774+ return reg_from_user(&vcpu->arch.vfp_guest.fpinst, uaddr, id);
775+ case KVM_REG_ARM_VFP_FPINST2:
776+ return reg_from_user(&vcpu->arch.vfp_guest.fpinst2, uaddr, id);
777+ /* These are invariant. */
778+ case KVM_REG_ARM_VFP_MVFR0:
779+ if (reg_from_user(&val, uaddr, id))
780+ return -EFAULT;
781+ if (val != fmrx(MVFR0))
782+ return -EINVAL;
783+ return 0;
784+ case KVM_REG_ARM_VFP_MVFR1:
785+ if (reg_from_user(&val, uaddr, id))
786+ return -EFAULT;
787+ if (val != fmrx(MVFR1))
788+ return -EINVAL;
789+ return 0;
790+ case KVM_REG_ARM_VFP_FPSID:
791+ if (reg_from_user(&val, uaddr, id))
792+ return -EFAULT;
793+ if (val != fmrx(FPSID))
794+ return -EINVAL;
795+ return 0;
796+ default:
797+ return -ENOENT;
798+ }
799+}
800+#else /* !CONFIG_VFPv3 */
801+static unsigned int num_vfp_regs(void)
802+{
803+ return 0;
804+}
805+
806+static int copy_vfp_regids(u64 __user *uindices)
807+{
808+ return 0;
809+}
810+
811+static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
812+{
813+ return -ENOENT;
814+}
815+
816+static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
817+{
818+ return -ENOENT;
819+}
820+#endif /* !CONFIG_VFPv3 */
821+
822+int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
823+{
824+ const struct coproc_reg *r;
825+ void __user *uaddr = (void __user *)(long)reg->addr;
826+
827+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
828+ return demux_c15_get(reg->id, uaddr);
829+
830+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
831+ return vfp_get_reg(vcpu, reg->id, uaddr);
832+
833+ r = index_to_coproc_reg(vcpu, reg->id);
834+ if (!r)
835+ return get_invariant_cp15(reg->id, uaddr);
836+
837+ /* Note: copies two regs if size is 64 bit. */
838+ return reg_to_user(uaddr, &vcpu->arch.cp15[r->reg], reg->id);
839+}
840+
841+int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
842+{
843+ const struct coproc_reg *r;
844+ void __user *uaddr = (void __user *)(long)reg->addr;
845+
846+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
847+ return demux_c15_set(reg->id, uaddr);
848+
849+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
850+ return vfp_set_reg(vcpu, reg->id, uaddr);
851+
852+ r = index_to_coproc_reg(vcpu, reg->id);
853+ if (!r)
854+ return set_invariant_cp15(reg->id, uaddr);
855+
856+ /* Note: copies two regs if size is 64 bit */
857+ return reg_from_user(&vcpu->arch.cp15[r->reg], uaddr, reg->id);
858+}
859+
860+static unsigned int num_demux_regs(void)
861+{
862+ unsigned int i, count = 0;
863+
864+ for (i = 0; i < CSSELR_MAX; i++)
865+ if (is_valid_cache(i))
866+ count++;
867+
868+ return count;
869+}
870+
871+static int write_demux_regids(u64 __user *uindices)
872+{
873+ u64 val = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
874+ unsigned int i;
875+
876+ val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
877+ for (i = 0; i < CSSELR_MAX; i++) {
878+ if (!is_valid_cache(i))
879+ continue;
880+ if (put_user(val | i, uindices))
881+ return -EFAULT;
882+ uindices++;
883+ }
884+ return 0;
885+}
886+
887+static u64 cp15_to_index(const struct coproc_reg *reg)
888+{
889+ u64 val = KVM_REG_ARM | (15 << KVM_REG_ARM_COPROC_SHIFT);
890+ if (reg->is_64) {
891+ val |= KVM_REG_SIZE_U64;
892+ val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
893+ val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT);
894+ } else {
895+ val |= KVM_REG_SIZE_U32;
896+ val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
897+ val |= (reg->Op2 << KVM_REG_ARM_32_OPC2_SHIFT);
898+ val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT);
899+ val |= (reg->CRn << KVM_REG_ARM_32_CRN_SHIFT);
900+ }
901+ return val;
902+}
903+
904+static bool copy_reg_to_user(const struct coproc_reg *reg, u64 __user **uind)
905+{
906+ if (!*uind)
907+ return true;
908+
909+ if (put_user(cp15_to_index(reg), *uind))
910+ return false;
911+
912+ (*uind)++;
913+ return true;
914+}
915+
916+/* Assumed ordered tables, see kvm_coproc_table_init. */
917+static int walk_cp15(struct kvm_vcpu *vcpu, u64 __user *uind)
918+{
919+ const struct coproc_reg *i1, *i2, *end1, *end2;
920+ unsigned int total = 0;
921+ size_t num;
922+
923+ /* We check for duplicates here, to allow arch-specific overrides. */
924+ i1 = get_target_table(vcpu->arch.target, &num);
925+ end1 = i1 + num;
926+ i2 = cp15_regs;
927+ end2 = cp15_regs + ARRAY_SIZE(cp15_regs);
928+
929+ BUG_ON(i1 == end1 || i2 == end2);
930+
931+ /* Walk carefully, as both tables may refer to the same register. */
932+ while (i1 || i2) {
933+ int cmp = cmp_reg(i1, i2);
934+ /* target-specific overrides generic entry. */
935+ if (cmp <= 0) {
936+ /* Ignore registers we trap but don't save. */
937+ if (i1->reg) {
938+ if (!copy_reg_to_user(i1, &uind))
939+ return -EFAULT;
940+ total++;
941+ }
942+ } else {
943+ /* Ignore registers we trap but don't save. */
944+ if (i2->reg) {
945+ if (!copy_reg_to_user(i2, &uind))
946+ return -EFAULT;
947+ total++;
948+ }
949+ }
950+
951+ if (cmp <= 0 && ++i1 == end1)
952+ i1 = NULL;
953+ if (cmp >= 0 && ++i2 == end2)
954+ i2 = NULL;
955+ }
956+ return total;
957+}
958+
959+unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu)
960+{
961+ return ARRAY_SIZE(invariant_cp15)
962+ + num_demux_regs()
963+ + num_vfp_regs()
964+ + walk_cp15(vcpu, (u64 __user *)NULL);
965+}
966+
967+int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
968+{
969+ unsigned int i;
970+ int err;
971+
972+ /* Then give them all the invariant registers' indices. */
973+ for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++) {
974+ if (put_user(cp15_to_index(&invariant_cp15[i]), uindices))
975+ return -EFAULT;
976+ uindices++;
977+ }
978+
979+ err = walk_cp15(vcpu, uindices);
980+ if (err < 0)
981+ return err;
982+ uindices += err;
983+
984+ err = copy_vfp_regids(uindices);
985+ if (err < 0)
986+ return err;
987+ uindices += err;
988+
989+ return write_demux_regids(uindices);
990+}
991+
992+void kvm_coproc_table_init(void)
993+{
994+ unsigned int i;
995+
996+ /* Make sure tables are unique and in order. */
997+ for (i = 1; i < ARRAY_SIZE(cp15_regs); i++)
998+ BUG_ON(cmp_reg(&cp15_regs[i-1], &cp15_regs[i]) >= 0);
999+
1000+ /* We abuse the reset function to overwrite the table itself. */
1001+ for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++)
1002+ invariant_cp15[i].reset(NULL, &invariant_cp15[i]);
1003+
1004+ /*
1005+ * CLIDR format is awkward, so clean it up. See ARM B4.1.20:
1006+ *
1007+ * If software reads the Cache Type fields from Ctype1
1008+ * upwards, once it has seen a value of 0b000, no caches
1009+ * exist at further-out levels of the hierarchy. So, for
1010+ * example, if Ctype3 is the first Cache Type field with a
1011+ * value of 0b000, the values of Ctype4 to Ctype7 must be
1012+ * ignored.
1013+ */
1014+ asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r" (cache_levels));
1015+ for (i = 0; i < 7; i++)
1016+ if (((cache_levels >> (i*3)) & 7) == 0)
1017+ break;
1018+ /* Clear all higher bits. */
1019+ cache_levels &= (1 << (i*3))-1;
1020+}
1021+
1022+/**
1023+ * kvm_reset_coprocs - sets cp15 registers to reset value
1024+ * @vcpu: The VCPU pointer
1025+ *
1026+ * This function finds the right table above and sets the registers on the
1027+ * virtual CPU struct to their architecturally defined reset values.
1028+ */
1029+void kvm_reset_coprocs(struct kvm_vcpu *vcpu)
1030+{
1031+ size_t num;
1032+ const struct coproc_reg *table;
1033+
1034+ /* Catch someone adding a register without putting in reset entry. */
1035+ memset(vcpu->arch.cp15, 0x42, sizeof(vcpu->arch.cp15));
1036+
1037+ /* Generic chip reset first (so target could override). */
1038+ reset_coproc_regs(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
1039+
1040+ table = get_target_table(vcpu->arch.target, &num);
1041+ reset_coproc_regs(vcpu, table, num);
1042+
1043+ for (num = 1; num < NR_CP15_REGS; num++)
1044+ if (vcpu->arch.cp15[num] == 0x42424242)
1045+ panic("Didn't reset vcpu->arch.cp15[%zi]", num);
1046+}
arch/arm/kvm/coproc.h+153−0
@@ -0,0 +1,153 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Authors: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#ifndef __ARM_KVM_COPROC_LOCAL_H__
20+#define __ARM_KVM_COPROC_LOCAL_H__
21+
22+struct coproc_params {
23+ unsigned long CRn;
24+ unsigned long CRm;
25+ unsigned long Op1;
26+ unsigned long Op2;
27+ unsigned long Rt1;
28+ unsigned long Rt2;
29+ bool is_64bit;
30+ bool is_write;
31+};
32+
33+struct coproc_reg {
34+ /* MRC/MCR/MRRC/MCRR instruction which accesses it. */
35+ unsigned long CRn;
36+ unsigned long CRm;
37+ unsigned long Op1;
38+ unsigned long Op2;
39+
40+ bool is_64;
41+
42+ /* Trapped access from guest, if non-NULL. */
43+ bool (*access)(struct kvm_vcpu *,
44+ const struct coproc_params *,
45+ const struct coproc_reg *);
46+
47+ /* Initialization for vcpu. */
48+ void (*reset)(struct kvm_vcpu *, const struct coproc_reg *);
49+
50+ /* Index into vcpu->arch.cp15[], or 0 if we don't need to save it. */
51+ unsigned long reg;
52+
53+ /* Value (usually reset value) */
54+ u64 val;
55+};
56+
57+static inline void print_cp_instr(const struct coproc_params *p)
58+{
59+ /* Look, we even formatted it for you to paste into the table! */
60+ if (p->is_64bit) {
61+ kvm_pr_unimpl(" { CRm(%2lu), Op1(%2lu), is64, func_%s },\n",
62+ p->CRm, p->Op1, p->is_write ? "write" : "read");
63+ } else {
64+ kvm_pr_unimpl(" { CRn(%2lu), CRm(%2lu), Op1(%2lu), Op2(%2lu), is32,"
65+ " func_%s },\n",
66+ p->CRn, p->CRm, p->Op1, p->Op2,
67+ p->is_write ? "write" : "read");
68+ }
69+}
70+
71+static inline bool ignore_write(struct kvm_vcpu *vcpu,
72+ const struct coproc_params *p)
73+{
74+ return true;
75+}
76+
77+static inline bool read_zero(struct kvm_vcpu *vcpu,
78+ const struct coproc_params *p)
79+{
80+ *vcpu_reg(vcpu, p->Rt1) = 0;
81+ return true;
82+}
83+
84+static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
85+ const struct coproc_params *params)
86+{
87+ kvm_debug("CP15 write to read-only register at: %08x\n",
88+ *vcpu_pc(vcpu));
89+ print_cp_instr(params);
90+ return false;
91+}
92+
93+static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
94+ const struct coproc_params *params)
95+{
96+ kvm_debug("CP15 read to write-only register at: %08x\n",
97+ *vcpu_pc(vcpu));
98+ print_cp_instr(params);
99+ return false;
100+}
101+
102+/* Reset functions */
103+static inline void reset_unknown(struct kvm_vcpu *vcpu,
104+ const struct coproc_reg *r)
105+{
106+ BUG_ON(!r->reg);
107+ BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
108+ vcpu->arch.cp15[r->reg] = 0xdecafbad;
109+}
110+
111+static inline void reset_val(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
112+{
113+ BUG_ON(!r->reg);
114+ BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
115+ vcpu->arch.cp15[r->reg] = r->val;
116+}
117+
118+static inline void reset_unknown64(struct kvm_vcpu *vcpu,
119+ const struct coproc_reg *r)
120+{
121+ BUG_ON(!r->reg);
122+ BUG_ON(r->reg + 1 >= ARRAY_SIZE(vcpu->arch.cp15));
123+
124+ vcpu->arch.cp15[r->reg] = 0xdecafbad;
125+ vcpu->arch.cp15[r->reg+1] = 0xd0c0ffee;
126+}
127+
128+static inline int cmp_reg(const struct coproc_reg *i1,
129+ const struct coproc_reg *i2)
130+{
131+ BUG_ON(i1 == i2);
132+ if (!i1)
133+ return 1;
134+ else if (!i2)
135+ return -1;
136+ if (i1->CRn != i2->CRn)
137+ return i1->CRn - i2->CRn;
138+ if (i1->CRm != i2->CRm)
139+ return i1->CRm - i2->CRm;
140+ if (i1->Op1 != i2->Op1)
141+ return i1->Op1 - i2->Op1;
142+ return i1->Op2 - i2->Op2;
143+}
144+
145+
146+#define CRn(_x) .CRn = _x
147+#define CRm(_x) .CRm = _x
148+#define Op1(_x) .Op1 = _x
149+#define Op2(_x) .Op2 = _x
150+#define is64 .is_64 = true
151+#define is32 .is_64 = false
152+
153+#endif /* __ARM_KVM_COPROC_LOCAL_H__ */
arch/arm/kvm/coproc_a15.c+162−0
@@ -0,0 +1,162 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Authors: Rusty Russell <rusty@rustcorp.au>
4+ * Christoffer Dall <c.dall@virtualopensystems.com>
5+ *
6+ * This program is free software; you can redistribute it and/or modify
7+ * it under the terms of the GNU General Public License, version 2, as
8+ * published by the Free Software Foundation.
9+ *
10+ * This program is distributed in the hope that it will be useful,
11+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
12+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+ * GNU General Public License for more details.
14+ *
15+ * You should have received a copy of the GNU General Public License
16+ * along with this program; if not, write to the Free Software
17+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18+ */
19+#include <linux/kvm_host.h>
20+#include <asm/cputype.h>
21+#include <asm/kvm_arm.h>
22+#include <asm/kvm_host.h>
23+#include <asm/kvm_emulate.h>
24+#include <asm/kvm_coproc.h>
25+#include <linux/init.h>
26+
27+static void reset_mpidr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
28+{
29+ /*
30+ * Compute guest MPIDR:
31+ * (Even if we present only one VCPU to the guest on an SMP
32+ * host we don't set the U bit in the MPIDR, or vice versa, as
33+ * revealing the underlying hardware properties is likely to
34+ * be the best choice).
35+ */
36+ vcpu->arch.cp15[c0_MPIDR] = (read_cpuid_mpidr() & ~MPIDR_LEVEL_MASK)
37+ | (vcpu->vcpu_id & MPIDR_LEVEL_MASK);
38+}
39+
40+#include "coproc.h"
41+
42+/* A15 TRM 4.3.28: RO WI */
43+static bool access_actlr(struct kvm_vcpu *vcpu,
44+ const struct coproc_params *p,
45+ const struct coproc_reg *r)
46+{
47+ if (p->is_write)
48+ return ignore_write(vcpu, p);
49+
50+ *vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c1_ACTLR];
51+ return true;
52+}
53+
54+/* A15 TRM 4.3.60: R/O. */
55+static bool access_cbar(struct kvm_vcpu *vcpu,
56+ const struct coproc_params *p,
57+ const struct coproc_reg *r)
58+{
59+ if (p->is_write)
60+ return write_to_read_only(vcpu, p);
61+ return read_zero(vcpu, p);
62+}
63+
64+/* A15 TRM 4.3.48: R/O WI. */
65+static bool access_l2ctlr(struct kvm_vcpu *vcpu,
66+ const struct coproc_params *p,
67+ const struct coproc_reg *r)
68+{
69+ if (p->is_write)
70+ return ignore_write(vcpu, p);
71+
72+ *vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c9_L2CTLR];
73+ return true;
74+}
75+
76+static void reset_l2ctlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
77+{
78+ u32 l2ctlr, ncores;
79+
80+ asm volatile("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr));
81+ l2ctlr &= ~(3 << 24);
82+ ncores = atomic_read(&vcpu->kvm->online_vcpus) - 1;
83+ l2ctlr |= (ncores & 3) << 24;
84+
85+ vcpu->arch.cp15[c9_L2CTLR] = l2ctlr;
86+}
87+
88+static void reset_actlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
89+{
90+ u32 actlr;
91+
92+ /* ACTLR contains SMP bit: make sure you create all cpus first! */
93+ asm volatile("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr));
94+ /* Make the SMP bit consistent with the guest configuration */
95+ if (atomic_read(&vcpu->kvm->online_vcpus) > 1)
96+ actlr |= 1U << 6;
97+ else
98+ actlr &= ~(1U << 6);
99+
100+ vcpu->arch.cp15[c1_ACTLR] = actlr;
101+}
102+
103+/* A15 TRM 4.3.49: R/O WI (even if NSACR.NS_L2ERR, a write of 1 is ignored). */
104+static bool access_l2ectlr(struct kvm_vcpu *vcpu,
105+ const struct coproc_params *p,
106+ const struct coproc_reg *r)
107+{
108+ if (p->is_write)
109+ return ignore_write(vcpu, p);
110+
111+ *vcpu_reg(vcpu, p->Rt1) = 0;
112+ return true;
113+}
114+
115+/*
116+ * A15-specific CP15 registers.
117+ * Important: Must be sorted ascending by CRn, CRM, Op1, Op2
118+ */
119+static const struct coproc_reg a15_regs[] = {
120+ /* MPIDR: we use VMPIDR for guest access. */
121+ { CRn( 0), CRm( 0), Op1( 0), Op2( 5), is32,
122+ NULL, reset_mpidr, c0_MPIDR },
123+
124+ /* SCTLR: swapped by interrupt.S. */
125+ { CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
126+ NULL, reset_val, c1_SCTLR, 0x00C50078 },
127+ /* ACTLR: trapped by HCR.TAC bit. */
128+ { CRn( 1), CRm( 0), Op1( 0), Op2( 1), is32,
129+ access_actlr, reset_actlr, c1_ACTLR },
130+ /* CPACR: swapped by interrupt.S. */
131+ { CRn( 1), CRm( 0), Op1( 0), Op2( 2), is32,
132+ NULL, reset_val, c1_CPACR, 0x00000000 },
133+
134+ /*
135+ * L2CTLR access (guest wants to know #CPUs).
136+ */
137+ { CRn( 9), CRm( 0), Op1( 1), Op2( 2), is32,
138+ access_l2ctlr, reset_l2ctlr, c9_L2CTLR },
139+ { CRn( 9), CRm( 0), Op1( 1), Op2( 3), is32, access_l2ectlr},
140+
141+ /* The Configuration Base Address Register. */
142+ { CRn(15), CRm( 0), Op1( 4), Op2( 0), is32, access_cbar},
143+};
144+
145+static struct kvm_coproc_target_table a15_target_table = {
146+ .target = KVM_ARM_TARGET_CORTEX_A15,
147+ .table = a15_regs,
148+ .num = ARRAY_SIZE(a15_regs),
149+};
150+
151+static int __init coproc_a15_init(void)
152+{
153+ unsigned int i;
154+
155+ for (i = 1; i < ARRAY_SIZE(a15_regs); i++)
156+ BUG_ON(cmp_reg(&a15_regs[i-1],
157+ &a15_regs[i]) >= 0);
158+
159+ kvm_register_target_coproc_table(&a15_target_table);
160+ return 0;
161+}
162+late_initcall(coproc_a15_init);
arch/arm/kvm/emulate.c+373−0
@@ -0,0 +1,373 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/mm.h>
20+#include <linux/kvm_host.h>
21+#include <asm/kvm_arm.h>
22+#include <asm/kvm_emulate.h>
23+#include <trace/events/kvm.h>
24+
25+#include "trace.h"
26+
27+#define VCPU_NR_MODES 6
28+#define VCPU_REG_OFFSET_USR 0
29+#define VCPU_REG_OFFSET_FIQ 1
30+#define VCPU_REG_OFFSET_IRQ 2
31+#define VCPU_REG_OFFSET_SVC 3
32+#define VCPU_REG_OFFSET_ABT 4
33+#define VCPU_REG_OFFSET_UND 5
34+#define REG_OFFSET(_reg) \
35+ (offsetof(struct kvm_regs, _reg) / sizeof(u32))
36+
37+#define USR_REG_OFFSET(_num) REG_OFFSET(usr_regs.uregs[_num])
38+
39+static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][15] = {
40+ /* USR/SYS Registers */
41+ [VCPU_REG_OFFSET_USR] = {
42+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
43+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
44+ USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
45+ USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
46+ USR_REG_OFFSET(12), USR_REG_OFFSET(13), USR_REG_OFFSET(14),
47+ },
48+
49+ /* FIQ Registers */
50+ [VCPU_REG_OFFSET_FIQ] = {
51+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
52+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
53+ USR_REG_OFFSET(6), USR_REG_OFFSET(7),
54+ REG_OFFSET(fiq_regs[0]), /* r8 */
55+ REG_OFFSET(fiq_regs[1]), /* r9 */
56+ REG_OFFSET(fiq_regs[2]), /* r10 */
57+ REG_OFFSET(fiq_regs[3]), /* r11 */
58+ REG_OFFSET(fiq_regs[4]), /* r12 */
59+ REG_OFFSET(fiq_regs[5]), /* r13 */
60+ REG_OFFSET(fiq_regs[6]), /* r14 */
61+ },
62+
63+ /* IRQ Registers */
64+ [VCPU_REG_OFFSET_IRQ] = {
65+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
66+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
67+ USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
68+ USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
69+ USR_REG_OFFSET(12),
70+ REG_OFFSET(irq_regs[0]), /* r13 */
71+ REG_OFFSET(irq_regs[1]), /* r14 */
72+ },
73+
74+ /* SVC Registers */
75+ [VCPU_REG_OFFSET_SVC] = {
76+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
77+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
78+ USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
79+ USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
80+ USR_REG_OFFSET(12),
81+ REG_OFFSET(svc_regs[0]), /* r13 */
82+ REG_OFFSET(svc_regs[1]), /* r14 */
83+ },
84+
85+ /* ABT Registers */
86+ [VCPU_REG_OFFSET_ABT] = {
87+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
88+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
89+ USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
90+ USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
91+ USR_REG_OFFSET(12),
92+ REG_OFFSET(abt_regs[0]), /* r13 */
93+ REG_OFFSET(abt_regs[1]), /* r14 */
94+ },
95+
96+ /* UND Registers */
97+ [VCPU_REG_OFFSET_UND] = {
98+ USR_REG_OFFSET(0), USR_REG_OFFSET(1), USR_REG_OFFSET(2),
99+ USR_REG_OFFSET(3), USR_REG_OFFSET(4), USR_REG_OFFSET(5),
100+ USR_REG_OFFSET(6), USR_REG_OFFSET(7), USR_REG_OFFSET(8),
101+ USR_REG_OFFSET(9), USR_REG_OFFSET(10), USR_REG_OFFSET(11),
102+ USR_REG_OFFSET(12),
103+ REG_OFFSET(und_regs[0]), /* r13 */
104+ REG_OFFSET(und_regs[1]), /* r14 */
105+ },
106+};
107+
108+/*
109+ * Return a pointer to the register number valid in the current mode of
110+ * the virtual CPU.
111+ */
112+u32 *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num)
113+{
114+ u32 *reg_array = (u32 *)&vcpu->arch.regs;
115+ u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK;
116+
117+ switch (mode) {
118+ case USR_MODE...SVC_MODE:
119+ mode &= ~MODE32_BIT; /* 0 ... 3 */
120+ break;
121+
122+ case ABT_MODE:
123+ mode = VCPU_REG_OFFSET_ABT;
124+ break;
125+
126+ case UND_MODE:
127+ mode = VCPU_REG_OFFSET_UND;
128+ break;
129+
130+ case SYSTEM_MODE:
131+ mode = VCPU_REG_OFFSET_USR;
132+ break;
133+
134+ default:
135+ BUG();
136+ }
137+
138+ return reg_array + vcpu_reg_offsets[mode][reg_num];
139+}
140+
141+/*
142+ * Return the SPSR for the current mode of the virtual CPU.
143+ */
144+u32 *vcpu_spsr(struct kvm_vcpu *vcpu)
145+{
146+ u32 mode = *vcpu_cpsr(vcpu) & MODE_MASK;
147+ switch (mode) {
148+ case SVC_MODE:
149+ return &vcpu->arch.regs.KVM_ARM_SVC_spsr;
150+ case ABT_MODE:
151+ return &vcpu->arch.regs.KVM_ARM_ABT_spsr;
152+ case UND_MODE:
153+ return &vcpu->arch.regs.KVM_ARM_UND_spsr;
154+ case IRQ_MODE:
155+ return &vcpu->arch.regs.KVM_ARM_IRQ_spsr;
156+ case FIQ_MODE:
157+ return &vcpu->arch.regs.KVM_ARM_FIQ_spsr;
158+ default:
159+ BUG();
160+ }
161+}
162+
163+/**
164+ * kvm_handle_wfi - handle a wait-for-interrupts instruction executed by a guest
165+ * @vcpu: the vcpu pointer
166+ * @run: the kvm_run structure pointer
167+ *
168+ * Simply sets the wait_for_interrupts flag on the vcpu structure, which will
169+ * halt execution of world-switches and schedule other host processes until
170+ * there is an incoming IRQ or FIQ to the VM.
171+ */
172+int kvm_handle_wfi(struct kvm_vcpu *vcpu, struct kvm_run *run)
173+{
174+ trace_kvm_wfi(*vcpu_pc(vcpu));
175+ kvm_vcpu_block(vcpu);
176+ return 1;
177+}
178+
179+/**
180+ * adjust_itstate - adjust ITSTATE when emulating instructions in IT-block
181+ * @vcpu: The VCPU pointer
182+ *
183+ * When exceptions occur while instructions are executed in Thumb IF-THEN
184+ * blocks, the ITSTATE field of the CPSR is not advanved (updated), so we have
185+ * to do this little bit of work manually. The fields map like this:
186+ *
187+ * IT[7:0] -> CPSR[26:25],CPSR[15:10]
188+ */
189+static void kvm_adjust_itstate(struct kvm_vcpu *vcpu)
190+{
191+ unsigned long itbits, cond;
192+ unsigned long cpsr = *vcpu_cpsr(vcpu);
193+ bool is_arm = !(cpsr & PSR_T_BIT);
194+
195+ BUG_ON(is_arm && (cpsr & PSR_IT_MASK));
196+
197+ if (!(cpsr & PSR_IT_MASK))
198+ return;
199+
200+ cond = (cpsr & 0xe000) >> 13;
201+ itbits = (cpsr & 0x1c00) >> (10 - 2);
202+ itbits |= (cpsr & (0x3 << 25)) >> 25;
203+
204+ /* Perform ITAdvance (see page A-52 in ARM DDI 0406C) */
205+ if ((itbits & 0x7) == 0)
206+ itbits = cond = 0;
207+ else
208+ itbits = (itbits << 1) & 0x1f;
209+
210+ cpsr &= ~PSR_IT_MASK;
211+ cpsr |= cond << 13;
212+ cpsr |= (itbits & 0x1c) << (10 - 2);
213+ cpsr |= (itbits & 0x3) << 25;
214+ *vcpu_cpsr(vcpu) = cpsr;
215+}
216+
217+/**
218+ * kvm_skip_instr - skip a trapped instruction and proceed to the next
219+ * @vcpu: The vcpu pointer
220+ */
221+void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
222+{
223+ bool is_thumb;
224+
225+ is_thumb = !!(*vcpu_cpsr(vcpu) & PSR_T_BIT);
226+ if (is_thumb && !is_wide_instr)
227+ *vcpu_pc(vcpu) += 2;
228+ else
229+ *vcpu_pc(vcpu) += 4;
230+ kvm_adjust_itstate(vcpu);
231+}
232+
233+
234+/******************************************************************************
235+ * Inject exceptions into the guest
236+ */
237+
238+static u32 exc_vector_base(struct kvm_vcpu *vcpu)
239+{
240+ u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
241+ u32 vbar = vcpu->arch.cp15[c12_VBAR];
242+
243+ if (sctlr & SCTLR_V)
244+ return 0xffff0000;
245+ else /* always have security exceptions */
246+ return vbar;
247+}
248+
249+/**
250+ * kvm_inject_undefined - inject an undefined exception into the guest
251+ * @vcpu: The VCPU to receive the undefined exception
252+ *
253+ * It is assumed that this code is called from the VCPU thread and that the
254+ * VCPU therefore is not currently executing guest code.
255+ *
256+ * Modelled after TakeUndefInstrException() pseudocode.
257+ */
258+void kvm_inject_undefined(struct kvm_vcpu *vcpu)
259+{
260+ u32 new_lr_value;
261+ u32 new_spsr_value;
262+ u32 cpsr = *vcpu_cpsr(vcpu);
263+ u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
264+ bool is_thumb = (cpsr & PSR_T_BIT);
265+ u32 vect_offset = 4;
266+ u32 return_offset = (is_thumb) ? 2 : 4;
267+
268+ new_spsr_value = cpsr;
269+ new_lr_value = *vcpu_pc(vcpu) - return_offset;
270+
271+ *vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | UND_MODE;
272+ *vcpu_cpsr(vcpu) |= PSR_I_BIT;
273+ *vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
274+
275+ if (sctlr & SCTLR_TE)
276+ *vcpu_cpsr(vcpu) |= PSR_T_BIT;
277+ if (sctlr & SCTLR_EE)
278+ *vcpu_cpsr(vcpu) |= PSR_E_BIT;
279+
280+ /* Note: These now point to UND banked copies */
281+ *vcpu_spsr(vcpu) = cpsr;
282+ *vcpu_reg(vcpu, 14) = new_lr_value;
283+
284+ /* Branch to exception vector */
285+ *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset;
286+}
287+
288+/*
289+ * Modelled after TakeDataAbortException() and TakePrefetchAbortException
290+ * pseudocode.
291+ */
292+static void inject_abt(struct kvm_vcpu *vcpu, bool is_pabt, unsigned long addr)
293+{
294+ u32 new_lr_value;
295+ u32 new_spsr_value;
296+ u32 cpsr = *vcpu_cpsr(vcpu);
297+ u32 sctlr = vcpu->arch.cp15[c1_SCTLR];
298+ bool is_thumb = (cpsr & PSR_T_BIT);
299+ u32 vect_offset;
300+ u32 return_offset = (is_thumb) ? 4 : 0;
301+ bool is_lpae;
302+
303+ new_spsr_value = cpsr;
304+ new_lr_value = *vcpu_pc(vcpu) + return_offset;
305+
306+ *vcpu_cpsr(vcpu) = (cpsr & ~MODE_MASK) | ABT_MODE;
307+ *vcpu_cpsr(vcpu) |= PSR_I_BIT | PSR_A_BIT;
308+ *vcpu_cpsr(vcpu) &= ~(PSR_IT_MASK | PSR_J_BIT | PSR_E_BIT | PSR_T_BIT);
309+
310+ if (sctlr & SCTLR_TE)
311+ *vcpu_cpsr(vcpu) |= PSR_T_BIT;
312+ if (sctlr & SCTLR_EE)
313+ *vcpu_cpsr(vcpu) |= PSR_E_BIT;
314+
315+ /* Note: These now point to ABT banked copies */
316+ *vcpu_spsr(vcpu) = cpsr;
317+ *vcpu_reg(vcpu, 14) = new_lr_value;
318+
319+ if (is_pabt)
320+ vect_offset = 12;
321+ else
322+ vect_offset = 16;
323+
324+ /* Branch to exception vector */
325+ *vcpu_pc(vcpu) = exc_vector_base(vcpu) + vect_offset;
326+
327+ if (is_pabt) {
328+ /* Set DFAR and DFSR */
329+ vcpu->arch.cp15[c6_IFAR] = addr;
330+ is_lpae = (vcpu->arch.cp15[c2_TTBCR] >> 31);
331+ /* Always give debug fault for now - should give guest a clue */
332+ if (is_lpae)
333+ vcpu->arch.cp15[c5_IFSR] = 1 << 9 | 0x22;
334+ else
335+ vcpu->arch.cp15[c5_IFSR] = 2;
336+ } else { /* !iabt */
337+ /* Set DFAR and DFSR */
338+ vcpu->arch.cp15[c6_DFAR] = addr;
339+ is_lpae = (vcpu->arch.cp15[c2_TTBCR] >> 31);
340+ /* Always give debug fault for now - should give guest a clue */
341+ if (is_lpae)
342+ vcpu->arch.cp15[c5_DFSR] = 1 << 9 | 0x22;
343+ else
344+ vcpu->arch.cp15[c5_DFSR] = 2;
345+ }
346+
347+}
348+
349+/**
350+ * kvm_inject_dabt - inject a data abort into the guest
351+ * @vcpu: The VCPU to receive the undefined exception
352+ * @addr: The address to report in the DFAR
353+ *
354+ * It is assumed that this code is called from the VCPU thread and that the
355+ * VCPU therefore is not currently executing guest code.
356+ */
357+void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
358+{
359+ inject_abt(vcpu, false, addr);
360+}
361+
362+/**
363+ * kvm_inject_pabt - inject a prefetch abort into the guest
364+ * @vcpu: The VCPU to receive the undefined exception
365+ * @addr: The address to report in the DFAR
366+ *
367+ * It is assumed that this code is called from the VCPU thread and that the
368+ * VCPU therefore is not currently executing guest code.
369+ */
370+void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
371+{
372+ inject_abt(vcpu, true, addr);
373+}
arch/arm/kvm/guest.c+222−0
@@ -0,0 +1,222 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/errno.h>
20+#include <linux/err.h>
21+#include <linux/kvm_host.h>
22+#include <linux/module.h>
23+#include <linux/vmalloc.h>
24+#include <linux/fs.h>
25+#include <asm/uaccess.h>
26+#include <asm/kvm.h>
27+#include <asm/kvm_asm.h>
28+#include <asm/kvm_emulate.h>
29+#include <asm/kvm_coproc.h>
30+
31+#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM }
32+#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU }
33+
34+struct kvm_stats_debugfs_item debugfs_entries[] = {
35+ { NULL }
36+};
37+
38+int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
39+{
40+ return 0;
41+}
42+
43+static u64 core_reg_offset_from_id(u64 id)
44+{
45+ return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
46+}
47+
48+static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
49+{
50+ u32 __user *uaddr = (u32 __user *)(long)reg->addr;
51+ struct kvm_regs *regs = &vcpu->arch.regs;
52+ u64 off;
53+
54+ if (KVM_REG_SIZE(reg->id) != 4)
55+ return -ENOENT;
56+
57+ /* Our ID is an index into the kvm_regs struct. */
58+ off = core_reg_offset_from_id(reg->id);
59+ if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id))
60+ return -ENOENT;
61+
62+ return put_user(((u32 *)regs)[off], uaddr);
63+}
64+
65+static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
66+{
67+ u32 __user *uaddr = (u32 __user *)(long)reg->addr;
68+ struct kvm_regs *regs = &vcpu->arch.regs;
69+ u64 off, val;
70+
71+ if (KVM_REG_SIZE(reg->id) != 4)
72+ return -ENOENT;
73+
74+ /* Our ID is an index into the kvm_regs struct. */
75+ off = core_reg_offset_from_id(reg->id);
76+ if (off >= sizeof(*regs) / KVM_REG_SIZE(reg->id))
77+ return -ENOENT;
78+
79+ if (get_user(val, uaddr) != 0)
80+ return -EFAULT;
81+
82+ if (off == KVM_REG_ARM_CORE_REG(usr_regs.ARM_cpsr)) {
83+ unsigned long mode = val & MODE_MASK;
84+ switch (mode) {
85+ case USR_MODE:
86+ case FIQ_MODE:
87+ case IRQ_MODE:
88+ case SVC_MODE:
89+ case ABT_MODE:
90+ case UND_MODE:
91+ break;
92+ default:
93+ return -EINVAL;
94+ }
95+ }
96+
97+ ((u32 *)regs)[off] = val;
98+ return 0;
99+}
100+
101+int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
102+{
103+ return -EINVAL;
104+}
105+
106+int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
107+{
108+ return -EINVAL;
109+}
110+
111+static unsigned long num_core_regs(void)
112+{
113+ return sizeof(struct kvm_regs) / sizeof(u32);
114+}
115+
116+/**
117+ * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
118+ *
119+ * This is for all registers.
120+ */
121+unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
122+{
123+ return num_core_regs() + kvm_arm_num_coproc_regs(vcpu);
124+}
125+
126+/**
127+ * kvm_arm_copy_reg_indices - get indices of all registers.
128+ *
129+ * We do core registers right here, then we apppend coproc regs.
130+ */
131+int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
132+{
133+ unsigned int i;
134+ const u64 core_reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE;
135+
136+ for (i = 0; i < sizeof(struct kvm_regs)/sizeof(u32); i++) {
137+ if (put_user(core_reg | i, uindices))
138+ return -EFAULT;
139+ uindices++;
140+ }
141+
142+ return kvm_arm_copy_coproc_indices(vcpu, uindices);
143+}
144+
145+int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
146+{
147+ /* We currently use nothing arch-specific in upper 32 bits */
148+ if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32)
149+ return -EINVAL;
150+
151+ /* Register group 16 means we want a core register. */
152+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
153+ return get_core_reg(vcpu, reg);
154+
155+ return kvm_arm_coproc_get_reg(vcpu, reg);
156+}
157+
158+int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
159+{
160+ /* We currently use nothing arch-specific in upper 32 bits */
161+ if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM >> 32)
162+ return -EINVAL;
163+
164+ /* Register group 16 means we set a core register. */
165+ if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_CORE)
166+ return set_core_reg(vcpu, reg);
167+
168+ return kvm_arm_coproc_set_reg(vcpu, reg);
169+}
170+
171+int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
172+ struct kvm_sregs *sregs)
173+{
174+ return -EINVAL;
175+}
176+
177+int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
178+ struct kvm_sregs *sregs)
179+{
180+ return -EINVAL;
181+}
182+
183+int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
184+ const struct kvm_vcpu_init *init)
185+{
186+ unsigned int i;
187+
188+ /* We can only do a cortex A15 for now. */
189+ if (init->target != kvm_target_cpu())
190+ return -EINVAL;
191+
192+ vcpu->arch.target = init->target;
193+ bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
194+
195+ /* -ENOENT for unknown features, -EINVAL for invalid combinations. */
196+ for (i = 0; i < sizeof(init->features) * 8; i++) {
197+ if (test_bit(i, (void *)init->features)) {
198+ if (i >= KVM_VCPU_MAX_FEATURES)
199+ return -ENOENT;
200+ set_bit(i, vcpu->arch.features);
201+ }
202+ }
203+
204+ /* Now we know what it is, we can reset it. */
205+ return kvm_reset_vcpu(vcpu);
206+}
207+
208+int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
209+{
210+ return -EINVAL;
211+}
212+
213+int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
214+{
215+ return -EINVAL;
216+}
217+
218+int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
219+ struct kvm_translation *tr)
220+{
221+ return -EINVAL;
222+}
arch/arm/kvm/init.S+114−0
@@ -0,0 +1,114 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/linkage.h>
20+#include <asm/unified.h>
21+#include <asm/asm-offsets.h>
22+#include <asm/kvm_asm.h>
23+#include <asm/kvm_arm.h>
24+
25+/********************************************************************
26+ * Hypervisor initialization
27+ * - should be called with:
28+ * r0,r1 = Hypervisor pgd pointer
29+ * r2 = top of Hyp stack (kernel VA)
30+ * r3 = pointer to hyp vectors
31+ */
32+
33+ .text
34+ .pushsection .hyp.idmap.text,"ax"
35+ .align 5
36+__kvm_hyp_init:
37+ .globl __kvm_hyp_init
38+
39+ @ Hyp-mode exception vector
40+ W(b) .
41+ W(b) .
42+ W(b) .
43+ W(b) .
44+ W(b) .
45+ W(b) __do_hyp_init
46+ W(b) .
47+ W(b) .
48+
49+__do_hyp_init:
50+ @ Set the HTTBR to point to the hypervisor PGD pointer passed
51+ mcrr p15, 4, r0, r1, c2
52+
53+ @ Set the HTCR and VTCR to the same shareability and cacheability
54+ @ settings as the non-secure TTBCR and with T0SZ == 0.
55+ mrc p15, 4, r0, c2, c0, 2 @ HTCR
56+ ldr r12, =HTCR_MASK
57+ bic r0, r0, r12
58+ mrc p15, 0, r1, c2, c0, 2 @ TTBCR
59+ and r1, r1, #(HTCR_MASK & ~TTBCR_T0SZ)
60+ orr r0, r0, r1
61+ mcr p15, 4, r0, c2, c0, 2 @ HTCR
62+
63+ mrc p15, 4, r1, c2, c1, 2 @ VTCR
64+ ldr r12, =VTCR_MASK
65+ bic r1, r1, r12
66+ bic r0, r0, #(~VTCR_HTCR_SH) @ clear non-reusable HTCR bits
67+ orr r1, r0, r1
68+ orr r1, r1, #(KVM_VTCR_SL0 | KVM_VTCR_T0SZ | KVM_VTCR_S)
69+ mcr p15, 4, r1, c2, c1, 2 @ VTCR
70+
71+ @ Use the same memory attributes for hyp. accesses as the kernel
72+ @ (copy MAIRx ro HMAIRx).
73+ mrc p15, 0, r0, c10, c2, 0
74+ mcr p15, 4, r0, c10, c2, 0
75+ mrc p15, 0, r0, c10, c2, 1
76+ mcr p15, 4, r0, c10, c2, 1
77+
78+ @ Set the HSCTLR to:
79+ @ - ARM/THUMB exceptions: Kernel config (Thumb-2 kernel)
80+ @ - Endianness: Kernel config
81+ @ - Fast Interrupt Features: Kernel config
82+ @ - Write permission implies XN: disabled
83+ @ - Instruction cache: enabled
84+ @ - Data/Unified cache: enabled
85+ @ - Memory alignment checks: enabled
86+ @ - MMU: enabled (this code must be run from an identity mapping)
87+ mrc p15, 4, r0, c1, c0, 0 @ HSCR
88+ ldr r12, =HSCTLR_MASK
89+ bic r0, r0, r12
90+ mrc p15, 0, r1, c1, c0, 0 @ SCTLR
91+ ldr r12, =(HSCTLR_EE | HSCTLR_FI | HSCTLR_I | HSCTLR_C)
92+ and r1, r1, r12
93+ ARM( ldr r12, =(HSCTLR_M | HSCTLR_A) )
94+ THUMB( ldr r12, =(HSCTLR_M | HSCTLR_A | HSCTLR_TE) )
95+ orr r1, r1, r12
96+ orr r0, r0, r1
97+ isb
98+ mcr p15, 4, r0, c1, c0, 0 @ HSCR
99+ isb
100+
101+ @ Set stack pointer and return to the kernel
102+ mov sp, r2
103+
104+ @ Set HVBAR to point to the HYP vectors
105+ mcr p15, 4, r3, c12, c0, 0 @ HVBAR
106+
107+ eret
108+
109+ .ltorg
110+
111+ .globl __kvm_hyp_init_end
112+__kvm_hyp_init_end:
113+
114+ .popsection
arch/arm/kvm/interrupts.S+478−0
@@ -0,0 +1,478 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/linkage.h>
20+#include <linux/const.h>
21+#include <asm/unified.h>
22+#include <asm/page.h>
23+#include <asm/ptrace.h>
24+#include <asm/asm-offsets.h>
25+#include <asm/kvm_asm.h>
26+#include <asm/kvm_arm.h>
27+#include <asm/vfpmacros.h>
28+#include "interrupts_head.S"
29+
30+ .text
31+
32+__kvm_hyp_code_start:
33+ .globl __kvm_hyp_code_start
34+
35+/********************************************************************
36+ * Flush per-VMID TLBs
37+ *
38+ * void __kvm_tlb_flush_vmid(struct kvm *kvm);
39+ *
40+ * We rely on the hardware to broadcast the TLB invalidation to all CPUs
41+ * inside the inner-shareable domain (which is the case for all v7
42+ * implementations). If we come across a non-IS SMP implementation, we'll
43+ * have to use an IPI based mechanism. Until then, we stick to the simple
44+ * hardware assisted version.
45+ */
46+ENTRY(__kvm_tlb_flush_vmid)
47+ push {r2, r3}
48+
49+ add r0, r0, #KVM_VTTBR
50+ ldrd r2, r3, [r0]
51+ mcrr p15, 6, r2, r3, c2 @ Write VTTBR
52+ isb
53+ mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
54+ dsb
55+ isb
56+ mov r2, #0
57+ mov r3, #0
58+ mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
59+ isb @ Not necessary if followed by eret
60+
61+ pop {r2, r3}
62+ bx lr
63+ENDPROC(__kvm_tlb_flush_vmid)
64+
65+/********************************************************************
66+ * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
67+ * domain, for all VMIDs
68+ *
69+ * void __kvm_flush_vm_context(void);
70+ */
71+ENTRY(__kvm_flush_vm_context)
72+ mov r0, #0 @ rn parameter for c15 flushes is SBZ
73+
74+ /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
75+ mcr p15, 4, r0, c8, c3, 4
76+ /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
77+ mcr p15, 0, r0, c7, c1, 0
78+ dsb
79+ isb @ Not necessary if followed by eret
80+
81+ bx lr
82+ENDPROC(__kvm_flush_vm_context)
83+
84+
85+/********************************************************************
86+ * Hypervisor world-switch code
87+ *
88+ *
89+ * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
90+ */
91+ENTRY(__kvm_vcpu_run)
92+ @ Save the vcpu pointer
93+ mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
94+
95+ save_host_regs
96+
97+ @ Store hardware CP15 state and load guest state
98+ read_cp15_state store_to_vcpu = 0
99+ write_cp15_state read_from_vcpu = 1
100+
101+ @ If the host kernel has not been configured with VFPv3 support,
102+ @ then it is safer if we deny guests from using it as well.
103+#ifdef CONFIG_VFPv3
104+ @ Set FPEXC_EN so the guest doesn't trap floating point instructions
105+ VFPFMRX r2, FPEXC @ VMRS
106+ push {r2}
107+ orr r2, r2, #FPEXC_EN
108+ VFPFMXR FPEXC, r2 @ VMSR
109+#endif
110+
111+ @ Configure Hyp-role
112+ configure_hyp_role vmentry
113+
114+ @ Trap coprocessor CRx accesses
115+ set_hstr vmentry
116+ set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
117+ set_hdcr vmentry
118+
119+ @ Write configured ID register into MIDR alias
120+ ldr r1, [vcpu, #VCPU_MIDR]
121+ mcr p15, 4, r1, c0, c0, 0
122+
123+ @ Write guest view of MPIDR into VMPIDR
124+ ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
125+ mcr p15, 4, r1, c0, c0, 5
126+
127+ @ Set up guest memory translation
128+ ldr r1, [vcpu, #VCPU_KVM]
129+ add r1, r1, #KVM_VTTBR
130+ ldrd r2, r3, [r1]
131+ mcrr p15, 6, r2, r3, c2 @ Write VTTBR
132+
133+ @ We're all done, just restore the GPRs and go to the guest
134+ restore_guest_regs
135+ clrex @ Clear exclusive monitor
136+ eret
137+
138+__kvm_vcpu_return:
139+ /*
140+ * return convention:
141+ * guest r0, r1, r2 saved on the stack
142+ * r0: vcpu pointer
143+ * r1: exception code
144+ */
145+ save_guest_regs
146+
147+ @ Set VMID == 0
148+ mov r2, #0
149+ mov r3, #0
150+ mcrr p15, 6, r2, r3, c2 @ Write VTTBR
151+
152+ @ Don't trap coprocessor accesses for host kernel
153+ set_hstr vmexit
154+ set_hdcr vmexit
155+ set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
156+
157+#ifdef CONFIG_VFPv3
158+ @ Save floating point registers we if let guest use them.
159+ tst r2, #(HCPTR_TCP(10) | HCPTR_TCP(11))
160+ bne after_vfp_restore
161+
162+ @ Switch VFP/NEON hardware state to the host's
163+ add r7, vcpu, #VCPU_VFP_GUEST
164+ store_vfp_state r7
165+ add r7, vcpu, #VCPU_VFP_HOST
166+ ldr r7, [r7]
167+ restore_vfp_state r7
168+
169+after_vfp_restore:
170+ @ Restore FPEXC_EN which we clobbered on entry
171+ pop {r2}
172+ VFPFMXR FPEXC, r2
173+#endif
174+
175+ @ Reset Hyp-role
176+ configure_hyp_role vmexit
177+
178+ @ Let host read hardware MIDR
179+ mrc p15, 0, r2, c0, c0, 0
180+ mcr p15, 4, r2, c0, c0, 0
181+
182+ @ Back to hardware MPIDR
183+ mrc p15, 0, r2, c0, c0, 5
184+ mcr p15, 4, r2, c0, c0, 5
185+
186+ @ Store guest CP15 state and restore host state
187+ read_cp15_state store_to_vcpu = 1
188+ write_cp15_state read_from_vcpu = 0
189+
190+ restore_host_regs
191+ clrex @ Clear exclusive monitor
192+ mov r0, r1 @ Return the return code
193+ mov r1, #0 @ Clear upper bits in return value
194+ bx lr @ return to IOCTL
195+
196+/********************************************************************
197+ * Call function in Hyp mode
198+ *
199+ *
200+ * u64 kvm_call_hyp(void *hypfn, ...);
201+ *
202+ * This is not really a variadic function in the classic C-way and care must
203+ * be taken when calling this to ensure parameters are passed in registers
204+ * only, since the stack will change between the caller and the callee.
205+ *
206+ * Call the function with the first argument containing a pointer to the
207+ * function you wish to call in Hyp mode, and subsequent arguments will be
208+ * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
209+ * function pointer can be passed). The function being called must be mapped
210+ * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
211+ * passed in r0 and r1.
212+ *
213+ * The calling convention follows the standard AAPCS:
214+ * r0 - r3: caller save
215+ * r12: caller save
216+ * rest: callee save
217+ */
218+ENTRY(kvm_call_hyp)
219+ hvc #0
220+ bx lr
221+
222+/********************************************************************
223+ * Hypervisor exception vector and handlers
224+ *
225+ *
226+ * The KVM/ARM Hypervisor ABI is defined as follows:
227+ *
228+ * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
229+ * instruction is issued since all traps are disabled when running the host
230+ * kernel as per the Hyp-mode initialization at boot time.
231+ *
232+ * HVC instructions cause a trap to the vector page + offset 0x18 (see hyp_hvc
233+ * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
234+ * host kernel) and they cause a trap to the vector page + offset 0xc when HVC
235+ * instructions are called from within Hyp-mode.
236+ *
237+ * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
238+ * Switching to Hyp mode is done through a simple HVC #0 instruction. The
239+ * exception vector code will check that the HVC comes from VMID==0 and if
240+ * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
241+ * - r0 contains a pointer to a HYP function
242+ * - r1, r2, and r3 contain arguments to the above function.
243+ * - The HYP function will be called with its arguments in r0, r1 and r2.
244+ * On HYP function return, we return directly to SVC.
245+ *
246+ * Note that the above is used to execute code in Hyp-mode from a host-kernel
247+ * point of view, and is a different concept from performing a world-switch and
248+ * executing guest code SVC mode (with a VMID != 0).
249+ */
250+
251+/* Handle undef, svc, pabt, or dabt by crashing with a user notice */
252+.macro bad_exception exception_code, panic_str
253+ push {r0-r2}
254+ mrrc p15, 6, r0, r1, c2 @ Read VTTBR
255+ lsr r1, r1, #16
256+ ands r1, r1, #0xff
257+ beq 99f
258+
259+ load_vcpu @ Load VCPU pointer
260+ .if \exception_code == ARM_EXCEPTION_DATA_ABORT
261+ mrc p15, 4, r2, c5, c2, 0 @ HSR
262+ mrc p15, 4, r1, c6, c0, 0 @ HDFAR
263+ str r2, [vcpu, #VCPU_HSR]
264+ str r1, [vcpu, #VCPU_HxFAR]
265+ .endif
266+ .if \exception_code == ARM_EXCEPTION_PREF_ABORT
267+ mrc p15, 4, r2, c5, c2, 0 @ HSR
268+ mrc p15, 4, r1, c6, c0, 2 @ HIFAR
269+ str r2, [vcpu, #VCPU_HSR]
270+ str r1, [vcpu, #VCPU_HxFAR]
271+ .endif
272+ mov r1, #\exception_code
273+ b __kvm_vcpu_return
274+
275+ @ We were in the host already. Let's craft a panic-ing return to SVC.
276+99: mrs r2, cpsr
277+ bic r2, r2, #MODE_MASK
278+ orr r2, r2, #SVC_MODE
279+THUMB( orr r2, r2, #PSR_T_BIT )
280+ msr spsr_cxsf, r2
281+ mrs r1, ELR_hyp
282+ ldr r2, =BSYM(panic)
283+ msr ELR_hyp, r2
284+ ldr r0, =\panic_str
285+ eret
286+.endm
287+
288+ .text
289+
290+ .align 5
291+__kvm_hyp_vector:
292+ .globl __kvm_hyp_vector
293+
294+ @ Hyp-mode exception vector
295+ W(b) hyp_reset
296+ W(b) hyp_undef
297+ W(b) hyp_svc
298+ W(b) hyp_pabt
299+ W(b) hyp_dabt
300+ W(b) hyp_hvc
301+ W(b) hyp_irq
302+ W(b) hyp_fiq
303+
304+ .align
305+hyp_reset:
306+ b hyp_reset
307+
308+ .align
309+hyp_undef:
310+ bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
311+
312+ .align
313+hyp_svc:
314+ bad_exception ARM_EXCEPTION_HVC, svc_die_str
315+
316+ .align
317+hyp_pabt:
318+ bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
319+
320+ .align
321+hyp_dabt:
322+ bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
323+
324+ .align
325+hyp_hvc:
326+ /*
327+ * Getting here is either becuase of a trap from a guest or from calling
328+ * HVC from the host kernel, which means "switch to Hyp mode".
329+ */
330+ push {r0, r1, r2}
331+
332+ @ Check syndrome register
333+ mrc p15, 4, r1, c5, c2, 0 @ HSR
334+ lsr r0, r1, #HSR_EC_SHIFT
335+#ifdef CONFIG_VFPv3
336+ cmp r0, #HSR_EC_CP_0_13
337+ beq switch_to_guest_vfp
338+#endif
339+ cmp r0, #HSR_EC_HVC
340+ bne guest_trap @ Not HVC instr.
341+
342+ /*
343+ * Let's check if the HVC came from VMID 0 and allow simple
344+ * switch to Hyp mode
345+ */
346+ mrrc p15, 6, r0, r2, c2
347+ lsr r2, r2, #16
348+ and r2, r2, #0xff
349+ cmp r2, #0
350+ bne guest_trap @ Guest called HVC
351+
352+host_switch_to_hyp:
353+ pop {r0, r1, r2}
354+
355+ push {lr}
356+ mrs lr, SPSR
357+ push {lr}
358+
359+ mov lr, r0
360+ mov r0, r1
361+ mov r1, r2
362+ mov r2, r3
363+
364+THUMB( orr lr, #1)
365+ blx lr @ Call the HYP function
366+
367+ pop {lr}
368+ msr SPSR_csxf, lr
369+ pop {lr}
370+ eret
371+
372+guest_trap:
373+ load_vcpu @ Load VCPU pointer to r0
374+ str r1, [vcpu, #VCPU_HSR]
375+
376+ @ Check if we need the fault information
377+ lsr r1, r1, #HSR_EC_SHIFT
378+ cmp r1, #HSR_EC_IABT
379+ mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
380+ beq 2f
381+ cmp r1, #HSR_EC_DABT
382+ bne 1f
383+ mrc p15, 4, r2, c6, c0, 0 @ HDFAR
384+
385+2: str r2, [vcpu, #VCPU_HxFAR]
386+
387+ /*
388+ * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
389+ *
390+ * Abort on the stage 2 translation for a memory access from a
391+ * Non-secure PL1 or PL0 mode:
392+ *
393+ * For any Access flag fault or Translation fault, and also for any
394+ * Permission fault on the stage 2 translation of a memory access
395+ * made as part of a translation table walk for a stage 1 translation,
396+ * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
397+ * is UNKNOWN.
398+ */
399+
400+ /* Check for permission fault, and S1PTW */
401+ mrc p15, 4, r1, c5, c2, 0 @ HSR
402+ and r0, r1, #HSR_FSC_TYPE
403+ cmp r0, #FSC_PERM
404+ tsteq r1, #(1 << 7) @ S1PTW
405+ mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
406+ bne 3f
407+
408+ /* Resolve IPA using the xFAR */
409+ mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
410+ isb
411+ mrrc p15, 0, r0, r1, c7 @ PAR
412+ tst r0, #1
413+ bne 4f @ Failed translation
414+ ubfx r2, r0, #12, #20
415+ lsl r2, r2, #4
416+ orr r2, r2, r1, lsl #24
417+
418+3: load_vcpu @ Load VCPU pointer to r0
419+ str r2, [r0, #VCPU_HPFAR]
420+
421+1: mov r1, #ARM_EXCEPTION_HVC
422+ b __kvm_vcpu_return
423+
424+4: pop {r0, r1, r2} @ Failed translation, return to guest
425+ eret
426+
427+/*
428+ * If VFPv3 support is not available, then we will not switch the VFP
429+ * registers; however cp10 and cp11 accesses will still trap and fallback
430+ * to the regular coprocessor emulation code, which currently will
431+ * inject an undefined exception to the guest.
432+ */
433+#ifdef CONFIG_VFPv3
434+switch_to_guest_vfp:
435+ load_vcpu @ Load VCPU pointer to r0
436+ push {r3-r7}
437+
438+ @ NEON/VFP used. Turn on VFP access.
439+ set_hcptr vmexit, (HCPTR_TCP(10) | HCPTR_TCP(11))
440+
441+ @ Switch VFP/NEON hardware state to the guest's
442+ add r7, r0, #VCPU_VFP_HOST
443+ ldr r7, [r7]
444+ store_vfp_state r7
445+ add r7, r0, #VCPU_VFP_GUEST
446+ restore_vfp_state r7
447+
448+ pop {r3-r7}
449+ pop {r0-r2}
450+ eret
451+#endif
452+
453+ .align
454+hyp_irq:
455+ push {r0, r1, r2}
456+ mov r1, #ARM_EXCEPTION_IRQ
457+ load_vcpu @ Load VCPU pointer to r0
458+ b __kvm_vcpu_return
459+
460+ .align
461+hyp_fiq:
462+ b hyp_fiq
463+
464+ .ltorg
465+
466+__kvm_hyp_code_end:
467+ .globl __kvm_hyp_code_end
468+
469+ .section ".rodata"
470+
471+und_die_str:
472+ .ascii "unexpected undefined exception in Hyp mode at: %#08x"
473+pabt_die_str:
474+ .ascii "unexpected prefetch abort in Hyp mode at: %#08x"
475+dabt_die_str:
476+ .ascii "unexpected data abort in Hyp mode at: %#08x"
477+svc_die_str:
478+ .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x"
arch/arm/kvm/interrupts_head.S+441−0
@@ -0,0 +1,441 @@
1+#define VCPU_USR_REG(_reg_nr) (VCPU_USR_REGS + (_reg_nr * 4))
2+#define VCPU_USR_SP (VCPU_USR_REG(13))
3+#define VCPU_USR_LR (VCPU_USR_REG(14))
4+#define CP15_OFFSET(_cp15_reg_idx) (VCPU_CP15 + (_cp15_reg_idx * 4))
5+
6+/*
7+ * Many of these macros need to access the VCPU structure, which is always
8+ * held in r0. These macros should never clobber r1, as it is used to hold the
9+ * exception code on the return path (except of course the macro that switches
10+ * all the registers before the final jump to the VM).
11+ */
12+vcpu .req r0 @ vcpu pointer always in r0
13+
14+/* Clobbers {r2-r6} */
15+.macro store_vfp_state vfp_base
16+ @ The VFPFMRX and VFPFMXR macros are the VMRS and VMSR instructions
17+ VFPFMRX r2, FPEXC
18+ @ Make sure VFP is enabled so we can touch the registers.
19+ orr r6, r2, #FPEXC_EN
20+ VFPFMXR FPEXC, r6
21+
22+ VFPFMRX r3, FPSCR
23+ tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
24+ beq 1f
25+ @ If FPEXC_EX is 0, then FPINST/FPINST2 reads are upredictable, so
26+ @ we only need to save them if FPEXC_EX is set.
27+ VFPFMRX r4, FPINST
28+ tst r2, #FPEXC_FP2V
29+ VFPFMRX r5, FPINST2, ne @ vmrsne
30+ bic r6, r2, #FPEXC_EX @ FPEXC_EX disable
31+ VFPFMXR FPEXC, r6
32+1:
33+ VFPFSTMIA \vfp_base, r6 @ Save VFP registers
34+ stm \vfp_base, {r2-r5} @ Save FPEXC, FPSCR, FPINST, FPINST2
35+.endm
36+
37+/* Assume FPEXC_EN is on and FPEXC_EX is off, clobbers {r2-r6} */
38+.macro restore_vfp_state vfp_base
39+ VFPFLDMIA \vfp_base, r6 @ Load VFP registers
40+ ldm \vfp_base, {r2-r5} @ Load FPEXC, FPSCR, FPINST, FPINST2
41+
42+ VFPFMXR FPSCR, r3
43+ tst r2, #FPEXC_EX @ Check for VFP Subarchitecture
44+ beq 1f
45+ VFPFMXR FPINST, r4
46+ tst r2, #FPEXC_FP2V
47+ VFPFMXR FPINST2, r5, ne
48+1:
49+ VFPFMXR FPEXC, r2 @ FPEXC (last, in case !EN)
50+.endm
51+
52+/* These are simply for the macros to work - value don't have meaning */
53+.equ usr, 0
54+.equ svc, 1
55+.equ abt, 2
56+.equ und, 3
57+.equ irq, 4
58+.equ fiq, 5
59+
60+.macro push_host_regs_mode mode
61+ mrs r2, SP_\mode
62+ mrs r3, LR_\mode
63+ mrs r4, SPSR_\mode
64+ push {r2, r3, r4}
65+.endm
66+
67+/*
68+ * Store all host persistent registers on the stack.
69+ * Clobbers all registers, in all modes, except r0 and r1.
70+ */
71+.macro save_host_regs
72+ /* Hyp regs. Only ELR_hyp (SPSR_hyp already saved) */
73+ mrs r2, ELR_hyp
74+ push {r2}
75+
76+ /* usr regs */
77+ push {r4-r12} @ r0-r3 are always clobbered
78+ mrs r2, SP_usr
79+ mov r3, lr
80+ push {r2, r3}
81+
82+ push_host_regs_mode svc
83+ push_host_regs_mode abt
84+ push_host_regs_mode und
85+ push_host_regs_mode irq
86+
87+ /* fiq regs */
88+ mrs r2, r8_fiq
89+ mrs r3, r9_fiq
90+ mrs r4, r10_fiq
91+ mrs r5, r11_fiq
92+ mrs r6, r12_fiq
93+ mrs r7, SP_fiq
94+ mrs r8, LR_fiq
95+ mrs r9, SPSR_fiq
96+ push {r2-r9}
97+.endm
98+
99+.macro pop_host_regs_mode mode
100+ pop {r2, r3, r4}
101+ msr SP_\mode, r2
102+ msr LR_\mode, r3
103+ msr SPSR_\mode, r4
104+.endm
105+
106+/*
107+ * Restore all host registers from the stack.
108+ * Clobbers all registers, in all modes, except r0 and r1.
109+ */
110+.macro restore_host_regs
111+ pop {r2-r9}
112+ msr r8_fiq, r2
113+ msr r9_fiq, r3
114+ msr r10_fiq, r4
115+ msr r11_fiq, r5
116+ msr r12_fiq, r6
117+ msr SP_fiq, r7
118+ msr LR_fiq, r8
119+ msr SPSR_fiq, r9
120+
121+ pop_host_regs_mode irq
122+ pop_host_regs_mode und
123+ pop_host_regs_mode abt
124+ pop_host_regs_mode svc
125+
126+ pop {r2, r3}
127+ msr SP_usr, r2
128+ mov lr, r3
129+ pop {r4-r12}
130+
131+ pop {r2}
132+ msr ELR_hyp, r2
133+.endm
134+
135+/*
136+ * Restore SP, LR and SPSR for a given mode. offset is the offset of
137+ * this mode's registers from the VCPU base.
138+ *
139+ * Assumes vcpu pointer in vcpu reg
140+ *
141+ * Clobbers r1, r2, r3, r4.
142+ */
143+.macro restore_guest_regs_mode mode, offset
144+ add r1, vcpu, \offset
145+ ldm r1, {r2, r3, r4}
146+ msr SP_\mode, r2
147+ msr LR_\mode, r3
148+ msr SPSR_\mode, r4
149+.endm
150+
151+/*
152+ * Restore all guest registers from the vcpu struct.
153+ *
154+ * Assumes vcpu pointer in vcpu reg
155+ *
156+ * Clobbers *all* registers.
157+ */
158+.macro restore_guest_regs
159+ restore_guest_regs_mode svc, #VCPU_SVC_REGS
160+ restore_guest_regs_mode abt, #VCPU_ABT_REGS
161+ restore_guest_regs_mode und, #VCPU_UND_REGS
162+ restore_guest_regs_mode irq, #VCPU_IRQ_REGS
163+
164+ add r1, vcpu, #VCPU_FIQ_REGS
165+ ldm r1, {r2-r9}
166+ msr r8_fiq, r2
167+ msr r9_fiq, r3
168+ msr r10_fiq, r4
169+ msr r11_fiq, r5
170+ msr r12_fiq, r6
171+ msr SP_fiq, r7
172+ msr LR_fiq, r8
173+ msr SPSR_fiq, r9
174+
175+ @ Load return state
176+ ldr r2, [vcpu, #VCPU_PC]
177+ ldr r3, [vcpu, #VCPU_CPSR]
178+ msr ELR_hyp, r2
179+ msr SPSR_cxsf, r3
180+
181+ @ Load user registers
182+ ldr r2, [vcpu, #VCPU_USR_SP]
183+ ldr r3, [vcpu, #VCPU_USR_LR]
184+ msr SP_usr, r2
185+ mov lr, r3
186+ add vcpu, vcpu, #(VCPU_USR_REGS)
187+ ldm vcpu, {r0-r12}
188+.endm
189+
190+/*
191+ * Save SP, LR and SPSR for a given mode. offset is the offset of
192+ * this mode's registers from the VCPU base.
193+ *
194+ * Assumes vcpu pointer in vcpu reg
195+ *
196+ * Clobbers r2, r3, r4, r5.
197+ */
198+.macro save_guest_regs_mode mode, offset
199+ add r2, vcpu, \offset
200+ mrs r3, SP_\mode
201+ mrs r4, LR_\mode
202+ mrs r5, SPSR_\mode
203+ stm r2, {r3, r4, r5}
204+.endm
205+
206+/*
207+ * Save all guest registers to the vcpu struct
208+ * Expects guest's r0, r1, r2 on the stack.
209+ *
210+ * Assumes vcpu pointer in vcpu reg
211+ *
212+ * Clobbers r2, r3, r4, r5.
213+ */
214+.macro save_guest_regs
215+ @ Store usr registers
216+ add r2, vcpu, #VCPU_USR_REG(3)
217+ stm r2, {r3-r12}
218+ add r2, vcpu, #VCPU_USR_REG(0)
219+ pop {r3, r4, r5} @ r0, r1, r2
220+ stm r2, {r3, r4, r5}
221+ mrs r2, SP_usr
222+ mov r3, lr
223+ str r2, [vcpu, #VCPU_USR_SP]
224+ str r3, [vcpu, #VCPU_USR_LR]
225+
226+ @ Store return state
227+ mrs r2, ELR_hyp
228+ mrs r3, spsr
229+ str r2, [vcpu, #VCPU_PC]
230+ str r3, [vcpu, #VCPU_CPSR]
231+
232+ @ Store other guest registers
233+ save_guest_regs_mode svc, #VCPU_SVC_REGS
234+ save_guest_regs_mode abt, #VCPU_ABT_REGS
235+ save_guest_regs_mode und, #VCPU_UND_REGS
236+ save_guest_regs_mode irq, #VCPU_IRQ_REGS
237+.endm
238+
239+/* Reads cp15 registers from hardware and stores them in memory
240+ * @store_to_vcpu: If 0, registers are written in-order to the stack,
241+ * otherwise to the VCPU struct pointed to by vcpup
242+ *
243+ * Assumes vcpu pointer in vcpu reg
244+ *
245+ * Clobbers r2 - r12
246+ */
247+.macro read_cp15_state store_to_vcpu
248+ mrc p15, 0, r2, c1, c0, 0 @ SCTLR
249+ mrc p15, 0, r3, c1, c0, 2 @ CPACR
250+ mrc p15, 0, r4, c2, c0, 2 @ TTBCR
251+ mrc p15, 0, r5, c3, c0, 0 @ DACR
252+ mrrc p15, 0, r6, r7, c2 @ TTBR 0
253+ mrrc p15, 1, r8, r9, c2 @ TTBR 1
254+ mrc p15, 0, r10, c10, c2, 0 @ PRRR
255+ mrc p15, 0, r11, c10, c2, 1 @ NMRR
256+ mrc p15, 2, r12, c0, c0, 0 @ CSSELR
257+
258+ .if \store_to_vcpu == 0
259+ push {r2-r12} @ Push CP15 registers
260+ .else
261+ str r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
262+ str r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
263+ str r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
264+ str r5, [vcpu, #CP15_OFFSET(c3_DACR)]
265+ add r2, vcpu, #CP15_OFFSET(c2_TTBR0)
266+ strd r6, r7, [r2]
267+ add r2, vcpu, #CP15_OFFSET(c2_TTBR1)
268+ strd r8, r9, [r2]
269+ str r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
270+ str r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
271+ str r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
272+ .endif
273+
274+ mrc p15, 0, r2, c13, c0, 1 @ CID
275+ mrc p15, 0, r3, c13, c0, 2 @ TID_URW
276+ mrc p15, 0, r4, c13, c0, 3 @ TID_URO
277+ mrc p15, 0, r5, c13, c0, 4 @ TID_PRIV
278+ mrc p15, 0, r6, c5, c0, 0 @ DFSR
279+ mrc p15, 0, r7, c5, c0, 1 @ IFSR
280+ mrc p15, 0, r8, c5, c1, 0 @ ADFSR
281+ mrc p15, 0, r9, c5, c1, 1 @ AIFSR
282+ mrc p15, 0, r10, c6, c0, 0 @ DFAR
283+ mrc p15, 0, r11, c6, c0, 2 @ IFAR
284+ mrc p15, 0, r12, c12, c0, 0 @ VBAR
285+
286+ .if \store_to_vcpu == 0
287+ push {r2-r12} @ Push CP15 registers
288+ .else
289+ str r2, [vcpu, #CP15_OFFSET(c13_CID)]
290+ str r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
291+ str r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
292+ str r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
293+ str r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
294+ str r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
295+ str r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
296+ str r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
297+ str r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
298+ str r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
299+ str r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
300+ .endif
301+.endm
302+
303+/*
304+ * Reads cp15 registers from memory and writes them to hardware
305+ * @read_from_vcpu: If 0, registers are read in-order from the stack,
306+ * otherwise from the VCPU struct pointed to by vcpup
307+ *
308+ * Assumes vcpu pointer in vcpu reg
309+ */
310+.macro write_cp15_state read_from_vcpu
311+ .if \read_from_vcpu == 0
312+ pop {r2-r12}
313+ .else
314+ ldr r2, [vcpu, #CP15_OFFSET(c13_CID)]
315+ ldr r3, [vcpu, #CP15_OFFSET(c13_TID_URW)]
316+ ldr r4, [vcpu, #CP15_OFFSET(c13_TID_URO)]
317+ ldr r5, [vcpu, #CP15_OFFSET(c13_TID_PRIV)]
318+ ldr r6, [vcpu, #CP15_OFFSET(c5_DFSR)]
319+ ldr r7, [vcpu, #CP15_OFFSET(c5_IFSR)]
320+ ldr r8, [vcpu, #CP15_OFFSET(c5_ADFSR)]
321+ ldr r9, [vcpu, #CP15_OFFSET(c5_AIFSR)]
322+ ldr r10, [vcpu, #CP15_OFFSET(c6_DFAR)]
323+ ldr r11, [vcpu, #CP15_OFFSET(c6_IFAR)]
324+ ldr r12, [vcpu, #CP15_OFFSET(c12_VBAR)]
325+ .endif
326+
327+ mcr p15, 0, r2, c13, c0, 1 @ CID
328+ mcr p15, 0, r3, c13, c0, 2 @ TID_URW
329+ mcr p15, 0, r4, c13, c0, 3 @ TID_URO
330+ mcr p15, 0, r5, c13, c0, 4 @ TID_PRIV
331+ mcr p15, 0, r6, c5, c0, 0 @ DFSR
332+ mcr p15, 0, r7, c5, c0, 1 @ IFSR
333+ mcr p15, 0, r8, c5, c1, 0 @ ADFSR
334+ mcr p15, 0, r9, c5, c1, 1 @ AIFSR
335+ mcr p15, 0, r10, c6, c0, 0 @ DFAR
336+ mcr p15, 0, r11, c6, c0, 2 @ IFAR
337+ mcr p15, 0, r12, c12, c0, 0 @ VBAR
338+
339+ .if \read_from_vcpu == 0
340+ pop {r2-r12}
341+ .else
342+ ldr r2, [vcpu, #CP15_OFFSET(c1_SCTLR)]
343+ ldr r3, [vcpu, #CP15_OFFSET(c1_CPACR)]
344+ ldr r4, [vcpu, #CP15_OFFSET(c2_TTBCR)]
345+ ldr r5, [vcpu, #CP15_OFFSET(c3_DACR)]
346+ add r12, vcpu, #CP15_OFFSET(c2_TTBR0)
347+ ldrd r6, r7, [r12]
348+ add r12, vcpu, #CP15_OFFSET(c2_TTBR1)
349+ ldrd r8, r9, [r12]
350+ ldr r10, [vcpu, #CP15_OFFSET(c10_PRRR)]
351+ ldr r11, [vcpu, #CP15_OFFSET(c10_NMRR)]
352+ ldr r12, [vcpu, #CP15_OFFSET(c0_CSSELR)]
353+ .endif
354+
355+ mcr p15, 0, r2, c1, c0, 0 @ SCTLR
356+ mcr p15, 0, r3, c1, c0, 2 @ CPACR
357+ mcr p15, 0, r4, c2, c0, 2 @ TTBCR
358+ mcr p15, 0, r5, c3, c0, 0 @ DACR
359+ mcrr p15, 0, r6, r7, c2 @ TTBR 0
360+ mcrr p15, 1, r8, r9, c2 @ TTBR 1
361+ mcr p15, 0, r10, c10, c2, 0 @ PRRR
362+ mcr p15, 0, r11, c10, c2, 1 @ NMRR
363+ mcr p15, 2, r12, c0, c0, 0 @ CSSELR
364+.endm
365+
366+/*
367+ * Save the VGIC CPU state into memory
368+ *
369+ * Assumes vcpu pointer in vcpu reg
370+ */
371+.macro save_vgic_state
372+.endm
373+
374+/*
375+ * Restore the VGIC CPU state from memory
376+ *
377+ * Assumes vcpu pointer in vcpu reg
378+ */
379+.macro restore_vgic_state
380+.endm
381+
382+.equ vmentry, 0
383+.equ vmexit, 1
384+
385+/* Configures the HSTR (Hyp System Trap Register) on entry/return
386+ * (hardware reset value is 0) */
387+.macro set_hstr operation
388+ mrc p15, 4, r2, c1, c1, 3
389+ ldr r3, =HSTR_T(15)
390+ .if \operation == vmentry
391+ orr r2, r2, r3 @ Trap CR{15}
392+ .else
393+ bic r2, r2, r3 @ Don't trap any CRx accesses
394+ .endif
395+ mcr p15, 4, r2, c1, c1, 3
396+.endm
397+
398+/* Configures the HCPTR (Hyp Coprocessor Trap Register) on entry/return
399+ * (hardware reset value is 0). Keep previous value in r2. */
400+.macro set_hcptr operation, mask
401+ mrc p15, 4, r2, c1, c1, 2
402+ ldr r3, =\mask
403+ .if \operation == vmentry
404+ orr r3, r2, r3 @ Trap coproc-accesses defined in mask
405+ .else
406+ bic r3, r2, r3 @ Don't trap defined coproc-accesses
407+ .endif
408+ mcr p15, 4, r3, c1, c1, 2
409+.endm
410+
411+/* Configures the HDCR (Hyp Debug Configuration Register) on entry/return
412+ * (hardware reset value is 0) */
413+.macro set_hdcr operation
414+ mrc p15, 4, r2, c1, c1, 1
415+ ldr r3, =(HDCR_TPM|HDCR_TPMCR)
416+ .if \operation == vmentry
417+ orr r2, r2, r3 @ Trap some perfmon accesses
418+ .else
419+ bic r2, r2, r3 @ Don't trap any perfmon accesses
420+ .endif
421+ mcr p15, 4, r2, c1, c1, 1
422+.endm
423+
424+/* Enable/Disable: stage-2 trans., trap interrupts, trap wfi, trap smc */
425+.macro configure_hyp_role operation
426+ mrc p15, 4, r2, c1, c1, 0 @ HCR
427+ bic r2, r2, #HCR_VIRT_EXCP_MASK
428+ ldr r3, =HCR_GUEST_MASK
429+ .if \operation == vmentry
430+ orr r2, r2, r3
431+ ldr r3, [vcpu, #VCPU_IRQ_LINES]
432+ orr r2, r2, r3
433+ .else
434+ bic r2, r2, r3
435+ .endif
436+ mcr p15, 4, r2, c1, c1, 0
437+.endm
438+
439+.macro load_vcpu
440+ mrc p15, 4, vcpu, c13, c0, 2 @ HTPIDR
441+.endm
arch/arm/kvm/mmio.c+153−0
@@ -0,0 +1,153 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/kvm_host.h>
20+#include <asm/kvm_mmio.h>
21+#include <asm/kvm_emulate.h>
22+#include <trace/events/kvm.h>
23+
24+#include "trace.h"
25+
26+/**
27+ * kvm_handle_mmio_return -- Handle MMIO loads after user space emulation
28+ * @vcpu: The VCPU pointer
29+ * @run: The VCPU run struct containing the mmio data
30+ *
31+ * This should only be called after returning from userspace for MMIO load
32+ * emulation.
33+ */
34+int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
35+{
36+ __u32 *dest;
37+ unsigned int len;
38+ int mask;
39+
40+ if (!run->mmio.is_write) {
41+ dest = vcpu_reg(vcpu, vcpu->arch.mmio_decode.rt);
42+ memset(dest, 0, sizeof(int));
43+
44+ len = run->mmio.len;
45+ if (len > 4)
46+ return -EINVAL;
47+
48+ memcpy(dest, run->mmio.data, len);
49+
50+ trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr,
51+ *((u64 *)run->mmio.data));
52+
53+ if (vcpu->arch.mmio_decode.sign_extend && len < 4) {
54+ mask = 1U << ((len * 8) - 1);
55+ *dest = (*dest ^ mask) - mask;
56+ }
57+ }
58+
59+ return 0;
60+}
61+
62+static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
63+ struct kvm_exit_mmio *mmio)
64+{
65+ unsigned long rt, len;
66+ bool is_write, sign_extend;
67+
68+ if ((vcpu->arch.hsr >> 8) & 1) {
69+ /* cache operation on I/O addr, tell guest unsupported */
70+ kvm_inject_dabt(vcpu, vcpu->arch.hxfar);
71+ return 1;
72+ }
73+
74+ if ((vcpu->arch.hsr >> 7) & 1) {
75+ /* page table accesses IO mem: tell guest to fix its TTBR */
76+ kvm_inject_dabt(vcpu, vcpu->arch.hxfar);
77+ return 1;
78+ }
79+
80+ switch ((vcpu->arch.hsr >> 22) & 0x3) {
81+ case 0:
82+ len = 1;
83+ break;
84+ case 1:
85+ len = 2;
86+ break;
87+ case 2:
88+ len = 4;
89+ break;
90+ default:
91+ kvm_err("Hardware is weird: SAS 0b11 is reserved\n");
92+ return -EFAULT;
93+ }
94+
95+ is_write = vcpu->arch.hsr & HSR_WNR;
96+ sign_extend = vcpu->arch.hsr & HSR_SSE;
97+ rt = (vcpu->arch.hsr & HSR_SRT_MASK) >> HSR_SRT_SHIFT;
98+
99+ if (kvm_vcpu_reg_is_pc(vcpu, rt)) {
100+ /* IO memory trying to read/write pc */
101+ kvm_inject_pabt(vcpu, vcpu->arch.hxfar);
102+ return 1;
103+ }
104+
105+ mmio->is_write = is_write;
106+ mmio->phys_addr = fault_ipa;
107+ mmio->len = len;
108+ vcpu->arch.mmio_decode.sign_extend = sign_extend;
109+ vcpu->arch.mmio_decode.rt = rt;
110+
111+ /*
112+ * The MMIO instruction is emulated and should not be re-executed
113+ * in the guest.
114+ */
115+ kvm_skip_instr(vcpu, (vcpu->arch.hsr >> 25) & 1);
116+ return 0;
117+}
118+
119+int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
120+ phys_addr_t fault_ipa)
121+{
122+ struct kvm_exit_mmio mmio;
123+ unsigned long rt;
124+ int ret;
125+
126+ /*
127+ * Prepare MMIO operation. First stash it in a private
128+ * structure that we can use for in-kernel emulation. If the
129+ * kernel can't handle it, copy it into run->mmio and let user
130+ * space do its magic.
131+ */
132+
133+ if (vcpu->arch.hsr & HSR_ISV) {
134+ ret = decode_hsr(vcpu, fault_ipa, &mmio);
135+ if (ret)
136+ return ret;
137+ } else {
138+ kvm_err("load/store instruction decoding not implemented\n");
139+ return -ENOSYS;
140+ }
141+
142+ rt = vcpu->arch.mmio_decode.rt;
143+ trace_kvm_mmio((mmio.is_write) ? KVM_TRACE_MMIO_WRITE :
144+ KVM_TRACE_MMIO_READ_UNSATISFIED,
145+ mmio.len, fault_ipa,
146+ (mmio.is_write) ? *vcpu_reg(vcpu, rt) : 0);
147+
148+ if (mmio.is_write)
149+ memcpy(mmio.data, vcpu_reg(vcpu, rt), mmio.len);
150+
151+ kvm_prepare_mmio(run, &mmio);
152+ return 0;
153+}
arch/arm/kvm/mmu.c+787−0
@@ -0,0 +1,787 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+
19+#include <linux/mman.h>
20+#include <linux/kvm_host.h>
21+#include <linux/io.h>
22+#include <trace/events/kvm.h>
23+#include <asm/idmap.h>
24+#include <asm/pgalloc.h>
25+#include <asm/cacheflush.h>
26+#include <asm/kvm_arm.h>
27+#include <asm/kvm_mmu.h>
28+#include <asm/kvm_mmio.h>
29+#include <asm/kvm_asm.h>
30+#include <asm/kvm_emulate.h>
31+#include <asm/mach/map.h>
32+#include <trace/events/kvm.h>
33+
34+#include "trace.h"
35+
36+extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
37+
38+static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
39+
40+static void kvm_tlb_flush_vmid(struct kvm *kvm)
41+{
42+ kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
43+}
44+
45+static void kvm_set_pte(pte_t *pte, pte_t new_pte)
46+{
47+ pte_val(*pte) = new_pte;
48+ /*
49+ * flush_pmd_entry just takes a void pointer and cleans the necessary
50+ * cache entries, so we can reuse the function for ptes.
51+ */
52+ flush_pmd_entry(pte);
53+}
54+
55+static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
56+ int min, int max)
57+{
58+ void *page;
59+
60+ BUG_ON(max > KVM_NR_MEM_OBJS);
61+ if (cache->nobjs >= min)
62+ return 0;
63+ while (cache->nobjs < max) {
64+ page = (void *)__get_free_page(PGALLOC_GFP);
65+ if (!page)
66+ return -ENOMEM;
67+ cache->objects[cache->nobjs++] = page;
68+ }
69+ return 0;
70+}
71+
72+static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
73+{
74+ while (mc->nobjs)
75+ free_page((unsigned long)mc->objects[--mc->nobjs]);
76+}
77+
78+static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
79+{
80+ void *p;
81+
82+ BUG_ON(!mc || !mc->nobjs);
83+ p = mc->objects[--mc->nobjs];
84+ return p;
85+}
86+
87+static void free_ptes(pmd_t *pmd, unsigned long addr)
88+{
89+ pte_t *pte;
90+ unsigned int i;
91+
92+ for (i = 0; i < PTRS_PER_PMD; i++, addr += PMD_SIZE) {
93+ if (!pmd_none(*pmd) && pmd_table(*pmd)) {
94+ pte = pte_offset_kernel(pmd, addr);
95+ pte_free_kernel(NULL, pte);
96+ }
97+ pmd++;
98+ }
99+}
100+
101+/**
102+ * free_hyp_pmds - free a Hyp-mode level-2 tables and child level-3 tables
103+ *
104+ * Assumes this is a page table used strictly in Hyp-mode and therefore contains
105+ * only mappings in the kernel memory area, which is above PAGE_OFFSET.
106+ */
107+void free_hyp_pmds(void)
108+{
109+ pgd_t *pgd;
110+ pud_t *pud;
111+ pmd_t *pmd;
112+ unsigned long addr;
113+
114+ mutex_lock(&kvm_hyp_pgd_mutex);
115+ for (addr = PAGE_OFFSET; addr != 0; addr += PGDIR_SIZE) {
116+ pgd = hyp_pgd + pgd_index(addr);
117+ pud = pud_offset(pgd, addr);
118+
119+ if (pud_none(*pud))
120+ continue;
121+ BUG_ON(pud_bad(*pud));
122+
123+ pmd = pmd_offset(pud, addr);
124+ free_ptes(pmd, addr);
125+ pmd_free(NULL, pmd);
126+ pud_clear(pud);
127+ }
128+ mutex_unlock(&kvm_hyp_pgd_mutex);
129+}
130+
131+static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
132+ unsigned long end)
133+{
134+ pte_t *pte;
135+ unsigned long addr;
136+ struct page *page;
137+
138+ for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
139+ pte = pte_offset_kernel(pmd, addr);
140+ BUG_ON(!virt_addr_valid(addr));
141+ page = virt_to_page(addr);
142+ kvm_set_pte(pte, mk_pte(page, PAGE_HYP));
143+ }
144+}
145+
146+static void create_hyp_io_pte_mappings(pmd_t *pmd, unsigned long start,
147+ unsigned long end,
148+ unsigned long *pfn_base)
149+{
150+ pte_t *pte;
151+ unsigned long addr;
152+
153+ for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
154+ pte = pte_offset_kernel(pmd, addr);
155+ BUG_ON(pfn_valid(*pfn_base));
156+ kvm_set_pte(pte, pfn_pte(*pfn_base, PAGE_HYP_DEVICE));
157+ (*pfn_base)++;
158+ }
159+}
160+
161+static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
162+ unsigned long end, unsigned long *pfn_base)
163+{
164+ pmd_t *pmd;
165+ pte_t *pte;
166+ unsigned long addr, next;
167+
168+ for (addr = start; addr < end; addr = next) {
169+ pmd = pmd_offset(pud, addr);
170+
171+ BUG_ON(pmd_sect(*pmd));
172+
173+ if (pmd_none(*pmd)) {
174+ pte = pte_alloc_one_kernel(NULL, addr);
175+ if (!pte) {
176+ kvm_err("Cannot allocate Hyp pte\n");
177+ return -ENOMEM;
178+ }
179+ pmd_populate_kernel(NULL, pmd, pte);
180+ }
181+
182+ next = pmd_addr_end(addr, end);
183+
184+ /*
185+ * If pfn_base is NULL, we map kernel pages into HYP with the
186+ * virtual address. Otherwise, this is considered an I/O
187+ * mapping and we map the physical region starting at
188+ * *pfn_base to [start, end[.
189+ */
190+ if (!pfn_base)
191+ create_hyp_pte_mappings(pmd, addr, next);
192+ else
193+ create_hyp_io_pte_mappings(pmd, addr, next, pfn_base);
194+ }
195+
196+ return 0;
197+}
198+
199+static int __create_hyp_mappings(void *from, void *to, unsigned long *pfn_base)
200+{
201+ unsigned long start = (unsigned long)from;
202+ unsigned long end = (unsigned long)to;
203+ pgd_t *pgd;
204+ pud_t *pud;
205+ pmd_t *pmd;
206+ unsigned long addr, next;
207+ int err = 0;
208+
209+ BUG_ON(start > end);
210+ if (start < PAGE_OFFSET)
211+ return -EINVAL;
212+
213+ mutex_lock(&kvm_hyp_pgd_mutex);
214+ for (addr = start; addr < end; addr = next) {
215+ pgd = hyp_pgd + pgd_index(addr);
216+ pud = pud_offset(pgd, addr);
217+
218+ if (pud_none_or_clear_bad(pud)) {
219+ pmd = pmd_alloc_one(NULL, addr);
220+ if (!pmd) {
221+ kvm_err("Cannot allocate Hyp pmd\n");
222+ err = -ENOMEM;
223+ goto out;
224+ }
225+ pud_populate(NULL, pud, pmd);
226+ }
227+
228+ next = pgd_addr_end(addr, end);
229+ err = create_hyp_pmd_mappings(pud, addr, next, pfn_base);
230+ if (err)
231+ goto out;
232+ }
233+out:
234+ mutex_unlock(&kvm_hyp_pgd_mutex);
235+ return err;
236+}
237+
238+/**
239+ * create_hyp_mappings - map a kernel virtual address range in Hyp mode
240+ * @from: The virtual kernel start address of the range
241+ * @to: The virtual kernel end address of the range (exclusive)
242+ *
243+ * The same virtual address as the kernel virtual address is also used in
244+ * Hyp-mode mapping to the same underlying physical pages.
245+ *
246+ * Note: Wrapping around zero in the "to" address is not supported.
247+ */
248+int create_hyp_mappings(void *from, void *to)
249+{
250+ return __create_hyp_mappings(from, to, NULL);
251+}
252+
253+/**
254+ * create_hyp_io_mappings - map a physical IO range in Hyp mode
255+ * @from: The virtual HYP start address of the range
256+ * @to: The virtual HYP end address of the range (exclusive)
257+ * @addr: The physical start address which gets mapped
258+ */
259+int create_hyp_io_mappings(void *from, void *to, phys_addr_t addr)
260+{
261+ unsigned long pfn = __phys_to_pfn(addr);
262+ return __create_hyp_mappings(from, to, &pfn);
263+}
264+
265+/**
266+ * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
267+ * @kvm: The KVM struct pointer for the VM.
268+ *
269+ * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can
270+ * support either full 40-bit input addresses or limited to 32-bit input
271+ * addresses). Clears the allocated pages.
272+ *
273+ * Note we don't need locking here as this is only called when the VM is
274+ * created, which can only be done once.
275+ */
276+int kvm_alloc_stage2_pgd(struct kvm *kvm)
277+{
278+ pgd_t *pgd;
279+
280+ if (kvm->arch.pgd != NULL) {
281+ kvm_err("kvm_arch already initialized?\n");
282+ return -EINVAL;
283+ }
284+
285+ pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, S2_PGD_ORDER);
286+ if (!pgd)
287+ return -ENOMEM;
288+
289+ /* stage-2 pgd must be aligned to its size */
290+ VM_BUG_ON((unsigned long)pgd & (S2_PGD_SIZE - 1));
291+
292+ memset(pgd, 0, PTRS_PER_S2_PGD * sizeof(pgd_t));
293+ clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t));
294+ kvm->arch.pgd = pgd;
295+
296+ return 0;
297+}
298+
299+static void clear_pud_entry(pud_t *pud)
300+{
301+ pmd_t *pmd_table = pmd_offset(pud, 0);
302+ pud_clear(pud);
303+ pmd_free(NULL, pmd_table);
304+ put_page(virt_to_page(pud));
305+}
306+
307+static void clear_pmd_entry(pmd_t *pmd)
308+{
309+ pte_t *pte_table = pte_offset_kernel(pmd, 0);
310+ pmd_clear(pmd);
311+ pte_free_kernel(NULL, pte_table);
312+ put_page(virt_to_page(pmd));
313+}
314+
315+static bool pmd_empty(pmd_t *pmd)
316+{
317+ struct page *pmd_page = virt_to_page(pmd);
318+ return page_count(pmd_page) == 1;
319+}
320+
321+static void clear_pte_entry(pte_t *pte)
322+{
323+ if (pte_present(*pte)) {
324+ kvm_set_pte(pte, __pte(0));
325+ put_page(virt_to_page(pte));
326+ }
327+}
328+
329+static bool pte_empty(pte_t *pte)
330+{
331+ struct page *pte_page = virt_to_page(pte);
332+ return page_count(pte_page) == 1;
333+}
334+
335+/**
336+ * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
337+ * @kvm: The VM pointer
338+ * @start: The intermediate physical base address of the range to unmap
339+ * @size: The size of the area to unmap
340+ *
341+ * Clear a range of stage-2 mappings, lowering the various ref-counts. Must
342+ * be called while holding mmu_lock (unless for freeing the stage2 pgd before
343+ * destroying the VM), otherwise another faulting VCPU may come in and mess
344+ * with things behind our backs.
345+ */
346+static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
347+{
348+ pgd_t *pgd;
349+ pud_t *pud;
350+ pmd_t *pmd;
351+ pte_t *pte;
352+ phys_addr_t addr = start, end = start + size;
353+ u64 range;
354+
355+ while (addr < end) {
356+ pgd = kvm->arch.pgd + pgd_index(addr);
357+ pud = pud_offset(pgd, addr);
358+ if (pud_none(*pud)) {
359+ addr += PUD_SIZE;
360+ continue;
361+ }
362+
363+ pmd = pmd_offset(pud, addr);
364+ if (pmd_none(*pmd)) {
365+ addr += PMD_SIZE;
366+ continue;
367+ }
368+
369+ pte = pte_offset_kernel(pmd, addr);
370+ clear_pte_entry(pte);
371+ range = PAGE_SIZE;
372+
373+ /* If we emptied the pte, walk back up the ladder */
374+ if (pte_empty(pte)) {
375+ clear_pmd_entry(pmd);
376+ range = PMD_SIZE;
377+ if (pmd_empty(pmd)) {
378+ clear_pud_entry(pud);
379+ range = PUD_SIZE;
380+ }
381+ }
382+
383+ addr += range;
384+ }
385+}
386+
387+/**
388+ * kvm_free_stage2_pgd - free all stage-2 tables
389+ * @kvm: The KVM struct pointer for the VM.
390+ *
391+ * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
392+ * underlying level-2 and level-3 tables before freeing the actual level-1 table
393+ * and setting the struct pointer to NULL.
394+ *
395+ * Note we don't need locking here as this is only called when the VM is
396+ * destroyed, which can only be done once.
397+ */
398+void kvm_free_stage2_pgd(struct kvm *kvm)
399+{
400+ if (kvm->arch.pgd == NULL)
401+ return;
402+
403+ unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
404+ free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER);
405+ kvm->arch.pgd = NULL;
406+}
407+
408+
409+static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
410+ phys_addr_t addr, const pte_t *new_pte, bool iomap)
411+{
412+ pgd_t *pgd;
413+ pud_t *pud;
414+ pmd_t *pmd;
415+ pte_t *pte, old_pte;
416+
417+ /* Create 2nd stage page table mapping - Level 1 */
418+ pgd = kvm->arch.pgd + pgd_index(addr);
419+ pud = pud_offset(pgd, addr);
420+ if (pud_none(*pud)) {
421+ if (!cache)
422+ return 0; /* ignore calls from kvm_set_spte_hva */
423+ pmd = mmu_memory_cache_alloc(cache);
424+ pud_populate(NULL, pud, pmd);
425+ pmd += pmd_index(addr);
426+ get_page(virt_to_page(pud));
427+ } else
428+ pmd = pmd_offset(pud, addr);
429+
430+ /* Create 2nd stage page table mapping - Level 2 */
431+ if (pmd_none(*pmd)) {
432+ if (!cache)
433+ return 0; /* ignore calls from kvm_set_spte_hva */
434+ pte = mmu_memory_cache_alloc(cache);
435+ clean_pte_table(pte);
436+ pmd_populate_kernel(NULL, pmd, pte);
437+ pte += pte_index(addr);
438+ get_page(virt_to_page(pmd));
439+ } else
440+ pte = pte_offset_kernel(pmd, addr);
441+
442+ if (iomap && pte_present(*pte))
443+ return -EFAULT;
444+
445+ /* Create 2nd stage page table mapping - Level 3 */
446+ old_pte = *pte;
447+ kvm_set_pte(pte, *new_pte);
448+ if (pte_present(old_pte))
449+ kvm_tlb_flush_vmid(kvm);
450+ else
451+ get_page(virt_to_page(pte));
452+
453+ return 0;
454+}
455+
456+/**
457+ * kvm_phys_addr_ioremap - map a device range to guest IPA
458+ *
459+ * @kvm: The KVM pointer
460+ * @guest_ipa: The IPA at which to insert the mapping
461+ * @pa: The physical address of the device
462+ * @size: The size of the mapping
463+ */
464+int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
465+ phys_addr_t pa, unsigned long size)
466+{
467+ phys_addr_t addr, end;
468+ int ret = 0;
469+ unsigned long pfn;
470+ struct kvm_mmu_memory_cache cache = { 0, };
471+
472+ end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
473+ pfn = __phys_to_pfn(pa);
474+
475+ for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
476+ pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE | L_PTE_S2_RDWR);
477+
478+ ret = mmu_topup_memory_cache(&cache, 2, 2);
479+ if (ret)
480+ goto out;
481+ spin_lock(&kvm->mmu_lock);
482+ ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
483+ spin_unlock(&kvm->mmu_lock);
484+ if (ret)
485+ goto out;
486+
487+ pfn++;
488+ }
489+
490+out:
491+ mmu_free_memory_cache(&cache);
492+ return ret;
493+}
494+
495+static void coherent_icache_guest_page(struct kvm *kvm, gfn_t gfn)
496+{
497+ /*
498+ * If we are going to insert an instruction page and the icache is
499+ * either VIPT or PIPT, there is a potential problem where the host
500+ * (or another VM) may have used the same page as this guest, and we
501+ * read incorrect data from the icache. If we're using a PIPT cache,
502+ * we can invalidate just that page, but if we are using a VIPT cache
503+ * we need to invalidate the entire icache - damn shame - as written
504+ * in the ARM ARM (DDI 0406C.b - Page B3-1393).
505+ *
506+ * VIVT caches are tagged using both the ASID and the VMID and doesn't
507+ * need any kind of flushing (DDI 0406C.b - Page B3-1392).
508+ */
509+ if (icache_is_pipt()) {
510+ unsigned long hva = gfn_to_hva(kvm, gfn);
511+ __cpuc_coherent_user_range(hva, hva + PAGE_SIZE);
512+ } else if (!icache_is_vivt_asid_tagged()) {
513+ /* any kind of VIPT cache */
514+ __flush_icache_all();
515+ }
516+}
517+
518+static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
519+ gfn_t gfn, struct kvm_memory_slot *memslot,
520+ unsigned long fault_status)
521+{
522+ pte_t new_pte;
523+ pfn_t pfn;
524+ int ret;
525+ bool write_fault, writable;
526+ unsigned long mmu_seq;
527+ struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
528+
529+ write_fault = kvm_is_write_fault(vcpu->arch.hsr);
530+ if (fault_status == FSC_PERM && !write_fault) {
531+ kvm_err("Unexpected L2 read permission error\n");
532+ return -EFAULT;
533+ }
534+
535+ /* We need minimum second+third level pages */
536+ ret = mmu_topup_memory_cache(memcache, 2, KVM_NR_MEM_OBJS);
537+ if (ret)
538+ return ret;
539+
540+ mmu_seq = vcpu->kvm->mmu_notifier_seq;
541+ /*
542+ * Ensure the read of mmu_notifier_seq happens before we call
543+ * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
544+ * the page we just got a reference to gets unmapped before we have a
545+ * chance to grab the mmu_lock, which ensure that if the page gets
546+ * unmapped afterwards, the call to kvm_unmap_hva will take it away
547+ * from us again properly. This smp_rmb() interacts with the smp_wmb()
548+ * in kvm_mmu_notifier_invalidate_<page|range_end>.
549+ */
550+ smp_rmb();
551+
552+ pfn = gfn_to_pfn_prot(vcpu->kvm, gfn, write_fault, &writable);
553+ if (is_error_pfn(pfn))
554+ return -EFAULT;
555+
556+ new_pte = pfn_pte(pfn, PAGE_S2);
557+ coherent_icache_guest_page(vcpu->kvm, gfn);
558+
559+ spin_lock(&vcpu->kvm->mmu_lock);
560+ if (mmu_notifier_retry(vcpu->kvm, mmu_seq))
561+ goto out_unlock;
562+ if (writable) {
563+ pte_val(new_pte) |= L_PTE_S2_RDWR;
564+ kvm_set_pfn_dirty(pfn);
565+ }
566+ stage2_set_pte(vcpu->kvm, memcache, fault_ipa, &new_pte, false);
567+
568+out_unlock:
569+ spin_unlock(&vcpu->kvm->mmu_lock);
570+ kvm_release_pfn_clean(pfn);
571+ return 0;
572+}
573+
574+/**
575+ * kvm_handle_guest_abort - handles all 2nd stage aborts
576+ * @vcpu: the VCPU pointer
577+ * @run: the kvm_run structure
578+ *
579+ * Any abort that gets to the host is almost guaranteed to be caused by a
580+ * missing second stage translation table entry, which can mean that either the
581+ * guest simply needs more memory and we must allocate an appropriate page or it
582+ * can mean that the guest tried to access I/O memory, which is emulated by user
583+ * space. The distinction is based on the IPA causing the fault and whether this
584+ * memory region has been registered as standard RAM by user space.
585+ */
586+int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
587+{
588+ unsigned long hsr_ec;
589+ unsigned long fault_status;
590+ phys_addr_t fault_ipa;
591+ struct kvm_memory_slot *memslot;
592+ bool is_iabt;
593+ gfn_t gfn;
594+ int ret, idx;
595+
596+ hsr_ec = vcpu->arch.hsr >> HSR_EC_SHIFT;
597+ is_iabt = (hsr_ec == HSR_EC_IABT);
598+ fault_ipa = ((phys_addr_t)vcpu->arch.hpfar & HPFAR_MASK) << 8;
599+
600+ trace_kvm_guest_fault(*vcpu_pc(vcpu), vcpu->arch.hsr,
601+ vcpu->arch.hxfar, fault_ipa);
602+
603+ /* Check the stage-2 fault is trans. fault or write fault */
604+ fault_status = (vcpu->arch.hsr & HSR_FSC_TYPE);
605+ if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
606+ kvm_err("Unsupported fault status: EC=%#lx DFCS=%#lx\n",
607+ hsr_ec, fault_status);
608+ return -EFAULT;
609+ }
610+
611+ idx = srcu_read_lock(&vcpu->kvm->srcu);
612+
613+ gfn = fault_ipa >> PAGE_SHIFT;
614+ if (!kvm_is_visible_gfn(vcpu->kvm, gfn)) {
615+ if (is_iabt) {
616+ /* Prefetch Abort on I/O address */
617+ kvm_inject_pabt(vcpu, vcpu->arch.hxfar);
618+ ret = 1;
619+ goto out_unlock;
620+ }
621+
622+ if (fault_status != FSC_FAULT) {
623+ kvm_err("Unsupported fault status on io memory: %#lx\n",
624+ fault_status);
625+ ret = -EFAULT;
626+ goto out_unlock;
627+ }
628+
629+ /* Adjust page offset */
630+ fault_ipa |= vcpu->arch.hxfar & ~PAGE_MASK;
631+ ret = io_mem_abort(vcpu, run, fault_ipa);
632+ goto out_unlock;
633+ }
634+
635+ memslot = gfn_to_memslot(vcpu->kvm, gfn);
636+ if (!memslot->user_alloc) {
637+ kvm_err("non user-alloc memslots not supported\n");
638+ ret = -EINVAL;
639+ goto out_unlock;
640+ }
641+
642+ ret = user_mem_abort(vcpu, fault_ipa, gfn, memslot, fault_status);
643+ if (ret == 0)
644+ ret = 1;
645+out_unlock:
646+ srcu_read_unlock(&vcpu->kvm->srcu, idx);
647+ return ret;
648+}
649+
650+static void handle_hva_to_gpa(struct kvm *kvm,
651+ unsigned long start,
652+ unsigned long end,
653+ void (*handler)(struct kvm *kvm,
654+ gpa_t gpa, void *data),
655+ void *data)
656+{
657+ struct kvm_memslots *slots;
658+ struct kvm_memory_slot *memslot;
659+
660+ slots = kvm_memslots(kvm);
661+
662+ /* we only care about the pages that the guest sees */
663+ kvm_for_each_memslot(memslot, slots) {
664+ unsigned long hva_start, hva_end;
665+ gfn_t gfn, gfn_end;
666+
667+ hva_start = max(start, memslot->userspace_addr);
668+ hva_end = min(end, memslot->userspace_addr +
669+ (memslot->npages << PAGE_SHIFT));
670+ if (hva_start >= hva_end)
671+ continue;
672+
673+ /*
674+ * {gfn(page) | page intersects with [hva_start, hva_end)} =
675+ * {gfn_start, gfn_start+1, ..., gfn_end-1}.
676+ */
677+ gfn = hva_to_gfn_memslot(hva_start, memslot);
678+ gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
679+
680+ for (; gfn < gfn_end; ++gfn) {
681+ gpa_t gpa = gfn << PAGE_SHIFT;
682+ handler(kvm, gpa, data);
683+ }
684+ }
685+}
686+
687+static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
688+{
689+ unmap_stage2_range(kvm, gpa, PAGE_SIZE);
690+ kvm_tlb_flush_vmid(kvm);
691+}
692+
693+int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
694+{
695+ unsigned long end = hva + PAGE_SIZE;
696+
697+ if (!kvm->arch.pgd)
698+ return 0;
699+
700+ trace_kvm_unmap_hva(hva);
701+ handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
702+ return 0;
703+}
704+
705+int kvm_unmap_hva_range(struct kvm *kvm,
706+ unsigned long start, unsigned long end)
707+{
708+ if (!kvm->arch.pgd)
709+ return 0;
710+
711+ trace_kvm_unmap_hva_range(start, end);
712+ handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
713+ return 0;
714+}
715+
716+static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
717+{
718+ pte_t *pte = (pte_t *)data;
719+
720+ stage2_set_pte(kvm, NULL, gpa, pte, false);
721+}
722+
723+
724+void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
725+{
726+ unsigned long end = hva + PAGE_SIZE;
727+ pte_t stage2_pte;
728+
729+ if (!kvm->arch.pgd)
730+ return;
731+
732+ trace_kvm_set_spte_hva(hva);
733+ stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
734+ handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
735+}
736+
737+void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
738+{
739+ mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
740+}
741+
742+phys_addr_t kvm_mmu_get_httbr(void)
743+{
744+ VM_BUG_ON(!virt_addr_valid(hyp_pgd));
745+ return virt_to_phys(hyp_pgd);
746+}
747+
748+int kvm_mmu_init(void)
749+{
750+ if (!hyp_pgd) {
751+ kvm_err("Hyp mode PGD not allocated\n");
752+ return -ENOMEM;
753+ }
754+
755+ return 0;
756+}
757+
758+/**
759+ * kvm_clear_idmap - remove all idmaps from the hyp pgd
760+ *
761+ * Free the underlying pmds for all pgds in range and clear the pgds (but
762+ * don't free them) afterwards.
763+ */
764+void kvm_clear_hyp_idmap(void)
765+{
766+ unsigned long addr, end;
767+ unsigned long next;
768+ pgd_t *pgd = hyp_pgd;
769+ pud_t *pud;
770+ pmd_t *pmd;
771+
772+ addr = virt_to_phys(__hyp_idmap_text_start);
773+ end = virt_to_phys(__hyp_idmap_text_end);
774+
775+ pgd += pgd_index(addr);
776+ do {
777+ next = pgd_addr_end(addr, end);
778+ if (pgd_none_or_clear_bad(pgd))
779+ continue;
780+ pud = pud_offset(pgd, addr);
781+ pmd = pmd_offset(pud, addr);
782+
783+ pud_clear(pud);
784+ clean_pmd_entry(pmd);
785+ pmd_free(NULL, (pmd_t *)((unsigned long)pmd & PAGE_MASK));
786+ } while (pgd++, addr = next, addr < end);
787+}
arch/arm/kvm/psci.c+108−0
@@ -0,0 +1,108 @@
1+/*
2+ * Copyright (C) 2012 - ARM Ltd
3+ * Author: Marc Zyngier <marc.zyngier@arm.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License version 2 as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
16+ */
17+
18+#include <linux/kvm_host.h>
19+#include <linux/wait.h>
20+
21+#include <asm/kvm_emulate.h>
22+#include <asm/kvm_psci.h>
23+
24+/*
25+ * This is an implementation of the Power State Coordination Interface
26+ * as described in ARM document number ARM DEN 0022A.
27+ */
28+
29+static void kvm_psci_vcpu_off(struct kvm_vcpu *vcpu)
30+{
31+ vcpu->arch.pause = true;
32+}
33+
34+static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
35+{
36+ struct kvm *kvm = source_vcpu->kvm;
37+ struct kvm_vcpu *vcpu;
38+ wait_queue_head_t *wq;
39+ unsigned long cpu_id;
40+ phys_addr_t target_pc;
41+
42+ cpu_id = *vcpu_reg(source_vcpu, 1);
43+ if (vcpu_mode_is_32bit(source_vcpu))
44+ cpu_id &= ~((u32) 0);
45+
46+ if (cpu_id >= atomic_read(&kvm->online_vcpus))
47+ return KVM_PSCI_RET_INVAL;
48+
49+ target_pc = *vcpu_reg(source_vcpu, 2);
50+
51+ vcpu = kvm_get_vcpu(kvm, cpu_id);
52+
53+ wq = kvm_arch_vcpu_wq(vcpu);
54+ if (!waitqueue_active(wq))
55+ return KVM_PSCI_RET_INVAL;
56+
57+ kvm_reset_vcpu(vcpu);
58+
59+ /* Gracefully handle Thumb2 entry point */
60+ if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {
61+ target_pc &= ~((phys_addr_t) 1);
62+ vcpu_set_thumb(vcpu);
63+ }
64+
65+ *vcpu_pc(vcpu) = target_pc;
66+ vcpu->arch.pause = false;
67+ smp_mb(); /* Make sure the above is visible */
68+
69+ wake_up_interruptible(wq);
70+
71+ return KVM_PSCI_RET_SUCCESS;
72+}
73+
74+/**
75+ * kvm_psci_call - handle PSCI call if r0 value is in range
76+ * @vcpu: Pointer to the VCPU struct
77+ *
78+ * Handle PSCI calls from guests through traps from HVC or SMC instructions.
79+ * The calling convention is similar to SMC calls to the secure world where
80+ * the function number is placed in r0 and this function returns true if the
81+ * function number specified in r0 is withing the PSCI range, and false
82+ * otherwise.
83+ */
84+bool kvm_psci_call(struct kvm_vcpu *vcpu)
85+{
86+ unsigned long psci_fn = *vcpu_reg(vcpu, 0) & ~((u32) 0);
87+ unsigned long val;
88+
89+ switch (psci_fn) {
90+ case KVM_PSCI_FN_CPU_OFF:
91+ kvm_psci_vcpu_off(vcpu);
92+ val = KVM_PSCI_RET_SUCCESS;
93+ break;
94+ case KVM_PSCI_FN_CPU_ON:
95+ val = kvm_psci_vcpu_on(vcpu);
96+ break;
97+ case KVM_PSCI_FN_CPU_SUSPEND:
98+ case KVM_PSCI_FN_MIGRATE:
99+ val = KVM_PSCI_RET_NI;
100+ break;
101+
102+ default:
103+ return false;
104+ }
105+
106+ *vcpu_reg(vcpu, 0) = val;
107+ return true;
108+}
arch/arm/kvm/reset.c+74−0
@@ -0,0 +1,74 @@
1+/*
2+ * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3+ * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4+ *
5+ * This program is free software; you can redistribute it and/or modify
6+ * it under the terms of the GNU General Public License, version 2, as
7+ * published by the Free Software Foundation.
8+ *
9+ * This program is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+ * GNU General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU General Public License
15+ * along with this program; if not, write to the Free Software
16+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+ */
18+#include <linux/compiler.h>
19+#include <linux/errno.h>
20+#include <linux/sched.h>
21+#include <linux/kvm_host.h>
22+#include <linux/kvm.h>
23+
24+#include <asm/unified.h>
25+#include <asm/ptrace.h>
26+#include <asm/cputype.h>
27+#include <asm/kvm_arm.h>
28+#include <asm/kvm_coproc.h>
29+
30+/******************************************************************************
31+ * Cortex-A15 Reset Values
32+ */
33+
34+static const int a15_max_cpu_idx = 3;
35+
36+static struct kvm_regs a15_regs_reset = {
37+ .usr_regs.ARM_cpsr = SVC_MODE | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT,
38+};
39+
40+
41+/*******************************************************************************
42+ * Exported reset function
43+ */
44+
45+/**
46+ * kvm_reset_vcpu - sets core registers and cp15 registers to reset value
47+ * @vcpu: The VCPU pointer
48+ *
49+ * This function finds the right table above and sets the registers on the
50+ * virtual CPU struct to their architectually defined reset values.
51+ */
52+int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
53+{
54+ struct kvm_regs *cpu_reset;
55+
56+ switch (vcpu->arch.target) {
57+ case KVM_ARM_TARGET_CORTEX_A15:
58+ if (vcpu->vcpu_id > a15_max_cpu_idx)
59+ return -EINVAL;
60+ cpu_reset = &a15_regs_reset;
61+ vcpu->arch.midr = read_cpuid_id();
62+ break;
63+ default:
64+ return -ENODEV;
65+ }
66+
67+ /* Reset core registers */
68+ memcpy(&vcpu->arch.regs, cpu_reset, sizeof(vcpu->arch.regs));
69+
70+ /* Reset CP15 registers */
71+ kvm_reset_coprocs(vcpu);
72+
73+ return 0;
74+}
arch/arm/kvm/trace.h+235−0
@@ -0,0 +1,235 @@
1+#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
2+#define _TRACE_KVM_H
3+
4+#include <linux/tracepoint.h>
5+
6+#undef TRACE_SYSTEM
7+#define TRACE_SYSTEM kvm
8+
9+/*
10+ * Tracepoints for entry/exit to guest
11+ */
12+TRACE_EVENT(kvm_entry,
13+ TP_PROTO(unsigned long vcpu_pc),
14+ TP_ARGS(vcpu_pc),
15+
16+ TP_STRUCT__entry(
17+ __field( unsigned long, vcpu_pc )
18+ ),
19+
20+ TP_fast_assign(
21+ __entry->vcpu_pc = vcpu_pc;
22+ ),
23+
24+ TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
25+);
26+
27+TRACE_EVENT(kvm_exit,
28+ TP_PROTO(unsigned long vcpu_pc),
29+ TP_ARGS(vcpu_pc),
30+
31+ TP_STRUCT__entry(
32+ __field( unsigned long, vcpu_pc )
33+ ),
34+
35+ TP_fast_assign(
36+ __entry->vcpu_pc = vcpu_pc;
37+ ),
38+
39+ TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
40+);
41+
42+TRACE_EVENT(kvm_guest_fault,
43+ TP_PROTO(unsigned long vcpu_pc, unsigned long hsr,
44+ unsigned long hxfar,
45+ unsigned long long ipa),
46+ TP_ARGS(vcpu_pc, hsr, hxfar, ipa),
47+
48+ TP_STRUCT__entry(
49+ __field( unsigned long, vcpu_pc )
50+ __field( unsigned long, hsr )
51+ __field( unsigned long, hxfar )
52+ __field( unsigned long long, ipa )
53+ ),
54+
55+ TP_fast_assign(
56+ __entry->vcpu_pc = vcpu_pc;
57+ __entry->hsr = hsr;
58+ __entry->hxfar = hxfar;
59+ __entry->ipa = ipa;
60+ ),
61+
62+ TP_printk("guest fault at PC %#08lx (hxfar %#08lx, "
63+ "ipa %#16llx, hsr %#08lx",
64+ __entry->vcpu_pc, __entry->hxfar,
65+ __entry->ipa, __entry->hsr)
66+);
67+
68+TRACE_EVENT(kvm_irq_line,
69+ TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
70+ TP_ARGS(type, vcpu_idx, irq_num, level),
71+
72+ TP_STRUCT__entry(
73+ __field( unsigned int, type )
74+ __field( int, vcpu_idx )
75+ __field( int, irq_num )
76+ __field( int, level )
77+ ),
78+
79+ TP_fast_assign(
80+ __entry->type = type;
81+ __entry->vcpu_idx = vcpu_idx;
82+ __entry->irq_num = irq_num;
83+ __entry->level = level;
84+ ),
85+
86+ TP_printk("Inject %s interrupt (%d), vcpu->idx: %d, num: %d, level: %d",
87+ (__entry->type == KVM_ARM_IRQ_TYPE_CPU) ? "CPU" :
88+ (__entry->type == KVM_ARM_IRQ_TYPE_PPI) ? "VGIC PPI" :
89+ (__entry->type == KVM_ARM_IRQ_TYPE_SPI) ? "VGIC SPI" : "UNKNOWN",
90+ __entry->type, __entry->vcpu_idx, __entry->irq_num, __entry->level)
91+);
92+
93+TRACE_EVENT(kvm_mmio_emulate,
94+ TP_PROTO(unsigned long vcpu_pc, unsigned long instr,
95+ unsigned long cpsr),
96+ TP_ARGS(vcpu_pc, instr, cpsr),
97+
98+ TP_STRUCT__entry(
99+ __field( unsigned long, vcpu_pc )
100+ __field( unsigned long, instr )
101+ __field( unsigned long, cpsr )
102+ ),
103+
104+ TP_fast_assign(
105+ __entry->vcpu_pc = vcpu_pc;
106+ __entry->instr = instr;
107+ __entry->cpsr = cpsr;
108+ ),
109+
110+ TP_printk("Emulate MMIO at: 0x%08lx (instr: %08lx, cpsr: %08lx)",
111+ __entry->vcpu_pc, __entry->instr, __entry->cpsr)
112+);
113+
114+/* Architecturally implementation defined CP15 register access */
115+TRACE_EVENT(kvm_emulate_cp15_imp,
116+ TP_PROTO(unsigned long Op1, unsigned long Rt1, unsigned long CRn,
117+ unsigned long CRm, unsigned long Op2, bool is_write),
118+ TP_ARGS(Op1, Rt1, CRn, CRm, Op2, is_write),
119+
120+ TP_STRUCT__entry(
121+ __field( unsigned int, Op1 )
122+ __field( unsigned int, Rt1 )
123+ __field( unsigned int, CRn )
124+ __field( unsigned int, CRm )
125+ __field( unsigned int, Op2 )
126+ __field( bool, is_write )
127+ ),
128+
129+ TP_fast_assign(
130+ __entry->is_write = is_write;
131+ __entry->Op1 = Op1;
132+ __entry->Rt1 = Rt1;
133+ __entry->CRn = CRn;
134+ __entry->CRm = CRm;
135+ __entry->Op2 = Op2;
136+ ),
137+
138+ TP_printk("Implementation defined CP15: %s\tp15, %u, r%u, c%u, c%u, %u",
139+ (__entry->is_write) ? "mcr" : "mrc",
140+ __entry->Op1, __entry->Rt1, __entry->CRn,
141+ __entry->CRm, __entry->Op2)
142+);
143+
144+TRACE_EVENT(kvm_wfi,
145+ TP_PROTO(unsigned long vcpu_pc),
146+ TP_ARGS(vcpu_pc),
147+
148+ TP_STRUCT__entry(
149+ __field( unsigned long, vcpu_pc )
150+ ),
151+
152+ TP_fast_assign(
153+ __entry->vcpu_pc = vcpu_pc;
154+ ),
155+
156+ TP_printk("guest executed wfi at: 0x%08lx", __entry->vcpu_pc)
157+);
158+
159+TRACE_EVENT(kvm_unmap_hva,
160+ TP_PROTO(unsigned long hva),
161+ TP_ARGS(hva),
162+
163+ TP_STRUCT__entry(
164+ __field( unsigned long, hva )
165+ ),
166+
167+ TP_fast_assign(
168+ __entry->hva = hva;
169+ ),
170+
171+ TP_printk("mmu notifier unmap hva: %#08lx", __entry->hva)
172+);
173+
174+TRACE_EVENT(kvm_unmap_hva_range,
175+ TP_PROTO(unsigned long start, unsigned long end),
176+ TP_ARGS(start, end),
177+
178+ TP_STRUCT__entry(
179+ __field( unsigned long, start )
180+ __field( unsigned long, end )
181+ ),
182+
183+ TP_fast_assign(
184+ __entry->start = start;
185+ __entry->end = end;
186+ ),
187+
188+ TP_printk("mmu notifier unmap range: %#08lx -- %#08lx",
189+ __entry->start, __entry->end)
190+);
191+
192+TRACE_EVENT(kvm_set_spte_hva,
193+ TP_PROTO(unsigned long hva),
194+ TP_ARGS(hva),
195+
196+ TP_STRUCT__entry(
197+ __field( unsigned long, hva )
198+ ),
199+
200+ TP_fast_assign(
201+ __entry->hva = hva;
202+ ),
203+
204+ TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
205+);
206+
207+TRACE_EVENT(kvm_hvc,
208+ TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
209+ TP_ARGS(vcpu_pc, r0, imm),
210+
211+ TP_STRUCT__entry(
212+ __field( unsigned long, vcpu_pc )
213+ __field( unsigned long, r0 )
214+ __field( unsigned long, imm )
215+ ),
216+
217+ TP_fast_assign(
218+ __entry->vcpu_pc = vcpu_pc;
219+ __entry->r0 = r0;
220+ __entry->imm = imm;
221+ ),
222+
223+ TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx",
224+ __entry->vcpu_pc, __entry->r0, __entry->imm)
225+);
226+
227+#endif /* _TRACE_KVM_H */
228+
229+#undef TRACE_INCLUDE_PATH
230+#define TRACE_INCLUDE_PATH arch/arm/kvm
231+#undef TRACE_INCLUDE_FILE
232+#define TRACE_INCLUDE_FILE trace
233+
234+/* This part must be outside protection */
235+#include <trace/define_trace.h>
arch/arm/mm/idmap.c+44−11
@@ -1,4 +1,6 @@
1+#include <linux/module.h>
12 #include <linux/kernel.h>
3+#include <linux/slab.h>
24
35 #include <asm/cputype.h>
46 #include <asm/idmap.h>
@@ -6,6 +8,7 @@ · 1 unmodified line
68 #include <asm/pgtable.h>
79 #include <asm/sections.h>
810 #include <asm/system_info.h>
11+#include <asm/virt.h>
912
1013 pgd_t *idmap_pgd;
1114
@@ -59,11 +62,17 @@ static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, · 47 unmodified lines
5962 } while (pud++, addr = next, addr != end);
6063 }
6164
62−static void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
65+static void identity_mapping_add(pgd_t *pgd, const char *text_start,
66+ const char *text_end, unsigned long prot)
6367 {
64− unsigned long prot, next;
68+ unsigned long addr, end;
69+ unsigned long next;
70+
71+ addr = virt_to_phys(text_start);
72+ end = virt_to_phys(text_end);
73+
74+ prot |= PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
6575
66− prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
6776 if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
6877 prot |= PMD_BIT4;
6978
@@ -74,28 +83,52 @@ static void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long e · 4 unmodified lines
7483 } while (pgd++, addr = next, addr != end);
7584 }
7685
86+#if defined(CONFIG_ARM_VIRT_EXT) && defined(CONFIG_ARM_LPAE)
87+pgd_t *hyp_pgd;
88+
89+extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
90+
91+static int __init init_static_idmap_hyp(void)
92+{
93+ hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
94+ if (!hyp_pgd)
95+ return -ENOMEM;
96+
97+ pr_info("Setting up static HYP identity map for 0x%p - 0x%p\n",
98+ __hyp_idmap_text_start, __hyp_idmap_text_end);
99+ identity_mapping_add(hyp_pgd, __hyp_idmap_text_start,
100+ __hyp_idmap_text_end, PMD_SECT_AP1);
101+
102+ return 0;
103+}
104+#else
105+static int __init init_static_idmap_hyp(void)
106+{
107+ return 0;
108+}
109+#endif
110+
77111 extern char __idmap_text_start[], __idmap_text_end[];
78112
79113 static int __init init_static_idmap(void)
80114 {
81− phys_addr_t idmap_start, idmap_end;
115+ int ret;
82116
83117 idmap_pgd = pgd_alloc(&init_mm);
84118 if (!idmap_pgd)
85119 return -ENOMEM;
86120
87− /* Add an identity mapping for the physical address of the section. */
88− idmap_start = virt_to_phys((void *)__idmap_text_start);
89− idmap_end = virt_to_phys((void *)__idmap_text_end);
121+ pr_info("Setting up static identity map for 0x%p - 0x%p\n",
122+ __idmap_text_start, __idmap_text_end);
123+ identity_mapping_add(idmap_pgd, __idmap_text_start,
124+ __idmap_text_end, 0);
90125
91− pr_info("Setting up static identity map for 0x%llx - 0x%llx\n",
92− (long long)idmap_start, (long long)idmap_end);
93− identity_mapping_add(idmap_pgd, idmap_start, idmap_end);
126+ ret = init_static_idmap_hyp();
94127
95128 /* Flush L1 for the hardware to see this page table content */
96129 flush_cache_louis();
97130
98− return 0;
131+ return ret;
99132 }
100133 early_initcall(init_static_idmap);
101134
arch/arm/mm/mmu.c+22−0
@@ -57,6 +57,9 @@ static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; · 56 unmodified lines
5757 static unsigned int ecc_mask __initdata = 0;
5858 pgprot_t pgprot_user;
5959 pgprot_t pgprot_kernel;
60+pgprot_t pgprot_hyp_device;
61+pgprot_t pgprot_s2;
62+pgprot_t pgprot_s2_device;
6063
6164 EXPORT_SYMBOL(pgprot_user);
6265 EXPORT_SYMBOL(pgprot_kernel);
@@ -66,34 +69,46 @@ struct cachepolicy { · 3 unmodified lines
6669 unsigned int cr_mask;
6770 pmdval_t pmd;
6871 pteval_t pte;
72+ pteval_t pte_s2;
6973 };
7074
75+#ifdef CONFIG_ARM_LPAE
76+#define s2_policy(policy) policy
77+#else
78+#define s2_policy(policy) 0
79+#endif
80+
7181 static struct cachepolicy cache_policies[] __initdata = {
7282 {
7383 .policy = "uncached",
7484 .cr_mask = CR_W|CR_C,
7585 .pmd = PMD_SECT_UNCACHED,
7686 .pte = L_PTE_MT_UNCACHED,
87+ .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
7788 }, {
7889 .policy = "buffered",
7990 .cr_mask = CR_C,
8091 .pmd = PMD_SECT_BUFFERED,
8192 .pte = L_PTE_MT_BUFFERABLE,
93+ .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
8294 }, {
8395 .policy = "writethrough",
8496 .cr_mask = 0,
8597 .pmd = PMD_SECT_WT,
8698 .pte = L_PTE_MT_WRITETHROUGH,
99+ .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH),
87100 }, {
88101 .policy = "writeback",
89102 .cr_mask = 0,
90103 .pmd = PMD_SECT_WB,
91104 .pte = L_PTE_MT_WRITEBACK,
105+ .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
92106 }, {
93107 .policy = "writealloc",
94108 .cr_mask = 0,
95109 .pmd = PMD_SECT_WBWA,
96110 .pte = L_PTE_MT_WRITEALLOC,
111+ .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
97112 }
98113 };
99114
@@ -310,6 +325,7 @@ static void __init build_mem_type_table(void) · 210 unmodified lines
310325 struct cachepolicy *cp;
311326 unsigned int cr = get_cr();
312327 pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
328+ pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
313329 int cpu_arch = cpu_architecture();
314330 int i;
315331
@@ -421,6 +437,8 @@ static void __init build_mem_type_table(void) · 105 unmodified lines
421437 */
422438 cp = &cache_policies[cachepolicy];
423439 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
440+ s2_pgprot = cp->pte_s2;
441+ hyp_device_pgprot = s2_device_pgprot = mem_types[MT_DEVICE].prot_pte;
424442
425443 /*
426444 * ARMv6 and above have extended page tables.
@@ -444,6 +462,7 @@ static void __init build_mem_type_table(void) · 17 unmodified lines
444462 user_pgprot |= L_PTE_SHARED;
445463 kern_pgprot |= L_PTE_SHARED;
446464 vecs_pgprot |= L_PTE_SHARED;
465+ s2_pgprot |= L_PTE_SHARED;
447466 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
448467 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
449468 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
@@ -498,6 +517,9 @@ static void __init build_mem_type_table(void) · 48 unmodified lines
498517 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
499518 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
500519 L_PTE_DIRTY | kern_pgprot);
520+ pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
521+ pgprot_s2_device = __pgprot(s2_device_pgprot);
522+ pgprot_hyp_device = __pgprot(hyp_device_pgprot);
501523
502524 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
503525 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
include/uapi/linux/kvm.h+9−0
@@ -115,6 +115,7 @@ struct kvm_irq_level { · 114 unmodified lines
115115 * ACPI gsi notion of irq.
116116 * For IA-64 (APIC model) IOAPIC0: irq 0-23; IOAPIC1: irq 24-47..
117117 * For X86 (standard AT mode) PIC0/1: irq 0-15. IOAPIC0: 0-23..
118+ * For ARM: See Documentation/virtual/kvm/api.txt
118119 */
119120 union {
120121 __u32 irq;
@@ -635,6 +636,7 @@ struct kvm_ppc_smmu_info { · 514 unmodified lines
635636 #define KVM_CAP_IRQFD_RESAMPLE 82
636637 #define KVM_CAP_PPC_BOOKE_WATCHDOG 83
637638 #define KVM_CAP_PPC_HTAB_FD 84
639+#define KVM_CAP_ARM_PSCI 87
638640
639641 #ifdef KVM_CAP_IRQ_ROUTING
640642
@@ -764,6 +766,11 @@ struct kvm_dirty_tlb { · 123 unmodified lines
764766 #define KVM_REG_SIZE_U512 0x0060000000000000ULL
765767 #define KVM_REG_SIZE_U1024 0x0070000000000000ULL
766768
769+struct kvm_reg_list {
770+ __u64 n; /* number of regs */
771+ __u64 reg[0];
772+};
773+
767774 struct kvm_one_reg {
768775 __u64 id;
769776 __u64 addr;
@@ -932,6 +939,8 @@ struct kvm_s390_ucas_mapping { · 162 unmodified lines
932939 #define KVM_SET_ONE_REG _IOW(KVMIO, 0xac, struct kvm_one_reg)
933940 /* VM is being stopped by host */
934941 #define KVM_KVMCLOCK_CTRL _IO(KVMIO, 0xad)
942+#define KVM_ARM_VCPU_INIT _IOW(KVMIO, 0xae, struct kvm_vcpu_init)
943+#define KVM_GET_REG_LIST _IOWR(KVMIO, 0xb0, struct kvm_reg_list)
935944
936945 #define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0)
937946 #define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t