History

Merge tag 'kvmarm-5.17' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 updates for Linux 5.16 - Simplification of the 'vcpu first run' by integrating it into KVM's 'pid change' flow - Refactoring of the FP and SVE state tracking, also leading to a simpler state and less shared data between EL1 and EL2 in the nVHE case - Tidy up the header file usage for the nvhe hyp object - New HYP unsharing mechanism, finally allowing pages to be unmapped from the Stage-1 EL2 page-tables - Various pKVM cleanups around refcounting and sharing - A couple of vgic fixes for bugs that would trigger once the vcpu xarray rework is merged, but not sooner - Add minimal support for ARMv8.7's PMU extension - Rework kvm_pgtable initialisation ahead of the NV work - New selftest for IRQ injection - Teach selftests about the lack of default IPA space and page sizes - Expand sysreg selftest to deal with Pointer Authentication - The usual bunch of cleanups and doc update

Paolo Bonzini committed 4 years ago

Commit 7fd55a02 parents 5e4e84f 1c53a1a60 files · +2532 −385Browse files

Unified · Split

60 files
  1. arch/arm64/include/asm/kvm_asm.h +1 −0
  2. arch/arm64/include/asm/kvm_emulate.h +1 −1
  3. arch/arm64/include/asm/kvm_host.h +19 −27
  4. arch/arm64/include/asm/kvm_hyp.h +0 −1
  5. arch/arm64/include/asm/kvm_mmu.h +2 −0
  6. arch/arm64/include/asm/kvm_pgtable.h +25 −5
  7. arch/arm64/include/asm/kvm_pkvm.h +71 −0
  8. arch/arm64/include/asm/mmu.h +1 −0
  9. arch/arm64/include/asm/sysreg.h +1 −0
  10. arch/arm64/kernel/asm-offsets.c +0 −1
  11. arch/arm64/kernel/fpsimd.c +5 −1
  12. arch/arm64/kvm/.gitignore +2 −0
  13. arch/arm64/kvm/Makefile +17 −1
  14. arch/arm64/kvm/arm.c +37 −27
  15. arch/arm64/kvm/fpsimd.c +41 −38
  16. arch/arm64/kvm/hyp/Makefile +1 −1
  17. arch/arm64/kvm/hyp/fpsimd.S +0 −6
  18. arch/arm64/kvm/hyp/hyp-constants.c +10 −0
  19. arch/arm64/kvm/hyp/include/hyp/switch.h +4 −26
  20. arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +6 −0
  21. arch/arm64/kvm/hyp/include/nvhe/mm.h +0 −59
  22. arch/arm64/kvm/hyp/nvhe/early_alloc.c +5 −0
  23. arch/arm64/kvm/hyp/nvhe/hyp-main.c +8 −0
  24. arch/arm64/kvm/hyp/nvhe/mem_protect.c +428 −77
  25. arch/arm64/kvm/hyp/nvhe/mm.c +3 −1
  26. arch/arm64/kvm/hyp/nvhe/page_alloc.c +1 −1
  27. arch/arm64/kvm/hyp/nvhe/setup.c +19 −6
  28. arch/arm64/kvm/hyp/nvhe/switch.c +0 −1
  29. arch/arm64/kvm/hyp/pgtable.c +85 −23
  30. arch/arm64/kvm/hyp/vhe/switch.c +0 −1
  31. arch/arm64/kvm/mmu.c +137 −13
  32. arch/arm64/kvm/pkvm.c +4 −4
  33. arch/arm64/kvm/pmu-emul.c +1 −0
  34. arch/arm64/kvm/psci.c +1 −1
  35. arch/arm64/kvm/reset.c +21 −7
  36. arch/arm64/kvm/vgic/vgic-init.c +1 −1
  37. arch/arm64/kvm/vgic/vgic-mmio-v3.c +5 −3
  38. arch/arm64/kvm/vgic/vgic-mmio.c +1 −1
  39. arch/arm64/kvm/vgic/vgic-mmio.h +1 −1
  40. arch/arm64/kvm/vgic/vgic-v2.c +7 −2
  41. arch/arm64/kvm/vgic/vgic-v3.c +3 −3
  42. include/kvm/arm_vgic.h +3 −1
  43. include/linux/kvm_host.h +0 −1
  44. tools/testing/selftests/kvm/.gitignore +1 −0
  45. tools/testing/selftests/kvm/Makefile +1 −0
  46. tools/testing/selftests/kvm/aarch64/arch_timer.c +1 −1
  47. tools/testing/selftests/kvm/aarch64/get-reg-list.c +50 −0
  48. tools/testing/selftests/kvm/aarch64/vgic_irq.c +853 −0
  49. tools/testing/selftests/kvm/include/aarch64/gic.h +26 −0
  50. tools/testing/selftests/kvm/include/aarch64/gic_v3.h +12 −0
  51. tools/testing/selftests/kvm/include/aarch64/processor.h +3 −0
  52. tools/testing/selftests/kvm/include/aarch64/vgic.h +16 −2
  53. tools/testing/selftests/kvm/include/kvm_util.h +19 −1
  54. tools/testing/selftests/kvm/lib/aarch64/gic.c +66 −0
  55. tools/testing/selftests/kvm/lib/aarch64/gic_private.h +11 −0
  56. tools/testing/selftests/kvm/lib/aarch64/gic_v3.c +181 −25
  57. tools/testing/selftests/kvm/lib/aarch64/processor.c +75 −7
  58. tools/testing/selftests/kvm/lib/aarch64/vgic.c +102 −1
  59. tools/testing/selftests/kvm/lib/guest_modes.c +43 −6
  60. tools/testing/selftests/kvm/lib/kvm_util.c +94 −0
arch/arm64/include/asm/kvm_asm.h+1−0
@@ -63,6 +63,7 @@ enum __kvm_host_smccc_func { · 62 unmodified lines
6363
6464 /* Hypercalls available after pKVM finalisation */
6565 __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
66+ __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
6667 __KVM_HOST_SMCCC_FUNC___kvm_adjust_pc,
6768 __KVM_HOST_SMCCC_FUNC___kvm_vcpu_run,
6869 __KVM_HOST_SMCCC_FUNC___kvm_flush_vm_context,
arch/arm64/include/asm/kvm_emulate.h+1−1
@@ -388,7 +388,7 @@ static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) · 387 unmodified lines
388388 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
389389 } else {
390390 u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
391− sctlr |= (1 << 25);
391+ sctlr |= SCTLR_ELx_EE;
392392 vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
393393 }
394394 }
arch/arm64/include/asm/kvm_host.h+19−27
@@ -26,7 +26,6 @@ · 25 unmodified lines
2626 #include <asm/fpsimd.h>
2727 #include <asm/kvm.h>
2828 #include <asm/kvm_asm.h>
29−#include <asm/thread_info.h>
3029
3130 #define __KVM_HAVE_ARCH_INTC_INITIALIZED
3231
@@ -298,9 +297,6 @@ struct kvm_vcpu_arch { · 265 unmodified lines
298297 /* Exception Information */
299298 struct kvm_vcpu_fault_info fault;
300299
301− /* State of various workarounds, see kvm_asm.h for bit assignment */
302− u64 workaround_flags;
303−
304300 /* Miscellaneous vcpu state flags */
305301 u64 flags;
306302
@@ -321,8 +317,8 @@ struct kvm_vcpu_arch { · 14 unmodified lines
321317 struct kvm_guest_debug_arch vcpu_debug_state;
322318 struct kvm_guest_debug_arch external_debug_state;
323319
324− struct thread_info *host_thread_info; /* hyp VA */
325320 struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */
321+ struct task_struct *parent_task;
326322
327323 struct {
328324 /* {Break,watch}point registers */
@@ -367,9 +363,6 @@ struct kvm_vcpu_arch { · 38 unmodified lines
367363 int target;
368364 DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
369365
370− /* Detect first run of a vcpu */
371− bool has_run_once;
372−
373366 /* Virtual SError ESR to restore when HCR_EL2.VSE is set */
374367 u64 vsesr_el2;
375368
@@ -411,20 +404,17 @@ struct kvm_vcpu_arch { · 35 unmodified lines
411404 #define KVM_ARM64_DEBUG_DIRTY (1 << 0)
412405 #define KVM_ARM64_FP_ENABLED (1 << 1) /* guest FP regs loaded */
413406 #define KVM_ARM64_FP_HOST (1 << 2) /* host FP regs loaded */
414−#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */
415407 #define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */
416408 #define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */
417409 #define KVM_ARM64_VCPU_SVE_FINALIZED (1 << 6) /* SVE config completed */
418410 #define KVM_ARM64_GUEST_HAS_PTRAUTH (1 << 7) /* PTRAUTH exposed to guest */
419411 #define KVM_ARM64_PENDING_EXCEPTION (1 << 8) /* Exception pending */
412+/*
413+ * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be
414+ * set together with an exception...
415+ */
416+#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */
420417 #define KVM_ARM64_EXCEPT_MASK (7 << 9) /* Target EL/MODE */
421−#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */
422−#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */
423−
424−#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
425− KVM_GUESTDBG_USE_SW_BP | \
426− KVM_GUESTDBG_USE_HW | \
427− KVM_GUESTDBG_SINGLESTEP)
428418 /*
429419 * When KVM_ARM64_PENDING_EXCEPTION is set, KVM_ARM64_EXCEPT_MASK can
430420 * take the following values:
@@ -442,11 +432,14 @@ struct kvm_vcpu_arch { · 11 unmodified lines
442432 #define KVM_ARM64_EXCEPT_AA64_EL1 (0 << 11)
443433 #define KVM_ARM64_EXCEPT_AA64_EL2 (1 << 11)
444434
445−/*
446− * Overlaps with KVM_ARM64_EXCEPT_MASK on purpose so that it can't be
447− * set together with an exception...
448− */
449−#define KVM_ARM64_INCREMENT_PC (1 << 9) /* Increment PC */
435+#define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */
436+#define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */
437+#define KVM_ARM64_FP_FOREIGN_FPSTATE (1 << 14)
438+
439+#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \
440+ KVM_GUESTDBG_USE_SW_BP | \
441+ KVM_GUESTDBG_USE_HW | \
442+ KVM_GUESTDBG_SINGLESTEP)
450443
451444 #define vcpu_has_sve(vcpu) (system_supports_sve() && \
452445 ((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))
@@ -606,6 +599,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, · 153 unmodified lines
606599 void kvm_arm_halt_guest(struct kvm *kvm);
607600 void kvm_arm_resume_guest(struct kvm *kvm);
608601
602+#define vcpu_has_run_once(vcpu) !!rcu_access_pointer((vcpu)->pid)
603+
609604 #ifndef __KVM_NVHE_HYPERVISOR__
610605 #define kvm_call_hyp_nvhe(f, ...) \
611606 ({ \
@@ -736,8 +731,10 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 124 unmodified lines
736731 /* Guest/host FPSIMD coordination helpers */
737732 int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu);
738733 void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu);
734+void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu);
739735 void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu);
740736 void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu);
737+void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu);
741738
742739 static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr)
743740 {
@@ -748,12 +745,7 @@ static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr) · 4 unmodified lines
748745 void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu);
749746 void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu);
750747
751−#ifdef CONFIG_KVM /* Avoid conflicts with core headers if CONFIG_KVM=n */
752−static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
753−{
754− return kvm_arch_vcpu_run_map_fp(vcpu);
755−}
756−
748+#ifdef CONFIG_KVM
757749 void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr);
758750 void kvm_clr_pmu_events(u32 clr);
759751
arch/arm64/include/asm/kvm_hyp.h+0−1
@@ -90,7 +90,6 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu); · 89 unmodified lines
9090
9191 void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
9292 void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
93−void __sve_save_state(void *sve_pffr, u32 *fpsr);
9493 void __sve_restore_state(void *sve_pffr, u32 *fpsr);
9594
9695 #ifndef __KVM_NVHE_HYPERVISOR__
arch/arm64/include/asm/kvm_mmu.h+2−0
@@ -150,6 +150,8 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) · 149 unmodified lines
150150 #include <asm/kvm_pgtable.h>
151151 #include <asm/stage2_pgtable.h>
152152
153+int kvm_share_hyp(void *from, void *to);
154+void kvm_unshare_hyp(void *from, void *to);
153155 int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
154156 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
155157 void __iomem **kaddr,
arch/arm64/include/asm/kvm_pgtable.h+25−5
@@ -251,6 +251,27 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt); · 250 unmodified lines
251251 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
252252 enum kvm_pgtable_prot prot);
253253
254+/**
255+ * kvm_pgtable_hyp_unmap() - Remove a mapping from a hypervisor stage-1 page-table.
256+ * @pgt: Page-table structure initialised by kvm_pgtable_hyp_init().
257+ * @addr: Virtual address from which to remove the mapping.
258+ * @size: Size of the mapping.
259+ *
260+ * The offset of @addr within a page is ignored, @size is rounded-up to
261+ * the next page boundary and @phys is rounded-down to the previous page
262+ * boundary.
263+ *
264+ * TLB invalidation is performed for each page-table entry cleared during the
265+ * unmapping operation and the reference count for the page-table page
266+ * containing the cleared entry is decremented, with unreferenced pages being
267+ * freed. The unmapping operation will stop early if it encounters either an
268+ * invalid page-table entry or a valid block mapping which maps beyond the range
269+ * being unmapped.
270+ *
271+ * Return: Number of bytes unmapped, which may be 0.
272+ */
273+u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
274+
254275 /**
255276 * kvm_get_vtcr() - Helper to construct VTCR_EL2
256277 * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
@@ -270,8 +291,7 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift); · 13 unmodified lines
270291 /**
271292 * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
272293 * @pgt: Uninitialised page-table structure to initialise.
273− * @arch: Arch-specific KVM structure representing the guest virtual
274− * machine.
294+ * @mmu: S2 MMU context for this S2 translation
275295 * @mm_ops: Memory management callbacks.
276296 * @flags: Stage-2 configuration flags.
277297 * @force_pte_cb: Function that returns true if page level mappings must
@@ -279,13 +299,13 @@ u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift); · 1 unmodified line
279299 *
280300 * Return: 0 on success, negative error code on failure.
281301 */
282−int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch,
302+int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
283303 struct kvm_pgtable_mm_ops *mm_ops,
284304 enum kvm_pgtable_stage2_flags flags,
285305 kvm_pgtable_force_pte_cb_t force_pte_cb);
286306
287−#define kvm_pgtable_stage2_init(pgt, arch, mm_ops) \
288− __kvm_pgtable_stage2_init(pgt, arch, mm_ops, 0, NULL)
307+#define kvm_pgtable_stage2_init(pgt, mmu, mm_ops) \
308+ __kvm_pgtable_stage2_init(pgt, mmu, mm_ops, 0, NULL)
289309
290310 /**
291311 * kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table.
arch/arm64/include/asm/kvm_pkvm.h+71−0
@@ -0,0 +1,71 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020 - Google LLC
4+ * Author: Quentin Perret <qperret@google.com>
5+ */
6+#ifndef __ARM64_KVM_PKVM_H__
7+#define __ARM64_KVM_PKVM_H__
8+
9+#include <linux/memblock.h>
10+#include <asm/kvm_pgtable.h>
11+
12+#define HYP_MEMBLOCK_REGIONS 128
13+
14+extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
15+extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
16+
17+static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
18+{
19+ unsigned long total = 0, i;
20+
21+ /* Provision the worst case scenario */
22+ for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
23+ nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
24+ total += nr_pages;
25+ }
26+
27+ return total;
28+}
29+
30+static inline unsigned long __hyp_pgtable_total_pages(void)
31+{
32+ unsigned long res = 0, i;
33+
34+ /* Cover all of memory with page-granularity */
35+ for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
36+ struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
37+ res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
38+ }
39+
40+ return res;
41+}
42+
43+static inline unsigned long hyp_s1_pgtable_pages(void)
44+{
45+ unsigned long res;
46+
47+ res = __hyp_pgtable_total_pages();
48+
49+ /* Allow 1 GiB for private mappings */
50+ res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
51+
52+ return res;
53+}
54+
55+static inline unsigned long host_s2_pgtable_pages(void)
56+{
57+ unsigned long res;
58+
59+ /*
60+ * Include an extra 16 pages to safely upper-bound the worst case of
61+ * concatenated pgds.
62+ */
63+ res = __hyp_pgtable_total_pages() + 16;
64+
65+ /* Allow 1 GiB for MMIO mappings */
66+ res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
67+
68+ return res;
69+}
70+
71+#endif /* __ARM64_KVM_PKVM_H__ */
arch/arm64/include/asm/mmu.h+1−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
1515 #ifndef __ASSEMBLY__
1616
1717 #include <linux/refcount.h>
18+#include <asm/cpufeature.h>
1819
1920 typedef struct {
2021 atomic64_t id;
arch/arm64/include/asm/sysreg.h+1−0
@@ -937,6 +937,7 @@ · 936 unmodified lines
937937 #define ID_AA64DFR0_PMUVER_8_1 0x4
938938 #define ID_AA64DFR0_PMUVER_8_4 0x5
939939 #define ID_AA64DFR0_PMUVER_8_5 0x6
940+#define ID_AA64DFR0_PMUVER_8_7 0x7
940941 #define ID_AA64DFR0_PMUVER_IMP_DEF 0xf
941942
942943 #define ID_AA64DFR0_PMSVER_8_2 0x1
arch/arm64/kernel/asm-offsets.c+0−1
@@ -111,7 +111,6 @@ int main(void) · 110 unmodified lines
111111 #ifdef CONFIG_KVM
112112 DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
113113 DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1));
114− DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags));
115114 DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
116115 DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs));
117116 DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1]));
arch/arm64/kernel/fpsimd.c+5−1
@@ -78,7 +78,11 @@ · 77 unmodified lines
7878 * indicate whether or not the userland FPSIMD state of the current task is
7979 * present in the registers. The flag is set unless the FPSIMD registers of this
8080 * CPU currently contain the most recent userland FPSIMD state of the current
81− * task.
81+ * task. If the task is behaving as a VMM, then this is will be managed by
82+ * KVM which will clear it to indicate that the vcpu FPSIMD state is currently
83+ * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
84+ * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and
85+ * flag the register state as invalid.
8286 *
8387 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
8488 * save the task's FPSIMD context back to task_struct from softirq context.
arch/arm64/kvm/.gitignore+2−0
@@ -0,0 +1,2 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+hyp_constants.h
arch/arm64/kvm/Makefile+17−1
@@ -13,7 +13,7 @@ obj-$(CONFIG_KVM) += hyp/ · 12 unmodified lines
1313 kvm-y += arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \
1414 inject_fault.o va_layout.o handle_exit.o \
1515 guest.o debug.o reset.o sys_regs.o \
16− vgic-sys-reg-v3.o fpsimd.o pmu.o \
16+ vgic-sys-reg-v3.o fpsimd.o pmu.o pkvm.o \
1717 arch_timer.o trng.o\
1818 vgic/vgic.o vgic/vgic-init.o \
1919 vgic/vgic-irqfd.o vgic/vgic-v2.o \
@@ -23,3 +23,19 @@ kvm-y += arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ · 3 unmodified lines
2323 vgic/vgic-its.o vgic/vgic-debug.o
2424
2525 kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o
26+
27+always-y := hyp_constants.h hyp-constants.s
28+
29+define rule_gen_hyp_constants
30+ $(call filechk,offsets,__HYP_CONSTANTS_H__)
31+endef
32+
33+CFLAGS_hyp-constants.o = -I $(srctree)/$(src)/hyp/include
34+$(obj)/hyp-constants.s: $(src)/hyp/hyp-constants.c FORCE
35+ $(call if_changed_dep,cc_s_c)
36+
37+$(obj)/hyp_constants.h: $(obj)/hyp-constants.s FORCE
38+ $(call if_changed_rule,gen_hyp_constants)
39+
40+obj-kvm := $(addprefix $(obj)/, $(kvm-y))
41+$(obj-kvm): $(obj)/hyp_constants.h
arch/arm64/kvm/arm.c+37−27
@@ -146,7 +146,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 145 unmodified lines
146146 if (ret)
147147 return ret;
148148
149− ret = create_hyp_mappings(kvm, kvm + 1, PAGE_HYP);
149+ ret = kvm_share_hyp(kvm, kvm + 1);
150150 if (ret)
151151 goto out_free_stage2_pgd;
152152
@@ -180,6 +180,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 27 unmodified lines
180180 kvm_vgic_destroy(kvm);
181181
182182 kvm_destroy_vcpus(kvm);
183+
184+ kvm_unshare_hyp(kvm, kvm + 1);
183185 }
184186
185187 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
@@ -334,7 +336,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 148 unmodified lines
334336 if (err)
335337 return err;
336338
337− return create_hyp_mappings(vcpu, vcpu + 1, PAGE_HYP);
339+ return kvm_share_hyp(vcpu, vcpu + 1);
338340 }
339341
340342 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
@@ -343,7 +345,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 2 unmodified lines
343345
344346 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
345347 {
346− if (vcpu->arch.has_run_once && unlikely(!irqchip_in_kernel(vcpu->kvm)))
348+ if (vcpu_has_run_once(vcpu) && unlikely(!irqchip_in_kernel(vcpu->kvm)))
347349 static_branch_dec(&userspace_irqchip_in_use);
348350
349351 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
@@ -561,18 +563,33 @@ static void update_vmid(struct kvm_vmid *vmid) · 211 unmodified lines
561563 spin_unlock(&kvm_vmid_lock);
562564 }
563565
564−static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
566+static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
567+{
568+ return vcpu->arch.target >= 0;
569+}
570+
571+/*
572+ * Handle both the initialisation that is being done when the vcpu is
573+ * run for the first time, as well as the updates that must be
574+ * performed each time we get a new thread dealing with this vcpu.
575+ */
576+int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
565577 {
566578 struct kvm *kvm = vcpu->kvm;
567− int ret = 0;
579+ int ret;
568580
569− if (likely(vcpu->arch.has_run_once))
570− return 0;
581+ if (!kvm_vcpu_initialized(vcpu))
582+ return -ENOEXEC;
571583
572584 if (!kvm_arm_vcpu_is_finalized(vcpu))
573585 return -EPERM;
574586
575− vcpu->arch.has_run_once = true;
587+ ret = kvm_arch_vcpu_run_map_fp(vcpu);
588+ if (ret)
589+ return ret;
590+
591+ if (likely(vcpu_has_run_once(vcpu)))
592+ return 0;
576593
577594 kvm_arm_vcpu_init_debug(vcpu);
578595
@@ -584,12 +601,6 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) · 5 unmodified lines
584601 ret = kvm_vgic_map_resources(kvm);
585602 if (ret)
586603 return ret;
587− } else {
588− /*
589− * Tell the rest of the code that there are userspace irqchip
590− * VMs in the wild.
591− */
592− static_branch_inc(&userspace_irqchip_in_use);
593604 }
594605
595606 ret = kvm_timer_enable(vcpu);
@@ -597,6 +608,16 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu) · 1 unmodified line
597608 return ret;
598609
599610 ret = kvm_arm_pmu_v3_enable(vcpu);
611+ if (ret)
612+ return ret;
613+
614+ if (!irqchip_in_kernel(kvm)) {
615+ /*
616+ * Tell the rest of the code that there are userspace irqchip
617+ * VMs in the wild.
618+ */
619+ static_branch_inc(&userspace_irqchip_in_use);
620+ }
600621
601622 /*
602623 * Initialize traps for protected VMs.
@@ -689,11 +710,6 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) · 86 unmodified lines
689710 preempt_enable();
690711 }
691712
692−static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
693−{
694− return vcpu->arch.target >= 0;
695−}
696−
697713 static void check_vcpu_requests(struct kvm_vcpu *vcpu)
698714 {
699715 if (kvm_request_pending(vcpu)) {
@@ -789,13 +805,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 89 unmodified lines
789805 struct kvm_run *run = vcpu->run;
790806 int ret;
791807
792− if (unlikely(!kvm_vcpu_initialized(vcpu)))
793− return -ENOEXEC;
794−
795− ret = kvm_vcpu_first_run_init(vcpu);
796− if (ret)
797− return ret;
798−
799808 if (run->exit_reason == KVM_EXIT_MMIO) {
800809 ret = kvm_handle_mmio_return(vcpu);
801810 if (ret)
@@ -859,6 +868,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 57 unmodified lines
859868 }
860869
861870 kvm_arm_setup_debug(vcpu);
871+ kvm_arch_vcpu_ctxflush_fp(vcpu);
862872
863873 /**************************************************************
864874 * Enter the guest
@@ -1133,7 +1143,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, · 268 unmodified lines
11331143 * need to invalidate the I-cache though, as FWB does *not*
11341144 * imply CTR_EL0.DIC.
11351145 */
1136− if (vcpu->arch.has_run_once) {
1146+ if (vcpu_has_run_once(vcpu)) {
11371147 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
11381148 stage2_unmap_vm(vcpu->kvm);
11391149 else
arch/arm64/kvm/fpsimd.c+41−38
@@ -7,7 +7,6 @@ · 6 unmodified lines
77 */
88 #include <linux/irqflags.h>
99 #include <linux/sched.h>
10−#include <linux/thread_info.h>
1110 #include <linux/kvm_host.h>
1211 #include <asm/fpsimd.h>
1312 #include <asm/kvm_asm.h>
@@ -15,6 +14,19 @@ · 1 unmodified line
1514 #include <asm/kvm_mmu.h>
1615 #include <asm/sysreg.h>
1716
17+void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu)
18+{
19+ struct task_struct *p = vcpu->arch.parent_task;
20+ struct user_fpsimd_state *fpsimd;
21+
22+ if (!is_protected_kvm_enabled() || !p)
23+ return;
24+
25+ fpsimd = &p->thread.uw.fpsimd_state;
26+ kvm_unshare_hyp(fpsimd, fpsimd + 1);
27+ put_task_struct(p);
28+}
29+
1830 /*
1931 * Called on entry to KVM_RUN unless this vcpu previously ran at least
2032 * once and the most recent prior KVM_RUN for this vcpu was called from
@@ -28,36 +40,29 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu) · 7 unmodified lines
2840 {
2941 int ret;
3042
31− struct thread_info *ti = &current->thread_info;
3243 struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;
3344
34− /*
35− * Make sure the host task thread flags and fpsimd state are
36− * visible to hyp:
37− */
38− ret = create_hyp_mappings(ti, ti + 1, PAGE_HYP);
39− if (ret)
40− goto error;
45+ kvm_vcpu_unshare_task_fp(vcpu);
4146
42− ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP);
47+ /* Make sure the host task fpsimd state is visible to hyp: */
48+ ret = kvm_share_hyp(fpsimd, fpsimd + 1);
4349 if (ret)
44− goto error;
45−
46− if (vcpu->arch.sve_state) {
47− void *sve_end;
50+ return ret;
4851
49− sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu);
52+ vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
5053
51− ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end,
52− PAGE_HYP);
53− if (ret)
54− goto error;
54+ /*
55+ * We need to keep current's task_struct pinned until its data has been
56+ * unshared with the hypervisor to make sure it is not re-used by the
57+ * kernel and donated to someone else while already shared -- see
58+ * kvm_vcpu_unshare_task_fp() for the matching put_task_struct().
59+ */
60+ if (is_protected_kvm_enabled()) {
61+ get_task_struct(current);
62+ vcpu->arch.parent_task = current;
5563 }
5664
57− vcpu->arch.host_thread_info = kern_hyp_va(ti);
58− vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
59−error:
60− return ret;
65+ return 0;
6166 }
6267
6368 /*
@@ -66,26 +71,27 @@ error: · 2 unmodified lines
6671 *
6772 * Here, we just set the correct metadata to indicate that the FPSIMD
6873 * state in the cpu regs (if any) belongs to current on the host.
69− *
70− * TIF_SVE is backed up here, since it may get clobbered with guest state.
71− * This flag is restored by kvm_arch_vcpu_put_fp(vcpu).
7274 */
7375 void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
7476 {
7577 BUG_ON(!current->mm);
78+ BUG_ON(test_thread_flag(TIF_SVE));
7679
77− vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
78− KVM_ARM64_HOST_SVE_IN_USE |
79− KVM_ARM64_HOST_SVE_ENABLED);
80+ vcpu->arch.flags &= ~KVM_ARM64_FP_ENABLED;
8081 vcpu->arch.flags |= KVM_ARM64_FP_HOST;
8182
82− if (test_thread_flag(TIF_SVE))
83− vcpu->arch.flags |= KVM_ARM64_HOST_SVE_IN_USE;
84−
8583 if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
8684 vcpu->arch.flags |= KVM_ARM64_HOST_SVE_ENABLED;
8785 }
8886
87+void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
88+{
89+ if (test_thread_flag(TIF_FOREIGN_FPSTATE))
90+ vcpu->arch.flags |= KVM_ARM64_FP_FOREIGN_FPSTATE;
91+ else
92+ vcpu->arch.flags &= ~KVM_ARM64_FP_FOREIGN_FPSTATE;
93+}
94+
8995 /*
9096 * If the guest FPSIMD state was loaded, update the host's context
9197 * tracking data mark the CPU FPSIMD regs as dirty and belonging to vcpu
@@ -115,13 +121,11 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu) · 23 unmodified lines
115121 void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
116122 {
117123 unsigned long flags;
118− bool host_has_sve = system_supports_sve();
119− bool guest_has_sve = vcpu_has_sve(vcpu);
120124
121125 local_irq_save(flags);
122126
123127 if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
124− if (guest_has_sve) {
128+ if (vcpu_has_sve(vcpu)) {
125129 __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
126130
127131 /* Restore the VL that was saved when bound to the CPU */
@@ -131,7 +135,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) · 3 unmodified lines
131135 }
132136
133137 fpsimd_save_and_flush_cpu_state();
134− } else if (has_vhe() && host_has_sve) {
138+ } else if (has_vhe() && system_supports_sve()) {
135139 /*
136140 * The FPSIMD/SVE state in the CPU has not been touched, and we
137141 * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
@@ -145,8 +149,7 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) · 7 unmodified lines
145149 sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
146150 }
147151
148− update_thread_flag(TIF_SVE,
149− vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE);
152+ update_thread_flag(TIF_SVE, 0);
150153
151154 local_irq_restore(flags);
152155 }
arch/arm64/kvm/hyp/Makefile+1−1
@@ -10,4 +10,4 @@ subdir-ccflags-y := -I$(incdir) \ · 9 unmodified lines
1010 -DDISABLE_BRANCH_PROFILING \
1111 $(DISABLE_STACKLEAK_PLUGIN)
1212
13−obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o reserved_mem.o
13+obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o
arch/arm64/kvm/hyp/fpsimd.S+0−6
@@ -25,9 +25,3 @@ SYM_FUNC_START(__sve_restore_state) · 24 unmodified lines
2525 sve_load 0, x1, x2, 3
2626 ret
2727 SYM_FUNC_END(__sve_restore_state)
28−
29−SYM_FUNC_START(__sve_save_state)
30− mov x2, #1
31− sve_save 0, x1, x2, 3
32− ret
33−SYM_FUNC_END(__sve_save_state)
arch/arm64/kvm/hyp/hyp-constants.c+10−0
@@ -0,0 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+
3+#include <linux/kbuild.h>
4+#include <nvhe/memory.h>
5+
6+int main(void)
7+{
8+ DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));
9+ return 0;
10+}
arch/arm64/kvm/hyp/include/hyp/switch.h+4−26
@@ -29,7 +29,6 @@ · 28 unmodified lines
2929 #include <asm/fpsimd.h>
3030 #include <asm/debug-monitors.h>
3131 #include <asm/processor.h>
32−#include <asm/thread_info.h>
3332
3433 struct kvm_exception_table_entry {
3534 int insn, fixup;
@@ -49,7 +48,7 @@ static inline bool update_fp_enabled(struct kvm_vcpu *vcpu) · 13 unmodified lines
4948 * trap the accesses.
5049 */
5150 if (!system_supports_fpsimd() ||
52− vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
51+ vcpu->arch.flags & KVM_ARM64_FP_FOREIGN_FPSTATE)
5352 vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
5453 KVM_ARM64_FP_HOST);
5554
@@ -143,16 +142,6 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) · 87 unmodified lines
143142 return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
144143 }
145144
146−static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu)
147−{
148− struct thread_struct *thread;
149−
150− thread = container_of(vcpu->arch.host_fpsimd_state, struct thread_struct,
151− uw.fpsimd_state);
152−
153− __sve_save_state(sve_pffr(thread), &vcpu->arch.host_fpsimd_state->fpsr);
154−}
155−
156145 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
157146 {
158147 sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
@@ -169,21 +158,14 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu) · 10 unmodified lines
169158 */
170159 static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
171160 {
172− bool sve_guest, sve_host;
161+ bool sve_guest;
173162 u8 esr_ec;
174163 u64 reg;
175164
176165 if (!system_supports_fpsimd())
177166 return false;
178167
179− if (system_supports_sve()) {
180− sve_guest = vcpu_has_sve(vcpu);
181− sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE;
182− } else {
183− sve_guest = false;
184− sve_host = false;
185− }
186−
168+ sve_guest = vcpu_has_sve(vcpu);
187169 esr_ec = kvm_vcpu_trap_get_class(vcpu);
188170
189171 /* Don't handle SVE traps for non-SVE vcpus here: */
@@ -207,11 +189,7 @@ static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) · 17 unmodified lines
207189 isb();
208190
209191 if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
210− if (sve_host)
211− __hyp_sve_save_host(vcpu);
212− else
213− __fpsimd_save_state(vcpu->arch.host_fpsimd_state);
214−
192+ __fpsimd_save_state(vcpu->arch.host_fpsimd_state);
215193 vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
216194 }
217195
arch/arm64/kvm/hyp/include/nvhe/mem_protect.h+6−0
@@ -24,6 +24,11 @@ enum pkvm_page_state { · 23 unmodified lines
2424 PKVM_PAGE_OWNED = 0ULL,
2525 PKVM_PAGE_SHARED_OWNED = KVM_PGTABLE_PROT_SW0,
2626 PKVM_PAGE_SHARED_BORROWED = KVM_PGTABLE_PROT_SW1,
27+ __PKVM_PAGE_RESERVED = KVM_PGTABLE_PROT_SW0 |
28+ KVM_PGTABLE_PROT_SW1,
29+
30+ /* Meta-states which aren't encoded directly in the PTE's SW bits */
31+ PKVM_NOPAGE,
2732 };
2833
2934 #define PKVM_PAGE_STATE_PROT_MASK (KVM_PGTABLE_PROT_SW0 | KVM_PGTABLE_PROT_SW1)
@@ -50,6 +55,7 @@ extern const u8 pkvm_hyp_id; · 20 unmodified lines
5055
5156 int __pkvm_prot_finalize(void);
5257 int __pkvm_host_share_hyp(u64 pfn);
58+int __pkvm_host_unshare_hyp(u64 pfn);
5359
5460 bool addr_is_memory(phys_addr_t phys);
5561 int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);
arch/arm64/kvm/hyp/include/nvhe/mm.h+0−59
@@ -10,13 +10,8 @@ · 9 unmodified lines
1010 #include <nvhe/memory.h>
1111 #include <nvhe/spinlock.h>
1212
13−#define HYP_MEMBLOCK_REGIONS 128
14−extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
15−extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
1613 extern struct kvm_pgtable pkvm_pgtable;
1714 extern hyp_spinlock_t pkvm_pgd_lock;
18−extern struct hyp_pool hpool;
19−extern u64 __io_map_base;
2015
2116 int hyp_create_idmap(u32 hyp_va_bits);
2217 int hyp_map_vectors(void);
@@ -39,58 +34,4 @@ static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, · 16 unmodified lines
3934 *end = ALIGN(*end, PAGE_SIZE);
4035 }
4136
42−static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
43−{
44− unsigned long total = 0, i;
45−
46− /* Provision the worst case scenario */
47− for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
48− nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
49− total += nr_pages;
50− }
51−
52− return total;
53−}
54−
55−static inline unsigned long __hyp_pgtable_total_pages(void)
56−{
57− unsigned long res = 0, i;
58−
59− /* Cover all of memory with page-granularity */
60− for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
61− struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
62− res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
63− }
64−
65− return res;
66−}
67−
68−static inline unsigned long hyp_s1_pgtable_pages(void)
69−{
70− unsigned long res;
71−
72− res = __hyp_pgtable_total_pages();
73−
74− /* Allow 1 GiB for private mappings */
75− res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
76−
77− return res;
78−}
79−
80−static inline unsigned long host_s2_pgtable_pages(void)
81−{
82− unsigned long res;
83−
84− /*
85− * Include an extra 16 pages to safely upper-bound the worst case of
86− * concatenated pgds.
87− */
88− res = __hyp_pgtable_total_pages() + 16;
89−
90− /* Allow 1 GiB for MMIO mappings */
91− res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
92−
93− return res;
94−}
95−
9637 #endif /* __KVM_HYP_MM_H */
arch/arm64/kvm/hyp/nvhe/early_alloc.c+5−0
@@ -43,6 +43,9 @@ void *hyp_early_alloc_page(void *arg) · 42 unmodified lines
4343 return hyp_early_alloc_contig(1);
4444 }
4545
46+static void hyp_early_alloc_get_page(void *addr) { }
47+static void hyp_early_alloc_put_page(void *addr) { }
48+
4649 void hyp_early_alloc_init(void *virt, unsigned long size)
4750 {
4851 base = cur = (unsigned long)virt;
@@ -51,4 +54,6 @@ void hyp_early_alloc_init(void *virt, unsigned long size) · 2 unmodified lines
5154 hyp_early_alloc_mm_ops.zalloc_page = hyp_early_alloc_page;
5255 hyp_early_alloc_mm_ops.phys_to_virt = hyp_phys_to_virt;
5356 hyp_early_alloc_mm_ops.virt_to_phys = hyp_virt_to_phys;
57+ hyp_early_alloc_mm_ops.get_page = hyp_early_alloc_get_page;
58+ hyp_early_alloc_mm_ops.put_page = hyp_early_alloc_put_page;
5459 }
arch/arm64/kvm/hyp/nvhe/hyp-main.c+8−0
@@ -147,6 +147,13 @@ static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) · 146 unmodified lines
147147 cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn);
148148 }
149149
150+static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt)
151+{
152+ DECLARE_REG(u64, pfn, host_ctxt, 1);
153+
154+ cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn);
155+}
156+
150157 static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
151158 {
152159 DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
@@ -184,6 +191,7 @@ static const hcall_t host_hcall[] = { · 31 unmodified lines
184191 HANDLE_FUNC(__pkvm_prot_finalize),
185192
186193 HANDLE_FUNC(__pkvm_host_share_hyp),
194+ HANDLE_FUNC(__pkvm_host_unshare_hyp),
187195 HANDLE_FUNC(__kvm_adjust_pc),
188196 HANDLE_FUNC(__kvm_vcpu_run),
189197 HANDLE_FUNC(__kvm_flush_vm_context),
arch/arm64/kvm/hyp/nvhe/mem_protect.c+428−77
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <asm/kvm_hyp.h>
1010 #include <asm/kvm_mmu.h>
1111 #include <asm/kvm_pgtable.h>
12+#include <asm/kvm_pkvm.h>
1213 #include <asm/stage2_pgtable.h>
1314
1415 #include <hyp/fault.h>
@@ -27,6 +28,26 @@ static struct hyp_pool host_s2_pool; · 12 unmodified lines
2728
2829 const u8 pkvm_hyp_id = 1;
2930
31+static void host_lock_component(void)
32+{
33+ hyp_spin_lock(&host_kvm.lock);
34+}
35+
36+static void host_unlock_component(void)
37+{
38+ hyp_spin_unlock(&host_kvm.lock);
39+}
40+
41+static void hyp_lock_component(void)
42+{
43+ hyp_spin_lock(&pkvm_pgd_lock);
44+}
45+
46+static void hyp_unlock_component(void)
47+{
48+ hyp_spin_unlock(&pkvm_pgd_lock);
49+}
50+
3051 static void *host_s2_zalloc_pages_exact(size_t size)
3152 {
3253 void *addr = hyp_alloc_pages(&host_s2_pool, get_order(size));
@@ -103,19 +124,19 @@ int kvm_host_prepare_stage2(void *pgt_pool_base) · 70 unmodified lines
103124
104125 prepare_host_vtcr();
105126 hyp_spin_lock_init(&host_kvm.lock);
127+ mmu->arch = &host_kvm.arch;
106128
107129 ret = prepare_s2_pool(pgt_pool_base);
108130 if (ret)
109131 return ret;
110132
111− ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, &host_kvm.arch,
133+ ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, mmu,
112134 &host_kvm.mm_ops, KVM_HOST_S2_FLAGS,
113135 host_stage2_force_pte_cb);
114136 if (ret)
115137 return ret;
116138
117139 mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);
118− mmu->arch = &host_kvm.arch;
119140 mmu->pgt = &host_kvm.pgt;
120141 WRITE_ONCE(mmu->vmid.vmid_gen, 0);
121142 WRITE_ONCE(mmu->vmid.vmid, 0);
@@ -338,116 +359,446 @@ static int host_stage2_idmap(u64 addr) · 216 unmodified lines
338359
339360 prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT;
340361
341− hyp_spin_lock(&host_kvm.lock);
362+ host_lock_component();
342363 ret = host_stage2_adjust_range(addr, &range);
343364 if (ret)
344365 goto unlock;
345366
346367 ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot);
347368 unlock:
348− hyp_spin_unlock(&host_kvm.lock);
369+ host_unlock_component();
349370
350371 return ret;
351372 }
352373
353−static inline bool check_prot(enum kvm_pgtable_prot prot,
354− enum kvm_pgtable_prot required,
355− enum kvm_pgtable_prot denied)
374+void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
356375 {
357− return (prot & (required | denied)) == required;
376+ struct kvm_vcpu_fault_info fault;
377+ u64 esr, addr;
378+ int ret = 0;
379+
380+ esr = read_sysreg_el2(SYS_ESR);
381+ BUG_ON(!__get_fault_info(esr, &fault));
382+
383+ addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
384+ ret = host_stage2_idmap(addr);
385+ BUG_ON(ret && ret != -EAGAIN);
358386 }
359387
360−int __pkvm_host_share_hyp(u64 pfn)
388+/* This corresponds to locking order */
389+enum pkvm_component_id {
390+ PKVM_ID_HOST,
391+ PKVM_ID_HYP,
392+};
393+
394+struct pkvm_mem_transition {
395+ u64 nr_pages;
396+
397+ struct {
398+ enum pkvm_component_id id;
399+ /* Address in the initiator's address space */
400+ u64 addr;
401+
402+ union {
403+ struct {
404+ /* Address in the completer's address space */
405+ u64 completer_addr;
406+ } host;
407+ };
408+ } initiator;
409+
410+ struct {
411+ enum pkvm_component_id id;
412+ } completer;
413+};
414+
415+struct pkvm_mem_share {
416+ const struct pkvm_mem_transition tx;
417+ const enum kvm_pgtable_prot completer_prot;
418+};
419+
420+struct check_walk_data {
421+ enum pkvm_page_state desired;
422+ enum pkvm_page_state (*get_page_state)(kvm_pte_t pte);
423+};
424+
425+static int __check_page_state_visitor(u64 addr, u64 end, u32 level,
426+ kvm_pte_t *ptep,
427+ enum kvm_pgtable_walk_flags flag,
428+ void * const arg)
361429 {
362− phys_addr_t addr = hyp_pfn_to_phys(pfn);
363− enum kvm_pgtable_prot prot, cur;
364− void *virt = __hyp_va(addr);
365− enum pkvm_page_state state;
366− kvm_pte_t pte;
367− int ret;
430+ struct check_walk_data *d = arg;
431+ kvm_pte_t pte = *ptep;
368432
369− if (!addr_is_memory(addr))
433+ if (kvm_pte_valid(pte) && !addr_is_memory(kvm_pte_to_phys(pte)))
370434 return -EINVAL;
371435
372− hyp_spin_lock(&host_kvm.lock);
373− hyp_spin_lock(&pkvm_pgd_lock);
436+ return d->get_page_state(pte) == d->desired ? 0 : -EPERM;
437+}
438+
439+static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,
440+ struct check_walk_data *data)
441+{
442+ struct kvm_pgtable_walker walker = {
443+ .cb = __check_page_state_visitor,
444+ .arg = data,
445+ .flags = KVM_PGTABLE_WALK_LEAF,
446+ };
447+
448+ return kvm_pgtable_walk(pgt, addr, size, &walker);
449+}
450+
451+static enum pkvm_page_state host_get_page_state(kvm_pte_t pte)
452+{
453+ if (!kvm_pte_valid(pte) && pte)
454+ return PKVM_NOPAGE;
455+
456+ return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte));
457+}
458+
459+static int __host_check_page_state_range(u64 addr, u64 size,
460+ enum pkvm_page_state state)
461+{
462+ struct check_walk_data d = {
463+ .desired = state,
464+ .get_page_state = host_get_page_state,
465+ };
466+
467+ hyp_assert_lock_held(&host_kvm.lock);
468+ return check_page_state_range(&host_kvm.pgt, addr, size, &d);
469+}
470+
471+static int __host_set_page_state_range(u64 addr, u64 size,
472+ enum pkvm_page_state state)
473+{
474+ enum kvm_pgtable_prot prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, state);
475+
476+ return host_stage2_idmap_locked(addr, size, prot);
477+}
478+
479+static int host_request_owned_transition(u64 *completer_addr,
480+ const struct pkvm_mem_transition *tx)
481+{
482+ u64 size = tx->nr_pages * PAGE_SIZE;
483+ u64 addr = tx->initiator.addr;
484+
485+ *completer_addr = tx->initiator.host.completer_addr;
486+ return __host_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
487+}
488+
489+static int host_request_unshare(u64 *completer_addr,
490+ const struct pkvm_mem_transition *tx)
491+{
492+ u64 size = tx->nr_pages * PAGE_SIZE;
493+ u64 addr = tx->initiator.addr;
494+
495+ *completer_addr = tx->initiator.host.completer_addr;
496+ return __host_check_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
497+}
498+
499+static int host_initiate_share(u64 *completer_addr,
500+ const struct pkvm_mem_transition *tx)
501+{
502+ u64 size = tx->nr_pages * PAGE_SIZE;
503+ u64 addr = tx->initiator.addr;
504+
505+ *completer_addr = tx->initiator.host.completer_addr;
506+ return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
507+}
508+
509+static int host_initiate_unshare(u64 *completer_addr,
510+ const struct pkvm_mem_transition *tx)
511+{
512+ u64 size = tx->nr_pages * PAGE_SIZE;
513+ u64 addr = tx->initiator.addr;
514+
515+ *completer_addr = tx->initiator.host.completer_addr;
516+ return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
517+}
518+
519+static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte)
520+{
521+ if (!kvm_pte_valid(pte))
522+ return PKVM_NOPAGE;
523+
524+ return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte));
525+}
526+
527+static int __hyp_check_page_state_range(u64 addr, u64 size,
528+ enum pkvm_page_state state)
529+{
530+ struct check_walk_data d = {
531+ .desired = state,
532+ .get_page_state = hyp_get_page_state,
533+ };
534+
535+ hyp_assert_lock_held(&pkvm_pgd_lock);
536+ return check_page_state_range(&pkvm_pgtable, addr, size, &d);
537+}
538+
539+static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
540+{
541+ return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
542+ tx->initiator.id != PKVM_ID_HOST);
543+}
544+
545+static int hyp_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
546+ enum kvm_pgtable_prot perms)
547+{
548+ u64 size = tx->nr_pages * PAGE_SIZE;
549+
550+ if (perms != PAGE_HYP)
551+ return -EPERM;
552+
553+ if (__hyp_ack_skip_pgtable_check(tx))
554+ return 0;
555+
556+ return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
557+}
558+
559+static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx)
560+{
561+ u64 size = tx->nr_pages * PAGE_SIZE;
562+
563+ if (__hyp_ack_skip_pgtable_check(tx))
564+ return 0;
565+
566+ return __hyp_check_page_state_range(addr, size,
567+ PKVM_PAGE_SHARED_BORROWED);
568+}
569+
570+static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
571+ enum kvm_pgtable_prot perms)
572+{
573+ void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
574+ enum kvm_pgtable_prot prot;
575+
576+ prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED);
577+ return pkvm_create_mappings_locked(start, end, prot);
578+}
579+
580+static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx)
581+{
582+ u64 size = tx->nr_pages * PAGE_SIZE;
583+ int ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size);
584+
585+ return (ret != size) ? -EFAULT : 0;
586+}
587+
588+static int check_share(struct pkvm_mem_share *share)
589+{
590+ const struct pkvm_mem_transition *tx = &share->tx;
591+ u64 completer_addr;
592+ int ret;
593+
594+ switch (tx->initiator.id) {
595+ case PKVM_ID_HOST:
596+ ret = host_request_owned_transition(&completer_addr, tx);
597+ break;
598+ default:
599+ ret = -EINVAL;
600+ }
374601
375− ret = kvm_pgtable_get_leaf(&host_kvm.pgt, addr, &pte, NULL);
376602 if (ret)
377− goto unlock;
378− if (!pte)
379− goto map_shared;
603+ return ret;
380604
381− /*
382− * Check attributes in the host stage-2 PTE. We need the page to be:
383− * - mapped RWX as we're sharing memory;
384− * - not borrowed, as that implies absence of ownership.
385− * Otherwise, we can't let it got through
386− */
387− cur = kvm_pgtable_stage2_pte_prot(pte);
388− prot = pkvm_mkstate(0, PKVM_PAGE_SHARED_BORROWED);
389− if (!check_prot(cur, PKVM_HOST_MEM_PROT, prot)) {
390− ret = -EPERM;
391− goto unlock;
605+ switch (tx->completer.id) {
606+ case PKVM_ID_HYP:
607+ ret = hyp_ack_share(completer_addr, tx, share->completer_prot);
608+ break;
609+ default:
610+ ret = -EINVAL;
392611 }
393612
394− state = pkvm_getstate(cur);
395− if (state == PKVM_PAGE_OWNED)
396− goto map_shared;
613+ return ret;
614+}
397615
398− /*
399− * Tolerate double-sharing the same page, but this requires
400− * cross-checking the hypervisor stage-1.
401− */
402− if (state != PKVM_PAGE_SHARED_OWNED) {
403− ret = -EPERM;
404− goto unlock;
616+static int __do_share(struct pkvm_mem_share *share)
617+{
618+ const struct pkvm_mem_transition *tx = &share->tx;
619+ u64 completer_addr;
620+ int ret;
621+
622+ switch (tx->initiator.id) {
623+ case PKVM_ID_HOST:
624+ ret = host_initiate_share(&completer_addr, tx);
625+ break;
626+ default:
627+ ret = -EINVAL;
405628 }
406629
407− ret = kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)virt, &pte, NULL);
408630 if (ret)
409− goto unlock;
631+ return ret;
410632
411− /*
412− * If the page has been shared with the hypervisor, it must be
413− * already mapped as SHARED_BORROWED in its stage-1.
414− */
415− cur = kvm_pgtable_hyp_pte_prot(pte);
416− prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED);
417− if (!check_prot(cur, prot, ~prot))
418− ret = -EPERM;
419− goto unlock;
633+ switch (tx->completer.id) {
634+ case PKVM_ID_HYP:
635+ ret = hyp_complete_share(completer_addr, tx, share->completer_prot);
636+ break;
637+ default:
638+ ret = -EINVAL;
639+ }
420640
421−map_shared:
422− /*
423− * If the page is not yet shared, adjust mappings in both page-tables
424− * while both locks are held.
425− */
426− prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_SHARED_BORROWED);
427− ret = pkvm_create_mappings_locked(virt, virt + PAGE_SIZE, prot);
428− BUG_ON(ret);
641+ return ret;
642+}
643+
644+/*
645+ * do_share():
646+ *
647+ * The page owner grants access to another component with a given set
648+ * of permissions.
649+ *
650+ * Initiator: OWNED => SHARED_OWNED
651+ * Completer: NOPAGE => SHARED_BORROWED
652+ */
653+static int do_share(struct pkvm_mem_share *share)
654+{
655+ int ret;
429656
430− prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_OWNED);
431− ret = host_stage2_idmap_locked(addr, PAGE_SIZE, prot);
432− BUG_ON(ret);
657+ ret = check_share(share);
658+ if (ret)
659+ return ret;
433660
434−unlock:
435− hyp_spin_unlock(&pkvm_pgd_lock);
436− hyp_spin_unlock(&host_kvm.lock);
661+ return WARN_ON(__do_share(share));
662+}
663+
664+static int check_unshare(struct pkvm_mem_share *share)
665+{
666+ const struct pkvm_mem_transition *tx = &share->tx;
667+ u64 completer_addr;
668+ int ret;
669+
670+ switch (tx->initiator.id) {
671+ case PKVM_ID_HOST:
672+ ret = host_request_unshare(&completer_addr, tx);
673+ break;
674+ default:
675+ ret = -EINVAL;
676+ }
677+
678+ if (ret)
679+ return ret;
680+
681+ switch (tx->completer.id) {
682+ case PKVM_ID_HYP:
683+ ret = hyp_ack_unshare(completer_addr, tx);
684+ break;
685+ default:
686+ ret = -EINVAL;
687+ }
437688
438689 return ret;
439690 }
440691
441−void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
692+static int __do_unshare(struct pkvm_mem_share *share)
442693 {
443− struct kvm_vcpu_fault_info fault;
444− u64 esr, addr;
445− int ret = 0;
694+ const struct pkvm_mem_transition *tx = &share->tx;
695+ u64 completer_addr;
696+ int ret;
446697
447− esr = read_sysreg_el2(SYS_ESR);
448− BUG_ON(!__get_fault_info(esr, &fault));
698+ switch (tx->initiator.id) {
699+ case PKVM_ID_HOST:
700+ ret = host_initiate_unshare(&completer_addr, tx);
701+ break;
702+ default:
703+ ret = -EINVAL;
704+ }
449705
450− addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
451− ret = host_stage2_idmap(addr);
452− BUG_ON(ret && ret != -EAGAIN);
706+ if (ret)
707+ return ret;
708+
709+ switch (tx->completer.id) {
710+ case PKVM_ID_HYP:
711+ ret = hyp_complete_unshare(completer_addr, tx);
712+ break;
713+ default:
714+ ret = -EINVAL;
715+ }
716+
717+ return ret;
718+}
719+
720+/*
721+ * do_unshare():
722+ *
723+ * The page owner revokes access from another component for a range of
724+ * pages which were previously shared using do_share().
725+ *
726+ * Initiator: SHARED_OWNED => OWNED
727+ * Completer: SHARED_BORROWED => NOPAGE
728+ */
729+static int do_unshare(struct pkvm_mem_share *share)
730+{
731+ int ret;
732+
733+ ret = check_unshare(share);
734+ if (ret)
735+ return ret;
736+
737+ return WARN_ON(__do_unshare(share));
738+}
739+
740+int __pkvm_host_share_hyp(u64 pfn)
741+{
742+ int ret;
743+ u64 host_addr = hyp_pfn_to_phys(pfn);
744+ u64 hyp_addr = (u64)__hyp_va(host_addr);
745+ struct pkvm_mem_share share = {
746+ .tx = {
747+ .nr_pages = 1,
748+ .initiator = {
749+ .id = PKVM_ID_HOST,
750+ .addr = host_addr,
751+ .host = {
752+ .completer_addr = hyp_addr,
753+ },
754+ },
755+ .completer = {
756+ .id = PKVM_ID_HYP,
757+ },
758+ },
759+ .completer_prot = PAGE_HYP,
760+ };
761+
762+ host_lock_component();
763+ hyp_lock_component();
764+
765+ ret = do_share(&share);
766+
767+ hyp_unlock_component();
768+ host_unlock_component();
769+
770+ return ret;
771+}
772+
773+int __pkvm_host_unshare_hyp(u64 pfn)
774+{
775+ int ret;
776+ u64 host_addr = hyp_pfn_to_phys(pfn);
777+ u64 hyp_addr = (u64)__hyp_va(host_addr);
778+ struct pkvm_mem_share share = {
779+ .tx = {
780+ .nr_pages = 1,
781+ .initiator = {
782+ .id = PKVM_ID_HOST,
783+ .addr = host_addr,
784+ .host = {
785+ .completer_addr = hyp_addr,
786+ },
787+ },
788+ .completer = {
789+ .id = PKVM_ID_HYP,
790+ },
791+ },
792+ .completer_prot = PAGE_HYP,
793+ };
794+
795+ host_lock_component();
796+ hyp_lock_component();
797+
798+ ret = do_unshare(&share);
799+
800+ hyp_unlock_component();
801+ host_unlock_component();
802+
803+ return ret;
453804 }
arch/arm64/kvm/hyp/nvhe/mm.c+3−1
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <asm/kvm_hyp.h>
99 #include <asm/kvm_mmu.h>
1010 #include <asm/kvm_pgtable.h>
11+#include <asm/kvm_pkvm.h>
1112 #include <asm/spectre.h>
1213
1314 #include <nvhe/early_alloc.h>
@@ -18,11 +19,12 @@ · 4 unmodified lines
1819
1920 struct kvm_pgtable pkvm_pgtable;
2021 hyp_spinlock_t pkvm_pgd_lock;
21−u64 __io_map_base;
2222
2323 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
2424 unsigned int hyp_memblock_nr;
2525
26+static u64 __io_map_base;
27+
2628 static int __pkvm_create_mappings(unsigned long start, unsigned long size,
2729 unsigned long phys, enum kvm_pgtable_prot prot)
2830 {
arch/arm64/kvm/hyp/nvhe/page_alloc.c+1−1
@@ -241,7 +241,7 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, · 240 unmodified lines
241241 int i;
242242
243243 hyp_spin_lock_init(&pool->lock);
244− pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT));
244+ pool->max_order = min(MAX_ORDER, get_order((nr_pages + 1) << PAGE_SHIFT));
245245 for (i = 0; i < pool->max_order; i++)
246246 INIT_LIST_HEAD(&pool->free_area[i]);
247247 pool->range_start = phys;
arch/arm64/kvm/hyp/nvhe/setup.c+19−6
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <asm/kvm_hyp.h>
99 #include <asm/kvm_mmu.h>
1010 #include <asm/kvm_pgtable.h>
11+#include <asm/kvm_pkvm.h>
1112
1213 #include <nvhe/early_alloc.h>
1314 #include <nvhe/fixed_config.h>
@@ -17,7 +18,6 @@ · 3 unmodified lines
1718 #include <nvhe/mm.h>
1819 #include <nvhe/trap_handler.h>
1920
20−struct hyp_pool hpool;
2121 unsigned long hyp_nr_cpus;
2222
2323 #define hyp_percpu_size ((unsigned long)__per_cpu_end - \
@@ -27,6 +27,7 @@ static void *vmemmap_base; · 3 unmodified lines
2727 static void *hyp_pgt_base;
2828 static void *host_s2_pgt_base;
2929 static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
30+static struct hyp_pool hpool;
3031
3132 static int divide_memory_pool(void *virt, unsigned long size)
3233 {
@@ -165,6 +166,7 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 132 unmodified lines
165166 enum kvm_pgtable_walk_flags flag,
166167 void * const arg)
167168 {
169+ struct kvm_pgtable_mm_ops *mm_ops = arg;
168170 enum kvm_pgtable_prot prot;
169171 enum pkvm_page_state state;
170172 kvm_pte_t pte = *ptep;
@@ -173,6 +175,15 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 2 unmodified lines
173175 if (!kvm_pte_valid(pte))
174176 return 0;
175177
178+ /*
179+ * Fix-up the refcount for the page-table pages as the early allocator
180+ * was unable to access the hyp_vmemmap and so the buddy allocator has
181+ * initialised the refcount to '1'.
182+ */
183+ mm_ops->get_page(ptep);
184+ if (flag != KVM_PGTABLE_WALK_LEAF)
185+ return 0;
186+
176187 if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
177188 return -EINVAL;
178189
@@ -205,7 +216,8 @@ static int finalize_host_mappings(void) · 26 unmodified lines
205216 {
206217 struct kvm_pgtable_walker walker = {
207218 .cb = finalize_host_mappings_walker,
208− .flags = KVM_PGTABLE_WALK_LEAF,
219+ .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
220+ .arg = pkvm_pgtable.mm_ops,
209221 };
210222 int i, ret;
211223
@@ -240,19 +252,20 @@ void __noreturn __pkvm_init_finalise(void) · 28 unmodified lines
240252 if (ret)
241253 goto out;
242254
243− ret = finalize_host_mappings();
244− if (ret)
245− goto out;
246−
247255 pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) {
248256 .zalloc_page = hyp_zalloc_hyp_page,
249257 .phys_to_virt = hyp_phys_to_virt,
250258 .virt_to_phys = hyp_virt_to_phys,
251259 .get_page = hpool_get_page,
252260 .put_page = hpool_put_page,
261+ .page_count = hyp_page_count,
253262 };
254263 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
255264
265+ ret = finalize_host_mappings();
266+ if (ret)
267+ goto out;
268+
256269 out:
257270 /*
258271 * We tail-called to here from handle___pkvm_init() and will not return,
arch/arm64/kvm/hyp/nvhe/switch.c+0−1
@@ -25,7 +25,6 @@ · 24 unmodified lines
2525 #include <asm/fpsimd.h>
2626 #include <asm/debug-monitors.h>
2727 #include <asm/processor.h>
28−#include <asm/thread_info.h>
2928
3029 #include <nvhe/fixed_config.h>
3130 #include <nvhe/mem_protect.h>
arch/arm64/kvm/hyp/pgtable.c+85−23
@@ -383,21 +383,6 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte) · 382 unmodified lines
383383 return prot;
384384 }
385385
386−static bool hyp_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
387−{
388− /*
389− * Tolerate KVM recreating the exact same mapping, or changing software
390− * bits if the existing mapping was valid.
391− */
392− if (old == new)
393− return false;
394−
395− if (!kvm_pte_valid(old))
396− return true;
397−
398− return !WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW);
399−}
400−
401386 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
402387 kvm_pte_t *ptep, struct hyp_map_data *data)
403388 {
@@ -407,11 +392,16 @@ static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, · 3 unmodified lines
407392 if (!kvm_block_mapping_supported(addr, end, phys, level))
408393 return false;
409394
395+ data->phys += granule;
410396 new = kvm_init_valid_leaf_pte(phys, data->attr, level);
411− if (hyp_pte_needs_update(old, new))
412− smp_store_release(ptep, new);
397+ if (old == new)
398+ return true;
399+ if (!kvm_pte_valid(old))
400+ data->mm_ops->get_page(ptep);
401+ else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
402+ return false;
413403
414− data->phys += granule;
404+ smp_store_release(ptep, new);
415405 return true;
416406 }
417407
@@ -433,6 +423,7 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 15 unmodified lines
433423 return -ENOMEM;
434424
435425 kvm_set_table_pte(ptep, childp, mm_ops);
426+ mm_ops->get_page(ptep);
436427 return 0;
437428 }
438429
@@ -460,6 +451,69 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, · 21 unmodified lines
460451 return ret;
461452 }
462453
454+struct hyp_unmap_data {
455+ u64 unmapped;
456+ struct kvm_pgtable_mm_ops *mm_ops;
457+};
458+
459+static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
460+ enum kvm_pgtable_walk_flags flag, void * const arg)
461+{
462+ kvm_pte_t pte = *ptep, *childp = NULL;
463+ u64 granule = kvm_granule_size(level);
464+ struct hyp_unmap_data *data = arg;
465+ struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
466+
467+ if (!kvm_pte_valid(pte))
468+ return -EINVAL;
469+
470+ if (kvm_pte_table(pte, level)) {
471+ childp = kvm_pte_follow(pte, mm_ops);
472+
473+ if (mm_ops->page_count(childp) != 1)
474+ return 0;
475+
476+ kvm_clear_pte(ptep);
477+ dsb(ishst);
478+ __tlbi_level(vae2is, __TLBI_VADDR(addr, 0), level);
479+ } else {
480+ if (end - addr < granule)
481+ return -EINVAL;
482+
483+ kvm_clear_pte(ptep);
484+ dsb(ishst);
485+ __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level);
486+ data->unmapped += granule;
487+ }
488+
489+ dsb(ish);
490+ isb();
491+ mm_ops->put_page(ptep);
492+
493+ if (childp)
494+ mm_ops->put_page(childp);
495+
496+ return 0;
497+}
498+
499+u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
500+{
501+ struct hyp_unmap_data unmap_data = {
502+ .mm_ops = pgt->mm_ops,
503+ };
504+ struct kvm_pgtable_walker walker = {
505+ .cb = hyp_unmap_walker,
506+ .arg = &unmap_data,
507+ .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
508+ };
509+
510+ if (!pgt->mm_ops->page_count)
511+ return 0;
512+
513+ kvm_pgtable_walk(pgt, addr, size, &walker);
514+ return unmap_data.unmapped;
515+}
516+
463517 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
464518 struct kvm_pgtable_mm_ops *mm_ops)
465519 {
@@ -482,8 +536,16 @@ static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 16 unmodified lines
482536 enum kvm_pgtable_walk_flags flag, void * const arg)
483537 {
484538 struct kvm_pgtable_mm_ops *mm_ops = arg;
539+ kvm_pte_t pte = *ptep;
540+
541+ if (!kvm_pte_valid(pte))
542+ return 0;
543+
544+ mm_ops->put_page(ptep);
545+
546+ if (kvm_pte_table(pte, level))
547+ mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
485548
486− mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));
487549 return 0;
488550 }
489551
@@ -491,7 +553,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) · 1 unmodified line
491553 {
492554 struct kvm_pgtable_walker walker = {
493555 .cb = hyp_free_walker,
494− .flags = KVM_PGTABLE_WALK_TABLE_POST,
556+ .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
495557 .arg = pgt->mm_ops,
496558 };
497559
@@ -1116,13 +1178,13 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) · 618 unmodified lines
11161178 }
11171179
11181180
1119−int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch,
1181+int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
11201182 struct kvm_pgtable_mm_ops *mm_ops,
11211183 enum kvm_pgtable_stage2_flags flags,
11221184 kvm_pgtable_force_pte_cb_t force_pte_cb)
11231185 {
11241186 size_t pgd_sz;
1125− u64 vtcr = arch->vtcr;
1187+ u64 vtcr = mmu->arch->vtcr;
11261188 u32 ia_bits = VTCR_EL2_IPA(vtcr);
11271189 u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
11281190 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
@@ -1135,7 +1197,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_arch *arch, · 6 unmodified lines
11351197 pgt->ia_bits = ia_bits;
11361198 pgt->start_level = start_level;
11371199 pgt->mm_ops = mm_ops;
1138− pgt->mmu = &arch->mmu;
1200+ pgt->mmu = mmu;
11391201 pgt->flags = flags;
11401202 pgt->force_pte_cb = force_pte_cb;
11411203
arch/arm64/kvm/hyp/vhe/switch.c+0−1
@@ -24,7 +24,6 @@ · 23 unmodified lines
2424 #include <asm/fpsimd.h>
2525 #include <asm/debug-monitors.h>
2626 #include <asm/processor.h>
27−#include <asm/thread_info.h>
2827
2928 /* VHE specific context */
3029 DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
arch/arm64/kvm/mmu.c+137−13
@@ -239,6 +239,9 @@ void free_hyp_pgds(void) · 238 unmodified lines
239239
240240 static bool kvm_host_owns_hyp_mappings(void)
241241 {
242+ if (is_kernel_in_hyp_mode())
243+ return false;
244+
242245 if (static_branch_likely(&kvm_protected_mode_initialized))
243246 return false;
244247
@@ -281,14 +284,117 @@ static phys_addr_t kvm_kaddr_to_phys(void *kaddr) · 36 unmodified lines
281284 }
282285 }
283286
284−static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end)
287+struct hyp_shared_pfn {
288+ u64 pfn;
289+ int count;
290+ struct rb_node node;
291+};
292+
293+static DEFINE_MUTEX(hyp_shared_pfns_lock);
294+static struct rb_root hyp_shared_pfns = RB_ROOT;
295+
296+static struct hyp_shared_pfn *find_shared_pfn(u64 pfn, struct rb_node ***node,
297+ struct rb_node **parent)
285298 {
286− phys_addr_t addr;
299+ struct hyp_shared_pfn *this;
300+
301+ *node = &hyp_shared_pfns.rb_node;
302+ *parent = NULL;
303+ while (**node) {
304+ this = container_of(**node, struct hyp_shared_pfn, node);
305+ *parent = **node;
306+ if (this->pfn < pfn)
307+ *node = &((**node)->rb_left);
308+ else if (this->pfn > pfn)
309+ *node = &((**node)->rb_right);
310+ else
311+ return this;
312+ }
313+
314+ return NULL;
315+}
316+
317+static int share_pfn_hyp(u64 pfn)
318+{
319+ struct rb_node **node, *parent;
320+ struct hyp_shared_pfn *this;
321+ int ret = 0;
322+
323+ mutex_lock(&hyp_shared_pfns_lock);
324+ this = find_shared_pfn(pfn, &node, &parent);
325+ if (this) {
326+ this->count++;
327+ goto unlock;
328+ }
329+
330+ this = kzalloc(sizeof(*this), GFP_KERNEL);
331+ if (!this) {
332+ ret = -ENOMEM;
333+ goto unlock;
334+ }
335+
336+ this->pfn = pfn;
337+ this->count = 1;
338+ rb_link_node(&this->node, parent, node);
339+ rb_insert_color(&this->node, &hyp_shared_pfns);
340+ ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn, 1);
341+unlock:
342+ mutex_unlock(&hyp_shared_pfns_lock);
343+
344+ return ret;
345+}
346+
347+static int unshare_pfn_hyp(u64 pfn)
348+{
349+ struct rb_node **node, *parent;
350+ struct hyp_shared_pfn *this;
351+ int ret = 0;
352+
353+ mutex_lock(&hyp_shared_pfns_lock);
354+ this = find_shared_pfn(pfn, &node, &parent);
355+ if (WARN_ON(!this)) {
356+ ret = -ENOENT;
357+ goto unlock;
358+ }
359+
360+ this->count--;
361+ if (this->count)
362+ goto unlock;
363+
364+ rb_erase(&this->node, &hyp_shared_pfns);
365+ kfree(this);
366+ ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn, 1);
367+unlock:
368+ mutex_unlock(&hyp_shared_pfns_lock);
369+
370+ return ret;
371+}
372+
373+int kvm_share_hyp(void *from, void *to)
374+{
375+ phys_addr_t start, end, cur;
376+ u64 pfn;
287377 int ret;
288378
289− for (addr = ALIGN_DOWN(start, PAGE_SIZE); addr < end; addr += PAGE_SIZE) {
290− ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp,
291− __phys_to_pfn(addr));
379+ if (is_kernel_in_hyp_mode())
380+ return 0;
381+
382+ /*
383+ * The share hcall maps things in the 'fixed-offset' region of the hyp
384+ * VA space, so we can only share physically contiguous data-structures
385+ * for now.
386+ */
387+ if (is_vmalloc_or_module_addr(from) || is_vmalloc_or_module_addr(to))
388+ return -EINVAL;
389+
390+ if (kvm_host_owns_hyp_mappings())
391+ return create_hyp_mappings(from, to, PAGE_HYP);
392+
393+ start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
394+ end = PAGE_ALIGN(__pa(to));
395+ for (cur = start; cur < end; cur += PAGE_SIZE) {
396+ pfn = __phys_to_pfn(cur);
397+ ret = share_pfn_hyp(pfn);
292398 if (ret)
293399 return ret;
294400 }
@@ -296,6 +402,22 @@ static int pkvm_share_hyp(phys_addr_t start, phys_addr_t end) · 1 unmodified line
296402 return 0;
297403 }
298404
405+void kvm_unshare_hyp(void *from, void *to)
406+{
407+ phys_addr_t start, end, cur;
408+ u64 pfn;
409+
410+ if (is_kernel_in_hyp_mode() || kvm_host_owns_hyp_mappings() || !from)
411+ return;
412+
413+ start = ALIGN_DOWN(__pa(from), PAGE_SIZE);
414+ end = PAGE_ALIGN(__pa(to));
415+ for (cur = start; cur < end; cur += PAGE_SIZE) {
416+ pfn = __phys_to_pfn(cur);
417+ WARN_ON(unshare_pfn_hyp(pfn));
418+ }
419+}
420+
299421 /**
300422 * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
301423 * @from: The virtual kernel start address of the range
@@ -316,12 +438,8 @@ int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot) · 14 unmodified lines
316438 if (is_kernel_in_hyp_mode())
317439 return 0;
318440
319− if (!kvm_host_owns_hyp_mappings()) {
320− if (WARN_ON(prot != PAGE_HYP))
321− return -EPERM;
322− return pkvm_share_hyp(kvm_kaddr_to_phys(from),
323− kvm_kaddr_to_phys(to));
324− }
441+ if (!kvm_host_owns_hyp_mappings())
442+ return -EPERM;
325443
326444 start = start & PAGE_MASK;
327445 end = PAGE_ALIGN(end);
@@ -407,6 +525,9 @@ int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, · 79 unmodified lines
407525 unsigned long addr;
408526 int ret;
409527
528+ if (is_protected_kvm_enabled())
529+ return -EPERM;
530+
410531 *kaddr = ioremap(phys_addr, size);
411532 if (!*kaddr)
412533 return -ENOMEM;
@@ -516,7 +637,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) · 103 unmodified lines
516637 if (!pgt)
517638 return -ENOMEM;
518639
519− err = kvm_pgtable_stage2_init(pgt, &kvm->arch, &kvm_s2_mm_ops);
640+ mmu->arch = &kvm->arch;
641+ err = kvm_pgtable_stage2_init(pgt, mmu, &kvm_s2_mm_ops);
520642 if (err)
521643 goto out_free_pgtable;
522644
@@ -529,7 +651,6 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) · 6 unmodified lines
529651 for_each_possible_cpu(cpu)
530652 *per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;
531653
532− mmu->arch = &kvm->arch;
533654 mmu->pgt = pgt;
534655 mmu->pgd_phys = __pa(pgt->pgd);
535656 WRITE_ONCE(mmu->vmid.vmid_gen, 0);
@@ -650,6 +771,9 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, · 114 unmodified lines
650771 KVM_PGTABLE_PROT_R |
651772 (writable ? KVM_PGTABLE_PROT_W : 0);
652773
774+ if (is_protected_kvm_enabled())
775+ return -EPERM;
776+
653777 size += offset_in_page(guest_ipa);
654778 guest_ipa &= PAGE_MASK;
655779
arch/arm64/kvm/pkvm.crenamed from arch/arm64/kvm/hyp/reserved_mem.c+4−4
@@ -8,10 +8,9 @@ · 7 unmodified lines
88 #include <linux/memblock.h>
99 #include <linux/sort.h>
1010
11−#include <asm/kvm_host.h>
11+#include <asm/kvm_pkvm.h>
1212
13−#include <nvhe/memory.h>
14−#include <nvhe/mm.h>
13+#include "hyp_constants.h"
1514
1615 static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
1716 static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
@@ -82,7 +81,8 @@ void __init kvm_hyp_reserve(void) · 64 unmodified lines
8281 do {
8382 prev = nr_pages;
8483 nr_pages = hyp_mem_pages + prev;
85− nr_pages = DIV_ROUND_UP(nr_pages * sizeof(struct hyp_page), PAGE_SIZE);
84+ nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE,
85+ PAGE_SIZE);
8686 nr_pages += __hyp_pgtable_max_pages(nr_pages);
8787 } while (nr_pages != prev);
8888 hyp_mem_pages += nr_pages;
arch/arm64/kvm/pmu-emul.c+1−0
@@ -28,6 +28,7 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) · 27 unmodified lines
2828 case ID_AA64DFR0_PMUVER_8_1:
2929 case ID_AA64DFR0_PMUVER_8_4:
3030 case ID_AA64DFR0_PMUVER_8_5:
31+ case ID_AA64DFR0_PMUVER_8_7:
3132 return GENMASK(15, 0);
3233 default: /* Shouldn't be here, just for sanity */
3334 WARN_ONCE(1, "Unknown PMU version %d\n", kvm->arch.pmuver);
arch/arm64/kvm/psci.c+1−1
@@ -109,7 +109,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) · 108 unmodified lines
109109
110110 /*
111111 * Make sure the reset request is observed if the change to
112− * power_state is observed.
112+ * power_off is observed.
113113 */
114114 smp_wmb();
115115
arch/arm64/kvm/reset.c+21−7
@@ -94,22 +94,31 @@ static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu) · 93 unmodified lines
9494 {
9595 void *buf;
9696 unsigned int vl;
97+ size_t reg_sz;
98+ int ret;
9799
98100 vl = vcpu->arch.sve_max_vl;
99101
100102 /*
101103 * Responsibility for these properties is shared between
102− * kvm_arm_init_arch_resources(), kvm_vcpu_enable_sve() and
104+ * kvm_arm_init_sve(), kvm_vcpu_enable_sve() and
103105 * set_sve_vls(). Double-check here just to be sure:
104106 */
105107 if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl() ||
106108 vl > SVE_VL_ARCH_MAX))
107109 return -EIO;
108110
109− buf = kzalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl)), GFP_KERNEL_ACCOUNT);
111+ reg_sz = vcpu_sve_state_size(vcpu);
112+ buf = kzalloc(reg_sz, GFP_KERNEL_ACCOUNT);
110113 if (!buf)
111114 return -ENOMEM;
112115
116+ ret = kvm_share_hyp(buf, buf + reg_sz);
117+ if (ret) {
118+ kfree(buf);
119+ return ret;
120+ }
121+
113122 vcpu->arch.sve_state = buf;
114123 vcpu->arch.flags |= KVM_ARM64_VCPU_SVE_FINALIZED;
115124 return 0;
@@ -141,7 +150,13 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) · 25 unmodified lines
141150
142151 void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)
143152 {
144− kfree(vcpu->arch.sve_state);
153+ void *sve_state = vcpu->arch.sve_state;
154+
155+ kvm_vcpu_unshare_task_fp(vcpu);
156+ kvm_unshare_hyp(vcpu, vcpu + 1);
157+ if (sve_state)
158+ kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
159+ kfree(sve_state);
145160 }
146161
147162 static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)
@@ -193,10 +208,9 @@ static bool vcpu_allowed_register_width(struct kvm_vcpu *vcpu) · 45 unmodified lines
193208 * kvm_reset_vcpu - sets core registers and sys_regs to reset value
194209 * @vcpu: The VCPU pointer
195210 *
196− * This function finds the right table above and sets the registers on
197− * the virtual CPU struct to their architecturally defined reset
198− * values, except for registers whose reset is deferred until
199− * kvm_arm_vcpu_finalize().
211+ * This function sets the registers on the virtual CPU struct to their
212+ * architecturally defined reset values, except for registers whose reset is
213+ * deferred until kvm_arm_vcpu_finalize().
200214 *
201215 * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT
202216 * ioctl or as part of handling a request issued by another VCPU in the PSCI
arch/arm64/kvm/vgic/vgic-init.c+1−1
@@ -92,7 +92,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) · 91 unmodified lines
9292 return ret;
9393
9494 kvm_for_each_vcpu(i, vcpu, kvm) {
95− if (vcpu->arch.has_run_once)
95+ if (vcpu_has_run_once(vcpu))
9696 goto out_unlock;
9797 }
9898 ret = 0;
arch/arm64/kvm/vgic/vgic-mmio-v3.c+5−3
@@ -764,10 +764,12 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm) · 763 unmodified lines
764764 }
765765
766766 if (ret) {
767− /* The current c failed, so we start with the previous one. */
767+ /* The current c failed, so iterate over the previous ones. */
768+ int i;
769+
768770 mutex_lock(&kvm->slots_lock);
769− for (c--; c >= 0; c--) {
770− vcpu = kvm_get_vcpu(kvm, c);
771+ for (i = 0; i < c; i++) {
772+ vcpu = kvm_get_vcpu(kvm, i);
771773 vgic_unregister_redist_iodev(vcpu);
772774 }
773775 mutex_unlock(&kvm->slots_lock);
arch/arm64/kvm/vgic/vgic-mmio.c+1−1
@@ -1050,7 +1050,7 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, · 1049 unmodified lines
10501050 return 0;
10511051 }
10521052
1053−struct kvm_io_device_ops kvm_io_gic_ops = {
1053+const struct kvm_io_device_ops kvm_io_gic_ops = {
10541054 .read = dispatch_mmio_read,
10551055 .write = dispatch_mmio_write,
10561056 };
arch/arm64/kvm/vgic/vgic-mmio.h+1−1
@@ -34,7 +34,7 @@ struct vgic_register_region { · 33 unmodified lines
3434 };
3535 };
3636
37−extern struct kvm_io_device_ops kvm_io_gic_ops;
37+extern const struct kvm_io_device_ops kvm_io_gic_ops;
3838
3939 #define VGIC_ACCESS_8bit 1
4040 #define VGIC_ACCESS_32bit 2
arch/arm64/kvm/vgic/vgic-v2.c+7−2
@@ -293,12 +293,12 @@ int vgic_v2_map_resources(struct kvm *kvm) · 292 unmodified lines
293293
294294 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base) ||
295295 IS_VGIC_ADDR_UNDEF(dist->vgic_cpu_base)) {
296− kvm_err("Need to set vgic cpu and dist addresses first\n");
296+ kvm_debug("Need to set vgic cpu and dist addresses first\n");
297297 return -ENXIO;
298298 }
299299
300300 if (!vgic_v2_check_base(dist->vgic_dist_base, dist->vgic_cpu_base)) {
301− kvm_err("VGIC CPU and dist frames overlap\n");
301+ kvm_debug("VGIC CPU and dist frames overlap\n");
302302 return -EINVAL;
303303 }
304304
@@ -345,6 +345,11 @@ int vgic_v2_probe(const struct gic_kvm_info *info) · 40 unmodified lines
345345 int ret;
346346 u32 vtr;
347347
348+ if (is_protected_kvm_enabled()) {
349+ kvm_err("GICv2 not supported in protected mode\n");
350+ return -ENXIO;
351+ }
352+
348353 if (!info->vctrl.start) {
349354 kvm_err("GICH not present in the firmware table\n");
350355 return -ENXIO;
arch/arm64/kvm/vgic/vgic-v3.c+3−3
@@ -554,12 +554,12 @@ int vgic_v3_map_resources(struct kvm *kvm) · 553 unmodified lines
554554 }
555555
556556 if (IS_VGIC_ADDR_UNDEF(dist->vgic_dist_base)) {
557− kvm_err("Need to set vgic distributor addresses first\n");
557+ kvm_debug("Need to set vgic distributor addresses first\n");
558558 return -ENXIO;
559559 }
560560
561561 if (!vgic_v3_check_base(kvm)) {
562− kvm_err("VGIC redist and dist frames overlap\n");
562+ kvm_debug("VGIC redist and dist frames overlap\n");
563563 return -EINVAL;
564564 }
565565
@@ -651,7 +651,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info) · 85 unmodified lines
651651 } else if (!PAGE_ALIGNED(info->vcpu.start)) {
652652 pr_warn("GICV physical address 0x%llx not page aligned\n",
653653 (unsigned long long)info->vcpu.start);
654− } else {
654+ } else if (kvm_get_mode() != KVM_MODE_PROTECTED) {
655655 kvm_vgic_global_state.vcpu_base = info->vcpu.start;
656656 kvm_vgic_global_state.can_emulate_gicv2 = true;
657657 ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2);
include/kvm/arm_vgic.h+3−1
@@ -5,9 +5,11 @@ · 4 unmodified lines
55 #ifndef __KVM_ARM_VGIC_H
66 #define __KVM_ARM_VGIC_H
77
8−#include <linux/kernel.h>
8+#include <linux/bits.h>
99 #include <linux/kvm.h>
1010 #include <linux/irqreturn.h>
11+#include <linux/kref.h>
12+#include <linux/mutex.h>
1113 #include <linux/spinlock.h>
1214 #include <linux/static_key.h>
1315 #include <linux/types.h>
include/linux/kvm_host.h+0−1
@@ -1322,7 +1322,6 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu); · 1321 unmodified lines
13221322
13231323 bool kvm_is_reserved_pfn(kvm_pfn_t pfn);
13241324 bool kvm_is_zone_device_pfn(kvm_pfn_t pfn);
1325−bool kvm_is_transparent_hugepage(kvm_pfn_t pfn);
13261325
13271326 struct kvm_irq_ack_notifier {
13281327 struct hlist_node link;
tools/testing/selftests/kvm/.gitignore+1−0
@@ -4,6 +4,7 @@ · 3 unmodified lines
44 /aarch64/get-reg-list
55 /aarch64/psci_cpu_on_test
66 /aarch64/vgic_init
7+/aarch64/vgic_irq
78 /s390x/memop
89 /s390x/resets
910 /s390x/sync_regs_test
tools/testing/selftests/kvm/Makefile+1−0
@@ -96,6 +96,7 @@ TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions · 95 unmodified lines
9696 TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list
9797 TEST_GEN_PROGS_aarch64 += aarch64/psci_cpu_on_test
9898 TEST_GEN_PROGS_aarch64 += aarch64/vgic_init
99+TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq
99100 TEST_GEN_PROGS_aarch64 += demand_paging_test
100101 TEST_GEN_PROGS_aarch64 += dirty_log_test
101102 TEST_GEN_PROGS_aarch64 += dirty_log_perf_test
tools/testing/selftests/kvm/aarch64/arch_timer.c+1−1
@@ -382,7 +382,7 @@ static struct kvm_vm *test_vm_create(void) · 381 unmodified lines
382382
383383 ucall_init(vm, NULL);
384384 test_init_timer_irq(vm);
385− vgic_v3_setup(vm, nr_vcpus, GICD_BASE_GPA, GICR_BASE_GPA);
385+ vgic_v3_setup(vm, nr_vcpus, 64, GICD_BASE_GPA, GICR_BASE_GPA);
386386
387387 /* Make all the test's cmdline args visible to the guest */
388388 sync_global_to_guest(vm, test_args);
tools/testing/selftests/kvm/aarch64/get-reg-list.c+50−0
@@ -1014,6 +1014,22 @@ static __u64 sve_rejects_set[] = { · 1013 unmodified lines
10141014 KVM_REG_ARM64_SVE_VLS,
10151015 };
10161016
1017+static __u64 pauth_addr_regs[] = {
1018+ ARM64_SYS_REG(3, 0, 2, 1, 0), /* APIAKEYLO_EL1 */
1019+ ARM64_SYS_REG(3, 0, 2, 1, 1), /* APIAKEYHI_EL1 */
1020+ ARM64_SYS_REG(3, 0, 2, 1, 2), /* APIBKEYLO_EL1 */
1021+ ARM64_SYS_REG(3, 0, 2, 1, 3), /* APIBKEYHI_EL1 */
1022+ ARM64_SYS_REG(3, 0, 2, 2, 0), /* APDAKEYLO_EL1 */
1023+ ARM64_SYS_REG(3, 0, 2, 2, 1), /* APDAKEYHI_EL1 */
1024+ ARM64_SYS_REG(3, 0, 2, 2, 2), /* APDBKEYLO_EL1 */
1025+ ARM64_SYS_REG(3, 0, 2, 2, 3) /* APDBKEYHI_EL1 */
1026+};
1027+
1028+static __u64 pauth_generic_regs[] = {
1029+ ARM64_SYS_REG(3, 0, 2, 3, 0), /* APGAKEYLO_EL1 */
1030+ ARM64_SYS_REG(3, 0, 2, 3, 1), /* APGAKEYHI_EL1 */
1031+};
1032+
10171033 #define BASE_SUBLIST \
10181034 { "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
10191035 #define VREGS_SUBLIST \
@@ -1025,6 +1041,21 @@ static __u64 sve_rejects_set[] = { · 5 unmodified lines
10251041 { "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \
10261042 .regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \
10271043 .rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }
1044+#define PAUTH_SUBLIST \
1045+ { \
1046+ .name = "pauth_address", \
1047+ .capability = KVM_CAP_ARM_PTRAUTH_ADDRESS, \
1048+ .feature = KVM_ARM_VCPU_PTRAUTH_ADDRESS, \
1049+ .regs = pauth_addr_regs, \
1050+ .regs_n = ARRAY_SIZE(pauth_addr_regs), \
1051+ }, \
1052+ { \
1053+ .name = "pauth_generic", \
1054+ .capability = KVM_CAP_ARM_PTRAUTH_GENERIC, \
1055+ .feature = KVM_ARM_VCPU_PTRAUTH_GENERIC, \
1056+ .regs = pauth_generic_regs, \
1057+ .regs_n = ARRAY_SIZE(pauth_generic_regs), \
1058+ }
10281059
10291060 static struct vcpu_config vregs_config = {
10301061 .sublists = {
@@ -1056,11 +1087,30 @@ static struct vcpu_config sve_pmu_config = { · 25 unmodified lines
10561087 {0},
10571088 },
10581089 };
1090+static struct vcpu_config pauth_config = {
1091+ .sublists = {
1092+ BASE_SUBLIST,
1093+ VREGS_SUBLIST,
1094+ PAUTH_SUBLIST,
1095+ {0},
1096+ },
1097+};
1098+static struct vcpu_config pauth_pmu_config = {
1099+ .sublists = {
1100+ BASE_SUBLIST,
1101+ VREGS_SUBLIST,
1102+ PAUTH_SUBLIST,
1103+ PMU_SUBLIST,
1104+ {0},
1105+ },
1106+};
10591107
10601108 static struct vcpu_config *vcpu_configs[] = {
10611109 &vregs_config,
10621110 &vregs_pmu_config,
10631111 &sve_config,
10641112 &sve_pmu_config,
1113+ &pauth_config,
1114+ &pauth_pmu_config,
10651115 };
10661116 static int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
tools/testing/selftests/kvm/aarch64/vgic_irq.c+853−0
@@ -0,0 +1,853 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * vgic_irq.c - Test userspace injection of IRQs
4+ *
5+ * This test validates the injection of IRQs from userspace using various
6+ * methods (e.g., KVM_IRQ_LINE) and modes (e.g., EOI). The guest "asks" the
7+ * host to inject a specific intid via a GUEST_SYNC call, and then checks that
8+ * it received it.
9+ */
10+
11+#include <asm/kvm.h>
12+#include <asm/kvm_para.h>
13+#include <sys/eventfd.h>
14+#include <linux/sizes.h>
15+
16+#include "processor.h"
17+#include "test_util.h"
18+#include "kvm_util.h"
19+#include "gic.h"
20+#include "gic_v3.h"
21+#include "vgic.h"
22+
23+#define GICD_BASE_GPA 0x08000000ULL
24+#define GICR_BASE_GPA 0x080A0000ULL
25+#define VCPU_ID 0
26+
27+/*
28+ * Stores the user specified args; it's passed to the guest and to every test
29+ * function.
30+ */
31+struct test_args {
32+ uint32_t nr_irqs; /* number of KVM supported IRQs. */
33+ bool eoi_split; /* 1 is eoir+dir, 0 is eoir only */
34+ bool level_sensitive; /* 1 is level, 0 is edge */
35+ int kvm_max_routes; /* output of KVM_CAP_IRQ_ROUTING */
36+ bool kvm_supports_irqfd; /* output of KVM_CAP_IRQFD */
37+};
38+
39+/*
40+ * KVM implements 32 priority levels:
41+ * 0x00 (highest priority) - 0xF8 (lowest priority), in steps of 8
42+ *
43+ * Note that these macros will still be correct in the case that KVM implements
44+ * more priority levels. Also note that 32 is the minimum for GICv3 and GICv2.
45+ */
46+#define KVM_NUM_PRIOS 32
47+#define KVM_PRIO_SHIFT 3 /* steps of 8 = 1 << 3 */
48+#define KVM_PRIO_STEPS (1 << KVM_PRIO_SHIFT) /* 8 */
49+#define LOWEST_PRIO (KVM_NUM_PRIOS - 1)
50+#define CPU_PRIO_MASK (LOWEST_PRIO << KVM_PRIO_SHIFT) /* 0xf8 */
51+#define IRQ_DEFAULT_PRIO (LOWEST_PRIO - 1)
52+#define IRQ_DEFAULT_PRIO_REG (IRQ_DEFAULT_PRIO << KVM_PRIO_SHIFT) /* 0xf0 */
53+
54+static void *dist = (void *)GICD_BASE_GPA;
55+static void *redist = (void *)GICR_BASE_GPA;
56+
57+/*
58+ * The kvm_inject_* utilities are used by the guest to ask the host to inject
59+ * interrupts (e.g., using the KVM_IRQ_LINE ioctl).
60+ */
61+
62+typedef enum {
63+ KVM_INJECT_EDGE_IRQ_LINE = 1,
64+ KVM_SET_IRQ_LINE,
65+ KVM_SET_IRQ_LINE_HIGH,
66+ KVM_SET_LEVEL_INFO_HIGH,
67+ KVM_INJECT_IRQFD,
68+ KVM_WRITE_ISPENDR,
69+ KVM_WRITE_ISACTIVER,
70+} kvm_inject_cmd;
71+
72+struct kvm_inject_args {
73+ kvm_inject_cmd cmd;
74+ uint32_t first_intid;
75+ uint32_t num;
76+ int level;
77+ bool expect_failure;
78+};
79+
80+/* Used on the guest side to perform the hypercall. */
81+static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
82+ uint32_t num, int level, bool expect_failure);
83+
84+/* Used on the host side to get the hypercall info. */
85+static void kvm_inject_get_call(struct kvm_vm *vm, struct ucall *uc,
86+ struct kvm_inject_args *args);
87+
88+#define _KVM_INJECT_MULTI(cmd, intid, num, expect_failure) \
89+ kvm_inject_call(cmd, intid, num, -1 /* not used */, expect_failure)
90+
91+#define KVM_INJECT_MULTI(cmd, intid, num) \
92+ _KVM_INJECT_MULTI(cmd, intid, num, false)
93+
94+#define _KVM_INJECT(cmd, intid, expect_failure) \
95+ _KVM_INJECT_MULTI(cmd, intid, 1, expect_failure)
96+
97+#define KVM_INJECT(cmd, intid) \
98+ _KVM_INJECT_MULTI(cmd, intid, 1, false)
99+
100+#define KVM_ACTIVATE(cmd, intid) \
101+ kvm_inject_call(cmd, intid, 1, 1, false);
102+
103+struct kvm_inject_desc {
104+ kvm_inject_cmd cmd;
105+ /* can inject PPIs, PPIs, and/or SPIs. */
106+ bool sgi, ppi, spi;
107+};
108+
109+static struct kvm_inject_desc inject_edge_fns[] = {
110+ /* sgi ppi spi */
111+ { KVM_INJECT_EDGE_IRQ_LINE, false, false, true },
112+ { KVM_INJECT_IRQFD, false, false, true },
113+ { KVM_WRITE_ISPENDR, true, false, true },
114+ { 0, },
115+};
116+
117+static struct kvm_inject_desc inject_level_fns[] = {
118+ /* sgi ppi spi */
119+ { KVM_SET_IRQ_LINE_HIGH, false, true, true },
120+ { KVM_SET_LEVEL_INFO_HIGH, false, true, true },
121+ { KVM_INJECT_IRQFD, false, false, true },
122+ { KVM_WRITE_ISPENDR, false, true, true },
123+ { 0, },
124+};
125+
126+static struct kvm_inject_desc set_active_fns[] = {
127+ /* sgi ppi spi */
128+ { KVM_WRITE_ISACTIVER, true, true, true },
129+ { 0, },
130+};
131+
132+#define for_each_inject_fn(t, f) \
133+ for ((f) = (t); (f)->cmd; (f)++)
134+
135+#define for_each_supported_inject_fn(args, t, f) \
136+ for_each_inject_fn(t, f) \
137+ if ((args)->kvm_supports_irqfd || (f)->cmd != KVM_INJECT_IRQFD)
138+
139+#define for_each_supported_activate_fn(args, t, f) \
140+ for_each_supported_inject_fn((args), (t), (f))
141+
142+/* Shared between the guest main thread and the IRQ handlers. */
143+volatile uint64_t irq_handled;
144+volatile uint32_t irqnr_received[MAX_SPI + 1];
145+
146+static void reset_stats(void)
147+{
148+ int i;
149+
150+ irq_handled = 0;
151+ for (i = 0; i <= MAX_SPI; i++)
152+ irqnr_received[i] = 0;
153+}
154+
155+static uint64_t gic_read_ap1r0(void)
156+{
157+ uint64_t reg = read_sysreg_s(SYS_ICV_AP1R0_EL1);
158+
159+ dsb(sy);
160+ return reg;
161+}
162+
163+static void gic_write_ap1r0(uint64_t val)
164+{
165+ write_sysreg_s(val, SYS_ICV_AP1R0_EL1);
166+ isb();
167+}
168+
169+static void guest_set_irq_line(uint32_t intid, uint32_t level);
170+
171+static void guest_irq_generic_handler(bool eoi_split, bool level_sensitive)
172+{
173+ uint32_t intid = gic_get_and_ack_irq();
174+
175+ if (intid == IAR_SPURIOUS)
176+ return;
177+
178+ GUEST_ASSERT(gic_irq_get_active(intid));
179+
180+ if (!level_sensitive)
181+ GUEST_ASSERT(!gic_irq_get_pending(intid));
182+
183+ if (level_sensitive)
184+ guest_set_irq_line(intid, 0);
185+
186+ GUEST_ASSERT(intid < MAX_SPI);
187+ irqnr_received[intid] += 1;
188+ irq_handled += 1;
189+
190+ gic_set_eoi(intid);
191+ GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
192+ if (eoi_split)
193+ gic_set_dir(intid);
194+
195+ GUEST_ASSERT(!gic_irq_get_active(intid));
196+ GUEST_ASSERT(!gic_irq_get_pending(intid));
197+}
198+
199+static void kvm_inject_call(kvm_inject_cmd cmd, uint32_t first_intid,
200+ uint32_t num, int level, bool expect_failure)
201+{
202+ struct kvm_inject_args args = {
203+ .cmd = cmd,
204+ .first_intid = first_intid,
205+ .num = num,
206+ .level = level,
207+ .expect_failure = expect_failure,
208+ };
209+ GUEST_SYNC(&args);
210+}
211+
212+#define GUEST_ASSERT_IAR_EMPTY() \
213+do { \
214+ uint32_t _intid; \
215+ _intid = gic_get_and_ack_irq(); \
216+ GUEST_ASSERT(_intid == 0 || _intid == IAR_SPURIOUS); \
217+} while (0)
218+
219+#define CAT_HELPER(a, b) a ## b
220+#define CAT(a, b) CAT_HELPER(a, b)
221+#define PREFIX guest_irq_handler_
222+#define GUEST_IRQ_HANDLER_NAME(split, lev) CAT(PREFIX, CAT(split, lev))
223+#define GENERATE_GUEST_IRQ_HANDLER(split, lev) \
224+static void CAT(PREFIX, CAT(split, lev))(struct ex_regs *regs) \
225+{ \
226+ guest_irq_generic_handler(split, lev); \
227+}
228+
229+GENERATE_GUEST_IRQ_HANDLER(0, 0);
230+GENERATE_GUEST_IRQ_HANDLER(0, 1);
231+GENERATE_GUEST_IRQ_HANDLER(1, 0);
232+GENERATE_GUEST_IRQ_HANDLER(1, 1);
233+
234+static void (*guest_irq_handlers[2][2])(struct ex_regs *) = {
235+ {GUEST_IRQ_HANDLER_NAME(0, 0), GUEST_IRQ_HANDLER_NAME(0, 1),},
236+ {GUEST_IRQ_HANDLER_NAME(1, 0), GUEST_IRQ_HANDLER_NAME(1, 1),},
237+};
238+
239+static void reset_priorities(struct test_args *args)
240+{
241+ int i;
242+
243+ for (i = 0; i < args->nr_irqs; i++)
244+ gic_set_priority(i, IRQ_DEFAULT_PRIO_REG);
245+}
246+
247+static void guest_set_irq_line(uint32_t intid, uint32_t level)
248+{
249+ kvm_inject_call(KVM_SET_IRQ_LINE, intid, 1, level, false);
250+}
251+
252+static void test_inject_fail(struct test_args *args,
253+ uint32_t intid, kvm_inject_cmd cmd)
254+{
255+ reset_stats();
256+
257+ _KVM_INJECT(cmd, intid, true);
258+ /* no IRQ to handle on entry */
259+
260+ GUEST_ASSERT_EQ(irq_handled, 0);
261+ GUEST_ASSERT_IAR_EMPTY();
262+}
263+
264+static void guest_inject(struct test_args *args,
265+ uint32_t first_intid, uint32_t num,
266+ kvm_inject_cmd cmd)
267+{
268+ uint32_t i;
269+
270+ reset_stats();
271+
272+ /* Cycle over all priorities to make things more interesting. */
273+ for (i = first_intid; i < num + first_intid; i++)
274+ gic_set_priority(i, (i % (KVM_NUM_PRIOS - 1)) << 3);
275+
276+ asm volatile("msr daifset, #2" : : : "memory");
277+ KVM_INJECT_MULTI(cmd, first_intid, num);
278+
279+ while (irq_handled < num) {
280+ asm volatile("wfi\n"
281+ "msr daifclr, #2\n"
282+ /* handle IRQ */
283+ "msr daifset, #2\n"
284+ : : : "memory");
285+ }
286+ asm volatile("msr daifclr, #2" : : : "memory");
287+
288+ GUEST_ASSERT_EQ(irq_handled, num);
289+ for (i = first_intid; i < num + first_intid; i++)
290+ GUEST_ASSERT_EQ(irqnr_received[i], 1);
291+ GUEST_ASSERT_IAR_EMPTY();
292+
293+ reset_priorities(args);
294+}
295+
296+/*
297+ * Restore the active state of multiple concurrent IRQs (given by
298+ * concurrent_irqs). This does what a live-migration would do on the
299+ * destination side assuming there are some active IRQs that were not
300+ * deactivated yet.
301+ */
302+static void guest_restore_active(struct test_args *args,
303+ uint32_t first_intid, uint32_t num,
304+ kvm_inject_cmd cmd)
305+{
306+ uint32_t prio, intid, ap1r;
307+ int i;
308+
309+ /* Set the priorities of the first (KVM_NUM_PRIOS - 1) IRQs
310+ * in descending order, so intid+1 can preempt intid.
311+ */
312+ for (i = 0, prio = (num - 1) * 8; i < num; i++, prio -= 8) {
313+ GUEST_ASSERT(prio >= 0);
314+ intid = i + first_intid;
315+ gic_set_priority(intid, prio);
316+ }
317+
318+ /* In a real migration, KVM would restore all GIC state before running
319+ * guest code.
320+ */
321+ for (i = 0; i < num; i++) {
322+ intid = i + first_intid;
323+ KVM_ACTIVATE(cmd, intid);
324+ ap1r = gic_read_ap1r0();
325+ ap1r |= 1U << i;
326+ gic_write_ap1r0(ap1r);
327+ }
328+
329+ /* This is where the "migration" would occur. */
330+
331+ /* finish handling the IRQs starting with the highest priority one. */
332+ for (i = 0; i < num; i++) {
333+ intid = num - i - 1 + first_intid;
334+ gic_set_eoi(intid);
335+ if (args->eoi_split)
336+ gic_set_dir(intid);
337+ }
338+
339+ for (i = 0; i < num; i++)
340+ GUEST_ASSERT(!gic_irq_get_active(i + first_intid));
341+ GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
342+ GUEST_ASSERT_IAR_EMPTY();
343+}
344+
345+/*
346+ * Polls the IAR until it's not a spurious interrupt.
347+ *
348+ * This function should only be used in test_inject_preemption (with IRQs
349+ * masked).
350+ */
351+static uint32_t wait_for_and_activate_irq(void)
352+{
353+ uint32_t intid;
354+
355+ do {
356+ asm volatile("wfi" : : : "memory");
357+ intid = gic_get_and_ack_irq();
358+ } while (intid == IAR_SPURIOUS);
359+
360+ return intid;
361+}
362+
363+/*
364+ * Inject multiple concurrent IRQs (num IRQs starting at first_intid) and
365+ * handle them without handling the actual exceptions. This is done by masking
366+ * interrupts for the whole test.
367+ */
368+static void test_inject_preemption(struct test_args *args,
369+ uint32_t first_intid, int num,
370+ kvm_inject_cmd cmd)
371+{
372+ uint32_t intid, prio, step = KVM_PRIO_STEPS;
373+ int i;
374+
375+ /* Set the priorities of the first (KVM_NUM_PRIOS - 1) IRQs
376+ * in descending order, so intid+1 can preempt intid.
377+ */
378+ for (i = 0, prio = (num - 1) * step; i < num; i++, prio -= step) {
379+ GUEST_ASSERT(prio >= 0);
380+ intid = i + first_intid;
381+ gic_set_priority(intid, prio);
382+ }
383+
384+ local_irq_disable();
385+
386+ for (i = 0; i < num; i++) {
387+ uint32_t tmp;
388+ intid = i + first_intid;
389+ KVM_INJECT(cmd, intid);
390+ /* Each successive IRQ will preempt the previous one. */
391+ tmp = wait_for_and_activate_irq();
392+ GUEST_ASSERT_EQ(tmp, intid);
393+ if (args->level_sensitive)
394+ guest_set_irq_line(intid, 0);
395+ }
396+
397+ /* finish handling the IRQs starting with the highest priority one. */
398+ for (i = 0; i < num; i++) {
399+ intid = num - i - 1 + first_intid;
400+ gic_set_eoi(intid);
401+ if (args->eoi_split)
402+ gic_set_dir(intid);
403+ }
404+
405+ local_irq_enable();
406+
407+ for (i = 0; i < num; i++)
408+ GUEST_ASSERT(!gic_irq_get_active(i + first_intid));
409+ GUEST_ASSERT_EQ(gic_read_ap1r0(), 0);
410+ GUEST_ASSERT_IAR_EMPTY();
411+
412+ reset_priorities(args);
413+}
414+
415+static void test_injection(struct test_args *args, struct kvm_inject_desc *f)
416+{
417+ uint32_t nr_irqs = args->nr_irqs;
418+
419+ if (f->sgi) {
420+ guest_inject(args, MIN_SGI, 1, f->cmd);
421+ guest_inject(args, 0, 16, f->cmd);
422+ }
423+
424+ if (f->ppi)
425+ guest_inject(args, MIN_PPI, 1, f->cmd);
426+
427+ if (f->spi) {
428+ guest_inject(args, MIN_SPI, 1, f->cmd);
429+ guest_inject(args, nr_irqs - 1, 1, f->cmd);
430+ guest_inject(args, MIN_SPI, nr_irqs - MIN_SPI, f->cmd);
431+ }
432+}
433+
434+static void test_injection_failure(struct test_args *args,
435+ struct kvm_inject_desc *f)
436+{
437+ uint32_t bad_intid[] = { args->nr_irqs, 1020, 1024, 1120, 5120, ~0U, };
438+ int i;
439+
440+ for (i = 0; i < ARRAY_SIZE(bad_intid); i++)
441+ test_inject_fail(args, bad_intid[i], f->cmd);
442+}
443+
444+static void test_preemption(struct test_args *args, struct kvm_inject_desc *f)
445+{
446+ /*
447+ * Test up to 4 levels of preemption. The reason is that KVM doesn't
448+ * currently implement the ability to have more than the number-of-LRs
449+ * number of concurrently active IRQs. The number of LRs implemented is
450+ * IMPLEMENTATION DEFINED, however, it seems that most implement 4.
451+ */
452+ if (f->sgi)
453+ test_inject_preemption(args, MIN_SGI, 4, f->cmd);
454+
455+ if (f->ppi)
456+ test_inject_preemption(args, MIN_PPI, 4, f->cmd);
457+
458+ if (f->spi)
459+ test_inject_preemption(args, MIN_SPI, 4, f->cmd);
460+}
461+
462+static void test_restore_active(struct test_args *args, struct kvm_inject_desc *f)
463+{
464+ /* Test up to 4 active IRQs. Same reason as in test_preemption. */
465+ if (f->sgi)
466+ guest_restore_active(args, MIN_SGI, 4, f->cmd);
467+
468+ if (f->ppi)
469+ guest_restore_active(args, MIN_PPI, 4, f->cmd);
470+
471+ if (f->spi)
472+ guest_restore_active(args, MIN_SPI, 4, f->cmd);
473+}
474+
475+static void guest_code(struct test_args args)
476+{
477+ uint32_t i, nr_irqs = args.nr_irqs;
478+ bool level_sensitive = args.level_sensitive;
479+ struct kvm_inject_desc *f, *inject_fns;
480+
481+ gic_init(GIC_V3, 1, dist, redist);
482+
483+ for (i = 0; i < nr_irqs; i++)
484+ gic_irq_enable(i);
485+
486+ for (i = MIN_SPI; i < nr_irqs; i++)
487+ gic_irq_set_config(i, !args.level_sensitive);
488+
489+ gic_set_eoi_split(args.eoi_split);
490+
491+ reset_priorities(&args);
492+ gic_set_priority_mask(CPU_PRIO_MASK);
493+
494+ inject_fns = level_sensitive ? inject_level_fns
495+ : inject_edge_fns;
496+
497+ local_irq_enable();
498+
499+ /* Start the tests. */
500+ for_each_supported_inject_fn(&args, inject_fns, f) {
501+ test_injection(&args, f);
502+ test_preemption(&args, f);
503+ test_injection_failure(&args, f);
504+ }
505+
506+ /* Restore the active state of IRQs. This would happen when live
507+ * migrating IRQs in the middle of being handled.
508+ */
509+ for_each_supported_activate_fn(&args, set_active_fns, f)
510+ test_restore_active(&args, f);
511+
512+ GUEST_DONE();
513+}
514+
515+static void kvm_irq_line_check(struct kvm_vm *vm, uint32_t intid, int level,
516+ struct test_args *test_args, bool expect_failure)
517+{
518+ int ret;
519+
520+ if (!expect_failure) {
521+ kvm_arm_irq_line(vm, intid, level);
522+ } else {
523+ /* The interface doesn't allow larger intid's. */
524+ if (intid > KVM_ARM_IRQ_NUM_MASK)
525+ return;
526+
527+ ret = _kvm_arm_irq_line(vm, intid, level);
528+ TEST_ASSERT(ret != 0 && errno == EINVAL,
529+ "Bad intid %i did not cause KVM_IRQ_LINE "
530+ "error: rc: %i errno: %i", intid, ret, errno);
531+ }
532+}
533+
534+void kvm_irq_set_level_info_check(int gic_fd, uint32_t intid, int level,
535+ bool expect_failure)
536+{
537+ if (!expect_failure) {
538+ kvm_irq_set_level_info(gic_fd, intid, level);
539+ } else {
540+ int ret = _kvm_irq_set_level_info(gic_fd, intid, level);
541+ /*
542+ * The kernel silently fails for invalid SPIs and SGIs (which
543+ * are not level-sensitive). It only checks for intid to not
544+ * spill over 1U << 10 (the max reserved SPI). Also, callers
545+ * are supposed to mask the intid with 0x3ff (1023).
546+ */
547+ if (intid > VGIC_MAX_RESERVED)
548+ TEST_ASSERT(ret != 0 && errno == EINVAL,
549+ "Bad intid %i did not cause VGIC_GRP_LEVEL_INFO "
550+ "error: rc: %i errno: %i", intid, ret, errno);
551+ else
552+ TEST_ASSERT(!ret, "KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO "
553+ "for intid %i failed, rc: %i errno: %i",
554+ intid, ret, errno);
555+ }
556+}
557+
558+static void kvm_set_gsi_routing_irqchip_check(struct kvm_vm *vm,
559+ uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
560+ bool expect_failure)
561+{
562+ struct kvm_irq_routing *routing;
563+ int ret;
564+ uint64_t i;
565+
566+ assert(num <= kvm_max_routes && kvm_max_routes <= KVM_MAX_IRQ_ROUTES);
567+
568+ routing = kvm_gsi_routing_create();
569+ for (i = intid; i < (uint64_t)intid + num; i++)
570+ kvm_gsi_routing_irqchip_add(routing, i - MIN_SPI, i - MIN_SPI);
571+
572+ if (!expect_failure) {
573+ kvm_gsi_routing_write(vm, routing);
574+ } else {
575+ ret = _kvm_gsi_routing_write(vm, routing);
576+ /* The kernel only checks for KVM_IRQCHIP_NUM_PINS. */
577+ if (intid >= KVM_IRQCHIP_NUM_PINS)
578+ TEST_ASSERT(ret != 0 && errno == EINVAL,
579+ "Bad intid %u did not cause KVM_SET_GSI_ROUTING "
580+ "error: rc: %i errno: %i", intid, ret, errno);
581+ else
582+ TEST_ASSERT(ret == 0, "KVM_SET_GSI_ROUTING "
583+ "for intid %i failed, rc: %i errno: %i",
584+ intid, ret, errno);
585+ }
586+}
587+
588+static void kvm_irq_write_ispendr_check(int gic_fd, uint32_t intid,
589+ uint32_t vcpu, bool expect_failure)
590+{
591+ /*
592+ * Ignore this when expecting failure as invalid intids will lead to
593+ * either trying to inject SGIs when we configured the test to be
594+ * level_sensitive (or the reverse), or inject large intids which
595+ * will lead to writing above the ISPENDR register space (and we
596+ * don't want to do that either).
597+ */
598+ if (!expect_failure)
599+ kvm_irq_write_ispendr(gic_fd, intid, vcpu);
600+}
601+
602+static void kvm_routing_and_irqfd_check(struct kvm_vm *vm,
603+ uint32_t intid, uint32_t num, uint32_t kvm_max_routes,
604+ bool expect_failure)
605+{
606+ int fd[MAX_SPI];
607+ uint64_t val;
608+ int ret, f;
609+ uint64_t i;
610+
611+ /*
612+ * There is no way to try injecting an SGI or PPI as the interface
613+ * starts counting from the first SPI (above the private ones), so just
614+ * exit.
615+ */
616+ if (INTID_IS_SGI(intid) || INTID_IS_PPI(intid))
617+ return;
618+
619+ kvm_set_gsi_routing_irqchip_check(vm, intid, num,
620+ kvm_max_routes, expect_failure);
621+
622+ /*
623+ * If expect_failure, then just to inject anyway. These
624+ * will silently fail. And in any case, the guest will check
625+ * that no actual interrupt was injected for those cases.
626+ */
627+
628+ for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
629+ fd[f] = eventfd(0, 0);
630+ TEST_ASSERT(fd[f] != -1,
631+ "eventfd failed, errno: %i\n", errno);
632+ }
633+
634+ for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
635+ struct kvm_irqfd irqfd = {
636+ .fd = fd[f],
637+ .gsi = i - MIN_SPI,
638+ };
639+ assert(i <= (uint64_t)UINT_MAX);
640+ vm_ioctl(vm, KVM_IRQFD, &irqfd);
641+ }
642+
643+ for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++) {
644+ val = 1;
645+ ret = write(fd[f], &val, sizeof(uint64_t));
646+ TEST_ASSERT(ret == sizeof(uint64_t),
647+ "Write to KVM_IRQFD failed with ret: %d\n", ret);
648+ }
649+
650+ for (f = 0, i = intid; i < (uint64_t)intid + num; i++, f++)
651+ close(fd[f]);
652+}
653+
654+/* handles the valid case: intid=0xffffffff num=1 */
655+#define for_each_intid(first, num, tmp, i) \
656+ for ((tmp) = (i) = (first); \
657+ (tmp) < (uint64_t)(first) + (uint64_t)(num); \
658+ (tmp)++, (i)++)
659+
660+static void run_guest_cmd(struct kvm_vm *vm, int gic_fd,
661+ struct kvm_inject_args *inject_args,
662+ struct test_args *test_args)
663+{
664+ kvm_inject_cmd cmd = inject_args->cmd;
665+ uint32_t intid = inject_args->first_intid;
666+ uint32_t num = inject_args->num;
667+ int level = inject_args->level;
668+ bool expect_failure = inject_args->expect_failure;
669+ uint64_t tmp;
670+ uint32_t i;
671+
672+ /* handles the valid case: intid=0xffffffff num=1 */
673+ assert(intid < UINT_MAX - num || num == 1);
674+
675+ switch (cmd) {
676+ case KVM_INJECT_EDGE_IRQ_LINE:
677+ for_each_intid(intid, num, tmp, i)
678+ kvm_irq_line_check(vm, i, 1, test_args,
679+ expect_failure);
680+ for_each_intid(intid, num, tmp, i)
681+ kvm_irq_line_check(vm, i, 0, test_args,
682+ expect_failure);
683+ break;
684+ case KVM_SET_IRQ_LINE:
685+ for_each_intid(intid, num, tmp, i)
686+ kvm_irq_line_check(vm, i, level, test_args,
687+ expect_failure);
688+ break;
689+ case KVM_SET_IRQ_LINE_HIGH:
690+ for_each_intid(intid, num, tmp, i)
691+ kvm_irq_line_check(vm, i, 1, test_args,
692+ expect_failure);
693+ break;
694+ case KVM_SET_LEVEL_INFO_HIGH:
695+ for_each_intid(intid, num, tmp, i)
696+ kvm_irq_set_level_info_check(gic_fd, i, 1,
697+ expect_failure);
698+ break;
699+ case KVM_INJECT_IRQFD:
700+ kvm_routing_and_irqfd_check(vm, intid, num,
701+ test_args->kvm_max_routes,
702+ expect_failure);
703+ break;
704+ case KVM_WRITE_ISPENDR:
705+ for (i = intid; i < intid + num; i++)
706+ kvm_irq_write_ispendr_check(gic_fd, i,
707+ VCPU_ID, expect_failure);
708+ break;
709+ case KVM_WRITE_ISACTIVER:
710+ for (i = intid; i < intid + num; i++)
711+ kvm_irq_write_isactiver(gic_fd, i, VCPU_ID);
712+ break;
713+ default:
714+ break;
715+ }
716+}
717+
718+static void kvm_inject_get_call(struct kvm_vm *vm, struct ucall *uc,
719+ struct kvm_inject_args *args)
720+{
721+ struct kvm_inject_args *kvm_args_hva;
722+ vm_vaddr_t kvm_args_gva;
723+
724+ kvm_args_gva = uc->args[1];
725+ kvm_args_hva = (struct kvm_inject_args *)addr_gva2hva(vm, kvm_args_gva);
726+ memcpy(args, kvm_args_hva, sizeof(struct kvm_inject_args));
727+}
728+
729+static void print_args(struct test_args *args)
730+{
731+ printf("nr-irqs=%d level-sensitive=%d eoi-split=%d\n",
732+ args->nr_irqs, args->level_sensitive,
733+ args->eoi_split);
734+}
735+
736+static void test_vgic(uint32_t nr_irqs, bool level_sensitive, bool eoi_split)
737+{
738+ struct ucall uc;
739+ int gic_fd;
740+ struct kvm_vm *vm;
741+ struct kvm_inject_args inject_args;
742+
743+ struct test_args args = {
744+ .nr_irqs = nr_irqs,
745+ .level_sensitive = level_sensitive,
746+ .eoi_split = eoi_split,
747+ .kvm_max_routes = kvm_check_cap(KVM_CAP_IRQ_ROUTING),
748+ .kvm_supports_irqfd = kvm_check_cap(KVM_CAP_IRQFD),
749+ };
750+
751+ print_args(&args);
752+
753+ vm = vm_create_default(VCPU_ID, 0, guest_code);
754+ ucall_init(vm, NULL);
755+
756+ vm_init_descriptor_tables(vm);
757+ vcpu_init_descriptor_tables(vm, VCPU_ID);
758+
759+ /* Setup the guest args page (so it gets the args). */
760+ vcpu_args_set(vm, 0, 1, args);
761+
762+ gic_fd = vgic_v3_setup(vm, 1, nr_irqs,
763+ GICD_BASE_GPA, GICR_BASE_GPA);
764+
765+ vm_install_exception_handler(vm, VECTOR_IRQ_CURRENT,
766+ guest_irq_handlers[args.eoi_split][args.level_sensitive]);
767+
768+ while (1) {
769+ vcpu_run(vm, VCPU_ID);
770+
771+ switch (get_ucall(vm, VCPU_ID, &uc)) {
772+ case UCALL_SYNC:
773+ kvm_inject_get_call(vm, &uc, &inject_args);
774+ run_guest_cmd(vm, gic_fd, &inject_args, &args);
775+ break;
776+ case UCALL_ABORT:
777+ TEST_FAIL("%s at %s:%ld\n\tvalues: %#lx, %#lx",
778+ (const char *)uc.args[0],
779+ __FILE__, uc.args[1], uc.args[2], uc.args[3]);
780+ break;
781+ case UCALL_DONE:
782+ goto done;
783+ default:
784+ TEST_FAIL("Unknown ucall %lu", uc.cmd);
785+ }
786+ }
787+
788+done:
789+ close(gic_fd);
790+ kvm_vm_free(vm);
791+}
792+
793+static void help(const char *name)
794+{
795+ printf(
796+ "\n"
797+ "usage: %s [-n num_irqs] [-e eoi_split] [-l level_sensitive]\n", name);
798+ printf(" -n: specify number of IRQs to setup the vgic with. "
799+ "It has to be a multiple of 32 and between 64 and 1024.\n");
800+ printf(" -e: if 1 then EOI is split into a write to DIR on top "
801+ "of writing EOI.\n");
802+ printf(" -l: specify whether the IRQs are level-sensitive (1) or not (0).");
803+ puts("");
804+ exit(1);
805+}
806+
807+int main(int argc, char **argv)
808+{
809+ uint32_t nr_irqs = 64;
810+ bool default_args = true;
811+ bool level_sensitive = false;
812+ int opt;
813+ bool eoi_split = false;
814+
815+ /* Tell stdout not to buffer its content */
816+ setbuf(stdout, NULL);
817+
818+ while ((opt = getopt(argc, argv, "hn:e:l:")) != -1) {
819+ switch (opt) {
820+ case 'n':
821+ nr_irqs = atoi(optarg);
822+ if (nr_irqs > 1024 || nr_irqs % 32)
823+ help(argv[0]);
824+ break;
825+ case 'e':
826+ eoi_split = (bool)atoi(optarg);
827+ default_args = false;
828+ break;
829+ case 'l':
830+ level_sensitive = (bool)atoi(optarg);
831+ default_args = false;
832+ break;
833+ case 'h':
834+ default:
835+ help(argv[0]);
836+ break;
837+ }
838+ }
839+
840+ /* If the user just specified nr_irqs and/or gic_version, then run all
841+ * combinations.
842+ */
843+ if (default_args) {
844+ test_vgic(nr_irqs, false /* level */, false /* eoi_split */);
845+ test_vgic(nr_irqs, false /* level */, true /* eoi_split */);
846+ test_vgic(nr_irqs, true /* level */, false /* eoi_split */);
847+ test_vgic(nr_irqs, true /* level */, true /* eoi_split */);
848+ } else {
849+ test_vgic(nr_irqs, level_sensitive, eoi_split);
850+ }
851+
852+ return 0;
853+}
tools/testing/selftests/kvm/include/aarch64/gic.h+26−0
@@ -11,11 +11,37 @@ enum gic_type { · 10 unmodified lines
1111 GIC_TYPE_MAX,
1212 };
1313
14+#define MIN_SGI 0
15+#define MIN_PPI 16
16+#define MIN_SPI 32
17+#define MAX_SPI 1019
18+#define IAR_SPURIOUS 1023
19+
20+#define INTID_IS_SGI(intid) (0 <= (intid) && (intid) < MIN_PPI)
21+#define INTID_IS_PPI(intid) (MIN_PPI <= (intid) && (intid) < MIN_SPI)
22+#define INTID_IS_SPI(intid) (MIN_SPI <= (intid) && (intid) <= MAX_SPI)
23+
1424 void gic_init(enum gic_type type, unsigned int nr_cpus,
1525 void *dist_base, void *redist_base);
1626 void gic_irq_enable(unsigned int intid);
1727 void gic_irq_disable(unsigned int intid);
1828 unsigned int gic_get_and_ack_irq(void);
1929 void gic_set_eoi(unsigned int intid);
30+void gic_set_dir(unsigned int intid);
31+
32+/*
33+ * Sets the EOI mode. When split is false, EOI just drops the priority. When
34+ * split is true, EOI drops the priority and deactivates the interrupt.
35+ */
36+void gic_set_eoi_split(bool split);
37+void gic_set_priority_mask(uint64_t mask);
38+void gic_set_priority(uint32_t intid, uint32_t prio);
39+void gic_irq_set_active(unsigned int intid);
40+void gic_irq_clear_active(unsigned int intid);
41+bool gic_irq_get_active(unsigned int intid);
42+void gic_irq_set_pending(unsigned int intid);
43+void gic_irq_clear_pending(unsigned int intid);
44+bool gic_irq_get_pending(unsigned int intid);
45+void gic_irq_set_config(unsigned int intid, bool is_edge);
2046
2147 #endif /* SELFTEST_KVM_GIC_H */
tools/testing/selftests/kvm/include/aarch64/gic_v3.hrenamed from tools/testing/selftests/kvm/lib/aarch64/gic_v3.h+12−0
@@ -16,8 +16,12 @@ · 15 unmodified lines
1616 #define GICD_IGROUPR 0x0080
1717 #define GICD_ISENABLER 0x0100
1818 #define GICD_ICENABLER 0x0180
19+#define GICD_ISPENDR 0x0200
20+#define GICD_ICPENDR 0x0280
1921 #define GICD_ICACTIVER 0x0380
22+#define GICD_ISACTIVER 0x0300
2023 #define GICD_IPRIORITYR 0x0400
24+#define GICD_ICFGR 0x0C00
2125
2226 /*
2327 * The assumption is that the guest runs in a non-secure mode.
@@ -49,16 +53,24 @@ · 25 unmodified lines
4953 #define GICR_IGROUPR0 GICD_IGROUPR
5054 #define GICR_ISENABLER0 GICD_ISENABLER
5155 #define GICR_ICENABLER0 GICD_ICENABLER
56+#define GICR_ISPENDR0 GICD_ISPENDR
57+#define GICR_ISACTIVER0 GICD_ISACTIVER
5258 #define GICR_ICACTIVER0 GICD_ICACTIVER
59+#define GICR_ICENABLER GICD_ICENABLER
60+#define GICR_ICACTIVER GICD_ICACTIVER
5361 #define GICR_IPRIORITYR0 GICD_IPRIORITYR
5462
5563 /* CPU interface registers */
5664 #define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
5765 #define SYS_ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
5866 #define SYS_ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
67+#define SYS_ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
68+#define SYS_ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
5969 #define SYS_ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
6070 #define SYS_ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
6171
72+#define SYS_ICV_AP1R0_EL1 sys_reg(3, 0, 12, 9, 0)
73+
6274 #define ICC_PMR_DEF_PRIO 0xf0
6375
6476 #define ICC_SRE_EL1_SRE (1U << 0)
tools/testing/selftests/kvm/include/aarch64/processor.h+3−0
@@ -113,6 +113,9 @@ enum { · 112 unmodified lines
113113 #define ESR_EC_WP_CURRENT 0x35
114114 #define ESR_EC_BRK_INS 0x3c
115115
116+void aarch64_get_supported_page_sizes(uint32_t ipa,
117+ bool *ps4k, bool *ps16k, bool *ps64k);
118+
116119 void vm_init_descriptor_tables(struct kvm_vm *vm);
117120 void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid);
118121
tools/testing/selftests/kvm/include/aarch64/vgic.h+16−2
@@ -14,7 +14,21 @@ · 13 unmodified lines
1414 ((uint64_t)(flags) << 12) | \
1515 index)
1616
17−int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus,
17+int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs,
1818 uint64_t gicd_base_gpa, uint64_t gicr_base_gpa);
1919
20−#endif /* SELFTEST_KVM_VGIC_H */
20+#define VGIC_MAX_RESERVED 1023
21+
22+void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
23+int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level);
24+
25+void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
26+int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level);
27+
28+/* The vcpu arg only applies to private interrupts. */
29+void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, uint32_t vcpu);
30+void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, uint32_t vcpu);
31+
32+#define KVM_IRQCHIP_NUM_PINS (1020 - 32)
33+
34+#endif // SELFTEST_KVM_VGIC_H
tools/testing/selftests/kvm/include/kvm_util.h+19−1
@@ -42,18 +42,26 @@ enum vm_guest_mode { · 41 unmodified lines
4242 VM_MODE_P52V48_4K,
4343 VM_MODE_P52V48_64K,
4444 VM_MODE_P48V48_4K,
45+ VM_MODE_P48V48_16K,
4546 VM_MODE_P48V48_64K,
4647 VM_MODE_P40V48_4K,
48+ VM_MODE_P40V48_16K,
4749 VM_MODE_P40V48_64K,
4850 VM_MODE_PXXV48_4K, /* For 48bits VA but ANY bits PA */
4951 VM_MODE_P47V64_4K,
5052 VM_MODE_P44V64_4K,
53+ VM_MODE_P36V48_4K,
54+ VM_MODE_P36V48_16K,
55+ VM_MODE_P36V48_64K,
56+ VM_MODE_P36V47_16K,
5157 NUM_VM_MODES,
5258 };
5359
5460 #if defined(__aarch64__)
5561
56−#define VM_MODE_DEFAULT VM_MODE_P40V48_4K
62+extern enum vm_guest_mode vm_mode_default;
63+
64+#define VM_MODE_DEFAULT vm_mode_default
5765 #define MIN_PAGE_SHIFT 12U
5866 #define ptes_per_page(page_size) ((page_size) / 8)
5967
@@ -240,6 +248,8 @@ int _kvm_device_access(int dev_fd, uint32_t group, uint64_t attr, · 180 unmodified lines
240248 void *val, bool write);
241249 int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
242250 void *val, bool write);
251+void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
252+int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
243253
244254 int _vcpu_has_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group,
245255 uint64_t attr);
@@ -250,6 +260,14 @@ int _vcpu_access_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group, · 4 unmodified lines
250260 int vcpu_access_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group,
251261 uint64_t attr, void *val, bool write);
252262
263+#define KVM_MAX_IRQ_ROUTES 4096
264+
265+struct kvm_irq_routing *kvm_gsi_routing_create(void);
266+void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
267+ uint32_t gsi, uint32_t pin);
268+int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
269+void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
270+
253271 const char *exit_reason_str(unsigned int exit_reason);
254272
255273 void virt_pgd_alloc(struct kvm_vm *vm);
tools/testing/selftests/kvm/lib/aarch64/gic.c+66−0
@@ -93,3 +93,69 @@ void gic_set_eoi(unsigned int intid) · 92 unmodified lines
9393 GUEST_ASSERT(gic_common_ops);
9494 gic_common_ops->gic_write_eoir(intid);
9595 }
96+
97+void gic_set_dir(unsigned int intid)
98+{
99+ GUEST_ASSERT(gic_common_ops);
100+ gic_common_ops->gic_write_dir(intid);
101+}
102+
103+void gic_set_eoi_split(bool split)
104+{
105+ GUEST_ASSERT(gic_common_ops);
106+ gic_common_ops->gic_set_eoi_split(split);
107+}
108+
109+void gic_set_priority_mask(uint64_t pmr)
110+{
111+ GUEST_ASSERT(gic_common_ops);
112+ gic_common_ops->gic_set_priority_mask(pmr);
113+}
114+
115+void gic_set_priority(unsigned int intid, unsigned int prio)
116+{
117+ GUEST_ASSERT(gic_common_ops);
118+ gic_common_ops->gic_set_priority(intid, prio);
119+}
120+
121+void gic_irq_set_active(unsigned int intid)
122+{
123+ GUEST_ASSERT(gic_common_ops);
124+ gic_common_ops->gic_irq_set_active(intid);
125+}
126+
127+void gic_irq_clear_active(unsigned int intid)
128+{
129+ GUEST_ASSERT(gic_common_ops);
130+ gic_common_ops->gic_irq_clear_active(intid);
131+}
132+
133+bool gic_irq_get_active(unsigned int intid)
134+{
135+ GUEST_ASSERT(gic_common_ops);
136+ return gic_common_ops->gic_irq_get_active(intid);
137+}
138+
139+void gic_irq_set_pending(unsigned int intid)
140+{
141+ GUEST_ASSERT(gic_common_ops);
142+ gic_common_ops->gic_irq_set_pending(intid);
143+}
144+
145+void gic_irq_clear_pending(unsigned int intid)
146+{
147+ GUEST_ASSERT(gic_common_ops);
148+ gic_common_ops->gic_irq_clear_pending(intid);
149+}
150+
151+bool gic_irq_get_pending(unsigned int intid)
152+{
153+ GUEST_ASSERT(gic_common_ops);
154+ return gic_common_ops->gic_irq_get_pending(intid);
155+}
156+
157+void gic_irq_set_config(unsigned int intid, bool is_edge)
158+{
159+ GUEST_ASSERT(gic_common_ops);
160+ gic_common_ops->gic_irq_set_config(intid, is_edge);
161+}
tools/testing/selftests/kvm/lib/aarch64/gic_private.h+11−0
@@ -14,6 +14,17 @@ struct gic_common_ops { · 13 unmodified lines
1414 void (*gic_irq_disable)(unsigned int intid);
1515 uint64_t (*gic_read_iar)(void);
1616 void (*gic_write_eoir)(uint32_t irq);
17+ void (*gic_write_dir)(uint32_t irq);
18+ void (*gic_set_eoi_split)(bool split);
19+ void (*gic_set_priority_mask)(uint64_t mask);
20+ void (*gic_set_priority)(uint32_t intid, uint32_t prio);
21+ void (*gic_irq_set_active)(uint32_t intid);
22+ void (*gic_irq_clear_active)(uint32_t intid);
23+ bool (*gic_irq_get_active)(uint32_t intid);
24+ void (*gic_irq_set_pending)(uint32_t intid);
25+ void (*gic_irq_clear_pending)(uint32_t intid);
26+ bool (*gic_irq_get_pending)(uint32_t intid);
27+ void (*gic_irq_set_config)(uint32_t intid, bool is_edge);
1728 };
1829
1930 extern const struct gic_common_ops gicv3_ops;
tools/testing/selftests/kvm/lib/aarch64/gic_v3.c+181−25
@@ -19,7 +19,8 @@ struct gicv3_data { · 18 unmodified lines
1919 unsigned int nr_spis;
2020 };
2121
22−#define sgi_base_from_redist(redist_base) (redist_base + SZ_64K)
22+#define sgi_base_from_redist(redist_base) (redist_base + SZ_64K)
23+#define DIST_BIT (1U << 31)
2324
2425 enum gicv3_intid_range {
2526 SGI_RANGE,
@@ -50,6 +51,14 @@ static void gicv3_gicr_wait_for_rwp(void *redist_base) · 24 unmodified lines
5051 }
5152 }
5253
54+static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)
55+{
56+ if (cpu_or_dist & DIST_BIT)
57+ gicv3_gicd_wait_for_rwp();
58+ else
59+ gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu_or_dist]);
60+}
61+
5362 static enum gicv3_intid_range get_intid_range(unsigned int intid)
5463 {
5564 switch (intid) {
@@ -81,39 +90,175 @@ static void gicv3_write_eoir(uint32_t irq) · 25 unmodified lines
8190 isb();
8291 }
8392
84−static void
85−gicv3_config_irq(unsigned int intid, unsigned int offset)
93+static void gicv3_write_dir(uint32_t irq)
94+{
95+ write_sysreg_s(irq, SYS_ICC_DIR_EL1);
96+ isb();
97+}
98+
99+static void gicv3_set_priority_mask(uint64_t mask)
100+{
101+ write_sysreg_s(mask, SYS_ICC_PMR_EL1);
102+}
103+
104+static void gicv3_set_eoi_split(bool split)
105+{
106+ uint32_t val;
107+
108+ /* All other fields are read-only, so no need to read CTLR first. In
109+ * fact, the kernel does the same.
110+ */
111+ val = split ? (1U << 1) : 0;
112+ write_sysreg_s(val, SYS_ICC_CTLR_EL1);
113+ isb();
114+}
115+
116+uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)
117+{
118+ void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
119+ : sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
120+ return readl(base + offset);
121+}
122+
123+void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)
124+{
125+ void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
126+ : sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
127+ writel(reg_val, base + offset);
128+}
129+
130+uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)
131+{
132+ return gicv3_reg_readl(cpu_or_dist, offset) & mask;
133+}
134+
135+void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
136+ uint32_t mask, uint32_t reg_val)
137+{
138+ uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
139+
140+ tmp |= (reg_val & mask);
141+ gicv3_reg_writel(cpu_or_dist, offset, tmp);
142+}
143+
144+/*
145+ * We use a single offset for the distributor and redistributor maps as they
146+ * have the same value in both. The only exceptions are registers that only
147+ * exist in one and not the other, like GICR_WAKER that doesn't exist in the
148+ * distributor map. Such registers are conveniently marked as reserved in the
149+ * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
150+ * marked as "Reserved" in the Distributor map.
151+ */
152+static void gicv3_access_reg(uint32_t intid, uint64_t offset,
153+ uint32_t reg_bits, uint32_t bits_per_field,
154+ bool write, uint32_t *val)
86155 {
87156 uint32_t cpu = guest_get_vcpuid();
88− uint32_t mask = 1 << (intid % 32);
89157 enum gicv3_intid_range intid_range = get_intid_range(intid);
90− void *reg;
91−
92− /* We care about 'cpu' only for SGIs or PPIs */
93− if (intid_range == SGI_RANGE || intid_range == PPI_RANGE) {
94− GUEST_ASSERT(cpu < gicv3_data.nr_cpus);
95−
96− reg = sgi_base_from_redist(gicv3_data.redist_base[cpu]) +
97− offset;
98− writel(mask, reg);
99− gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu]);
100− } else if (intid_range == SPI_RANGE) {
101− reg = gicv3_data.dist_base + offset + (intid / 32) * 4;
102− writel(mask, reg);
103− gicv3_gicd_wait_for_rwp();
104− } else {
105− GUEST_ASSERT(0);
106− }
158+ uint32_t fields_per_reg, index, mask, shift;
159+ uint32_t cpu_or_dist;
160+
161+ GUEST_ASSERT(bits_per_field <= reg_bits);
162+ GUEST_ASSERT(*val < (1U << bits_per_field));
163+ /* Some registers like IROUTER are 64 bit long. Those are currently not
164+ * supported by readl nor writel, so just asserting here until then.
165+ */
166+ GUEST_ASSERT(reg_bits == 32);
167+
168+ fields_per_reg = reg_bits / bits_per_field;
169+ index = intid % fields_per_reg;
170+ shift = index * bits_per_field;
171+ mask = ((1U << bits_per_field) - 1) << shift;
172+
173+ /* Set offset to the actual register holding intid's config. */
174+ offset += (intid / fields_per_reg) * (reg_bits / 8);
175+
176+ cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;
177+
178+ if (write)
179+ gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);
180+ *val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
181+}
182+
183+static void gicv3_write_reg(uint32_t intid, uint64_t offset,
184+ uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)
185+{
186+ gicv3_access_reg(intid, offset, reg_bits,
187+ bits_per_field, true, &val);
188+}
189+
190+static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,
191+ uint32_t reg_bits, uint32_t bits_per_field)
192+{
193+ uint32_t val;
194+
195+ gicv3_access_reg(intid, offset, reg_bits,
196+ bits_per_field, false, &val);
197+ return val;
198+}
199+
200+static void gicv3_set_priority(uint32_t intid, uint32_t prio)
201+{
202+ gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
203+}
204+
205+/* Sets the intid to be level-sensitive or edge-triggered. */
206+static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
207+{
208+ uint32_t val;
209+
210+ /* N/A for private interrupts. */
211+ GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
212+ val = is_edge ? 2 : 0;
213+ gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
214+}
215+
216+static void gicv3_irq_enable(uint32_t intid)
217+{
218+ bool is_spi = get_intid_range(intid) == SPI_RANGE;
219+ uint32_t cpu = guest_get_vcpuid();
220+
221+ gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
222+ gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
223+}
224+
225+static void gicv3_irq_disable(uint32_t intid)
226+{
227+ bool is_spi = get_intid_range(intid) == SPI_RANGE;
228+ uint32_t cpu = guest_get_vcpuid();
229+
230+ gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
231+ gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
232+}
233+
234+static void gicv3_irq_set_active(uint32_t intid)
235+{
236+ gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
237+}
238+
239+static void gicv3_irq_clear_active(uint32_t intid)
240+{
241+ gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
242+}
243+
244+static bool gicv3_irq_get_active(uint32_t intid)
245+{
246+ return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
247+}
248+
249+static void gicv3_irq_set_pending(uint32_t intid)
250+{
251+ gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
107252 }
108253
109−static void gicv3_irq_enable(unsigned int intid)
254+static void gicv3_irq_clear_pending(uint32_t intid)
110255 {
111− gicv3_config_irq(intid, GICD_ISENABLER);
256+ gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
112257 }
113258
114−static void gicv3_irq_disable(unsigned int intid)
259+static bool gicv3_irq_get_pending(uint32_t intid)
115260 {
116− gicv3_config_irq(intid, GICD_ICENABLER);
261+ return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
117262 }
118263
119264 static void gicv3_enable_redist(void *redist_base)
@@ -237,4 +382,15 @@ const struct gic_common_ops gicv3_ops = { · 117 unmodified lines
237382 .gic_irq_disable = gicv3_irq_disable,
238383 .gic_read_iar = gicv3_read_iar,
239384 .gic_write_eoir = gicv3_write_eoir,
385+ .gic_write_dir = gicv3_write_dir,
386+ .gic_set_priority_mask = gicv3_set_priority_mask,
387+ .gic_set_eoi_split = gicv3_set_eoi_split,
388+ .gic_set_priority = gicv3_set_priority,
389+ .gic_irq_set_active = gicv3_irq_set_active,
390+ .gic_irq_clear_active = gicv3_irq_clear_active,
391+ .gic_irq_get_active = gicv3_irq_get_active,
392+ .gic_irq_set_pending = gicv3_irq_set_pending,
393+ .gic_irq_clear_pending = gicv3_irq_clear_pending,
394+ .gic_irq_get_pending = gicv3_irq_get_pending,
395+ .gic_irq_set_config = gicv3_irq_set_config,
240396 };
tools/testing/selftests/kvm/lib/aarch64/processor.c+75−7
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <linux/compiler.h>
99 #include <assert.h>
1010
11+#include "guest_modes.h"
1112 #include "kvm_util.h"
1213 #include "../kvm_util_internal.h"
1314 #include "processor.h"
@@ -237,6 +238,7 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init · 223 unmodified lines
237238 get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), &sctlr_el1);
238239 get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TCR_EL1), &tcr_el1);
239240
241+ /* Configure base granule size */
240242 switch (vm->mode) {
241243 case VM_MODE_P52V48_4K:
242244 TEST_FAIL("AArch64 does not support 4K sized pages "
@@ -245,25 +247,47 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init · 2 unmodified lines
245247 TEST_FAIL("AArch64 does not support 4K sized pages "
246248 "with ANY-bit physical address ranges");
247249 case VM_MODE_P52V48_64K:
250+ case VM_MODE_P48V48_64K:
251+ case VM_MODE_P40V48_64K:
252+ case VM_MODE_P36V48_64K:
248253 tcr_el1 |= 1ul << 14; /* TG0 = 64KB */
249− tcr_el1 |= 6ul << 32; /* IPS = 52 bits */
254+ break;
255+ case VM_MODE_P48V48_16K:
256+ case VM_MODE_P40V48_16K:
257+ case VM_MODE_P36V48_16K:
258+ case VM_MODE_P36V47_16K:
259+ tcr_el1 |= 2ul << 14; /* TG0 = 16KB */
250260 break;
251261 case VM_MODE_P48V48_4K:
262+ case VM_MODE_P40V48_4K:
263+ case VM_MODE_P36V48_4K:
252264 tcr_el1 |= 0ul << 14; /* TG0 = 4KB */
253− tcr_el1 |= 5ul << 32; /* IPS = 48 bits */
254265 break;
266+ default:
267+ TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
268+ }
269+
270+ /* Configure output size */
271+ switch (vm->mode) {
272+ case VM_MODE_P52V48_64K:
273+ tcr_el1 |= 6ul << 32; /* IPS = 52 bits */
274+ break;
275+ case VM_MODE_P48V48_4K:
276+ case VM_MODE_P48V48_16K:
255277 case VM_MODE_P48V48_64K:
256− tcr_el1 |= 1ul << 14; /* TG0 = 64KB */
257278 tcr_el1 |= 5ul << 32; /* IPS = 48 bits */
258279 break;
259280 case VM_MODE_P40V48_4K:
260− tcr_el1 |= 0ul << 14; /* TG0 = 4KB */
261− tcr_el1 |= 2ul << 32; /* IPS = 40 bits */
262− break;
281+ case VM_MODE_P40V48_16K:
263282 case VM_MODE_P40V48_64K:
264− tcr_el1 |= 1ul << 14; /* TG0 = 64KB */
265283 tcr_el1 |= 2ul << 32; /* IPS = 40 bits */
266284 break;
285+ case VM_MODE_P36V48_4K:
286+ case VM_MODE_P36V48_16K:
287+ case VM_MODE_P36V48_64K:
288+ case VM_MODE_P36V47_16K:
289+ tcr_el1 |= 1ul << 32; /* IPS = 36 bits */
290+ break;
267291 default:
268292 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
269293 }
@@ -432,3 +456,47 @@ uint32_t guest_get_vcpuid(void) · 162 unmodified lines
432456 {
433457 return read_sysreg(tpidr_el1);
434458 }
459+
460+void aarch64_get_supported_page_sizes(uint32_t ipa,
461+ bool *ps4k, bool *ps16k, bool *ps64k)
462+{
463+ struct kvm_vcpu_init preferred_init;
464+ int kvm_fd, vm_fd, vcpu_fd, err;
465+ uint64_t val;
466+ struct kvm_one_reg reg = {
467+ .id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1),
468+ .addr = (uint64_t)&val,
469+ };
470+
471+ kvm_fd = open_kvm_dev_path_or_exit();
472+ vm_fd = ioctl(kvm_fd, KVM_CREATE_VM, ipa);
473+ TEST_ASSERT(vm_fd >= 0, "Can't create VM");
474+
475+ vcpu_fd = ioctl(vm_fd, KVM_CREATE_VCPU, 0);
476+ TEST_ASSERT(vcpu_fd >= 0, "Can't create vcpu");
477+
478+ err = ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init);
479+ TEST_ASSERT(err == 0, "Can't get target");
480+ err = ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init);
481+ TEST_ASSERT(err == 0, "Can't get init vcpu");
482+
483+ err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);
484+ TEST_ASSERT(err == 0, "Can't get MMFR0");
485+
486+ *ps4k = ((val >> 28) & 0xf) != 0xf;
487+ *ps64k = ((val >> 24) & 0xf) == 0;
488+ *ps16k = ((val >> 20) & 0xf) != 0;
489+
490+ close(vcpu_fd);
491+ close(vm_fd);
492+ close(kvm_fd);
493+}
494+
495+/*
496+ * arm64 doesn't have a true default mode, so start by computing the
497+ * available IPA space and page sizes early.
498+ */
499+void __attribute__((constructor)) init_guest_modes(void)
500+{
501+ guest_modes_append_default();
502+}
tools/testing/selftests/kvm/lib/aarch64/vgic.c+102−1
@@ -5,11 +5,14 @@ · 4 unmodified lines
55
66 #include <linux/kvm.h>
77 #include <linux/sizes.h>
8+#include <asm/kvm_para.h>
89 #include <asm/kvm.h>
910
1011 #include "kvm_util.h"
1112 #include "../kvm_util_internal.h"
1213 #include "vgic.h"
14+#include "gic.h"
15+#include "gic_v3.h"
1316
1417 /*
1518 * vGIC-v3 default host setup
@@ -28,7 +31,7 @@ · 12 unmodified lines
2831 * redistributor regions of the guest. Since it depends on the number of
2932 * vCPUs for the VM, it must be called after all the vCPUs have been created.
3033 */
31−int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus,
34+int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, uint32_t nr_irqs,
3235 uint64_t gicd_base_gpa, uint64_t gicr_base_gpa)
3336 {
3437 int gic_fd;
@@ -50,6 +53,13 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, · 15 unmodified lines
5053
5154 /* Distributor setup */
5255 gic_fd = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3, false);
56+
57+ kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
58+ 0, &nr_irqs, true);
59+
60+ kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
61+ KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
62+
5363 kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
5464 KVM_VGIC_V3_ADDR_TYPE_DIST, &gicd_base_gpa, true);
5565 nr_gic_pages = vm_calc_num_guest_pages(vm->mode, KVM_VGIC_V3_DIST_SIZE);
@@ -68,3 +78,94 @@ int vgic_v3_setup(struct kvm_vm *vm, unsigned int nr_vcpus, · 12 unmodified lines
6878
6979 return gic_fd;
7080 }
81+
82+/* should only work for level sensitive interrupts */
83+int _kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
84+{
85+ uint64_t attr = 32 * (intid / 32);
86+ uint64_t index = intid % 32;
87+ uint64_t val;
88+ int ret;
89+
90+ ret = _kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
91+ attr, &val, false);
92+ if (ret != 0)
93+ return ret;
94+
95+ val |= 1U << index;
96+ ret = _kvm_device_access(gic_fd, KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO,
97+ attr, &val, true);
98+ return ret;
99+}
100+
101+void kvm_irq_set_level_info(int gic_fd, uint32_t intid, int level)
102+{
103+ int ret = _kvm_irq_set_level_info(gic_fd, intid, level);
104+
105+ TEST_ASSERT(ret == 0, "KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO failed, "
106+ "rc: %i errno: %i", ret, errno);
107+}
108+
109+int _kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
110+{
111+ uint32_t irq = intid & KVM_ARM_IRQ_NUM_MASK;
112+
113+ TEST_ASSERT(!INTID_IS_SGI(intid), "KVM_IRQ_LINE's interface itself "
114+ "doesn't allow injecting SGIs. There's no mask for it.");
115+
116+ if (INTID_IS_PPI(intid))
117+ irq |= KVM_ARM_IRQ_TYPE_PPI << KVM_ARM_IRQ_TYPE_SHIFT;
118+ else
119+ irq |= KVM_ARM_IRQ_TYPE_SPI << KVM_ARM_IRQ_TYPE_SHIFT;
120+
121+ return _kvm_irq_line(vm, irq, level);
122+}
123+
124+void kvm_arm_irq_line(struct kvm_vm *vm, uint32_t intid, int level)
125+{
126+ int ret = _kvm_arm_irq_line(vm, intid, level);
127+
128+ TEST_ASSERT(ret == 0, "KVM_IRQ_LINE failed, rc: %i errno: %i",
129+ ret, errno);
130+}
131+
132+static void vgic_poke_irq(int gic_fd, uint32_t intid,
133+ uint32_t vcpu, uint64_t reg_off)
134+{
135+ uint64_t reg = intid / 32;
136+ uint64_t index = intid % 32;
137+ uint64_t attr = reg_off + reg * 4;
138+ uint64_t val;
139+ bool intid_is_private = INTID_IS_SGI(intid) || INTID_IS_PPI(intid);
140+
141+ /* Check that the addr part of the attr is within 32 bits. */
142+ assert(attr <= KVM_DEV_ARM_VGIC_OFFSET_MASK);
143+
144+ uint32_t group = intid_is_private ? KVM_DEV_ARM_VGIC_GRP_REDIST_REGS
145+ : KVM_DEV_ARM_VGIC_GRP_DIST_REGS;
146+
147+ if (intid_is_private) {
148+ /* TODO: only vcpu 0 implemented for now. */
149+ assert(vcpu == 0);
150+ attr += SZ_64K;
151+ }
152+
153+ /* All calls will succeed, even with invalid intid's, as long as the
154+ * addr part of the attr is within 32 bits (checked above). An invalid
155+ * intid will just make the read/writes point to above the intended
156+ * register space (i.e., ICPENDR after ISPENDR).
157+ */
158+ kvm_device_access(gic_fd, group, attr, &val, false);
159+ val |= 1ULL << index;
160+ kvm_device_access(gic_fd, group, attr, &val, true);
161+}
162+
163+void kvm_irq_write_ispendr(int gic_fd, uint32_t intid, uint32_t vcpu)
164+{
165+ vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISPENDR);
166+}
167+
168+void kvm_irq_write_isactiver(int gic_fd, uint32_t intid, uint32_t vcpu)
169+{
170+ vgic_poke_irq(gic_fd, intid, vcpu, GICD_ISACTIVER);
171+}
tools/testing/selftests/kvm/lib/guest_modes.c+43−6
@@ -4,22 +4,59 @@ · 3 unmodified lines
44 */
55 #include "guest_modes.h"
66
7+#ifdef __aarch64__
8+#include "processor.h"
9+enum vm_guest_mode vm_mode_default;
10+#endif
11+
712 struct guest_mode guest_modes[NUM_VM_MODES];
813
914 void guest_modes_append_default(void)
1015 {
16+#ifndef __aarch64__
1117 guest_mode_append(VM_MODE_DEFAULT, true, true);
12−
13−#ifdef __aarch64__
14− guest_mode_append(VM_MODE_P40V48_64K, true, true);
18+#else
1519 {
1620 unsigned int limit = kvm_check_cap(KVM_CAP_ARM_VM_IPA_SIZE);
21+ bool ps4k, ps16k, ps64k;
22+ int i;
23+
24+ aarch64_get_supported_page_sizes(limit, &ps4k, &ps16k, &ps64k);
25+
26+ vm_mode_default = NUM_VM_MODES;
27+
1728 if (limit >= 52)
18− guest_mode_append(VM_MODE_P52V48_64K, true, true);
29+ guest_mode_append(VM_MODE_P52V48_64K, ps64k, ps64k);
1930 if (limit >= 48) {
20− guest_mode_append(VM_MODE_P48V48_4K, true, true);
21− guest_mode_append(VM_MODE_P48V48_64K, true, true);
31+ guest_mode_append(VM_MODE_P48V48_4K, ps4k, ps4k);
32+ guest_mode_append(VM_MODE_P48V48_16K, ps16k, ps16k);
33+ guest_mode_append(VM_MODE_P48V48_64K, ps64k, ps64k);
34+ }
35+ if (limit >= 40) {
36+ guest_mode_append(VM_MODE_P40V48_4K, ps4k, ps4k);
37+ guest_mode_append(VM_MODE_P40V48_16K, ps16k, ps16k);
38+ guest_mode_append(VM_MODE_P40V48_64K, ps64k, ps64k);
39+ if (ps4k)
40+ vm_mode_default = VM_MODE_P40V48_4K;
41+ }
42+ if (limit >= 36) {
43+ guest_mode_append(VM_MODE_P36V48_4K, ps4k, ps4k);
44+ guest_mode_append(VM_MODE_P36V48_16K, ps16k, ps16k);
45+ guest_mode_append(VM_MODE_P36V48_64K, ps64k, ps64k);
46+ guest_mode_append(VM_MODE_P36V47_16K, ps16k, ps16k);
2247 }
48+
49+ /*
50+ * Pick the first supported IPA size if the default
51+ * isn't available.
52+ */
53+ for (i = 0; vm_mode_default == NUM_VM_MODES && i < NUM_VM_MODES; i++) {
54+ if (guest_modes[i].supported && guest_modes[i].enabled)
55+ vm_mode_default = i;
56+ }
57+
58+ TEST_ASSERT(vm_mode_default != NUM_VM_MODES,
59+ "No supported mode!");
2360 }
2461 #endif
2562 #ifdef __s390x__
tools/testing/selftests/kvm/lib/kvm_util.c+94−0
@@ -166,12 +166,18 @@ const char *vm_guest_mode_string(uint32_t i) · 165 unmodified lines
166166 [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
167167 [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",
168168 [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",
169+ [VM_MODE_P48V48_16K] = "PA-bits:48, VA-bits:48, 16K pages",
169170 [VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",
170171 [VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
172+ [VM_MODE_P40V48_16K] = "PA-bits:40, VA-bits:48, 16K pages",
171173 [VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
172174 [VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",
173175 [VM_MODE_P47V64_4K] = "PA-bits:47, VA-bits:64, 4K pages",
174176 [VM_MODE_P44V64_4K] = "PA-bits:44, VA-bits:64, 4K pages",
177+ [VM_MODE_P36V48_4K] = "PA-bits:36, VA-bits:48, 4K pages",
178+ [VM_MODE_P36V48_16K] = "PA-bits:36, VA-bits:48, 16K pages",
179+ [VM_MODE_P36V48_64K] = "PA-bits:36, VA-bits:48, 64K pages",
180+ [VM_MODE_P36V47_16K] = "PA-bits:36, VA-bits:47, 16K pages",
175181 };
176182 _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
177183 "Missing new mode strings?");
@@ -185,12 +191,18 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { · 7 unmodified lines
185191 [VM_MODE_P52V48_4K] = { 52, 48, 0x1000, 12 },
186192 [VM_MODE_P52V48_64K] = { 52, 48, 0x10000, 16 },
187193 [VM_MODE_P48V48_4K] = { 48, 48, 0x1000, 12 },
194+ [VM_MODE_P48V48_16K] = { 48, 48, 0x4000, 14 },
188195 [VM_MODE_P48V48_64K] = { 48, 48, 0x10000, 16 },
189196 [VM_MODE_P40V48_4K] = { 40, 48, 0x1000, 12 },
197+ [VM_MODE_P40V48_16K] = { 40, 48, 0x4000, 14 },
190198 [VM_MODE_P40V48_64K] = { 40, 48, 0x10000, 16 },
191199 [VM_MODE_PXXV48_4K] = { 0, 0, 0x1000, 12 },
192200 [VM_MODE_P47V64_4K] = { 47, 64, 0x1000, 12 },
193201 [VM_MODE_P44V64_4K] = { 44, 64, 0x1000, 12 },
202+ [VM_MODE_P36V48_4K] = { 36, 48, 0x1000, 12 },
203+ [VM_MODE_P36V48_16K] = { 36, 48, 0x4000, 14 },
204+ [VM_MODE_P36V48_64K] = { 36, 48, 0x10000, 16 },
205+ [VM_MODE_P36V47_16K] = { 36, 47, 0x4000, 14 },
194206 };
195207 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
196208 "Missing new mode params?");
@@ -252,9 +264,19 @@ struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) · 55 unmodified lines
252264 vm->pgtable_levels = 3;
253265 break;
254266 case VM_MODE_P40V48_4K:
267+ case VM_MODE_P36V48_4K:
255268 vm->pgtable_levels = 4;
256269 break;
257270 case VM_MODE_P40V48_64K:
271+ case VM_MODE_P36V48_64K:
272+ vm->pgtable_levels = 3;
273+ break;
274+ case VM_MODE_P48V48_16K:
275+ case VM_MODE_P40V48_16K:
276+ case VM_MODE_P36V48_16K:
277+ vm->pgtable_levels = 4;
278+ break;
279+ case VM_MODE_P36V47_16K:
258280 vm->pgtable_levels = 3;
259281 break;
260282 case VM_MODE_PXXV48_4K:
@@ -2086,6 +2108,78 @@ int vcpu_access_device_attr(struct kvm_vm *vm, uint32_t vcpuid, uint32_t group, · 1825 unmodified lines
20862108 return ret;
20872109 }
20882110
2111+/*
2112+ * IRQ related functions.
2113+ */
2114+
2115+int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
2116+{
2117+ struct kvm_irq_level irq_level = {
2118+ .irq = irq,
2119+ .level = level,
2120+ };
2121+
2122+ return _vm_ioctl(vm, KVM_IRQ_LINE, &irq_level);
2123+}
2124+
2125+void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level)
2126+{
2127+ int ret = _kvm_irq_line(vm, irq, level);
2128+
2129+ TEST_ASSERT(ret >= 0, "KVM_IRQ_LINE failed, rc: %i errno: %i", ret, errno);
2130+}
2131+
2132+struct kvm_irq_routing *kvm_gsi_routing_create(void)
2133+{
2134+ struct kvm_irq_routing *routing;
2135+ size_t size;
2136+
2137+ size = sizeof(struct kvm_irq_routing);
2138+ /* Allocate space for the max number of entries: this wastes 196 KBs. */
2139+ size += KVM_MAX_IRQ_ROUTES * sizeof(struct kvm_irq_routing_entry);
2140+ routing = calloc(1, size);
2141+ assert(routing);
2142+
2143+ return routing;
2144+}
2145+
2146+void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
2147+ uint32_t gsi, uint32_t pin)
2148+{
2149+ int i;
2150+
2151+ assert(routing);
2152+ assert(routing->nr < KVM_MAX_IRQ_ROUTES);
2153+
2154+ i = routing->nr;
2155+ routing->entries[i].gsi = gsi;
2156+ routing->entries[i].type = KVM_IRQ_ROUTING_IRQCHIP;
2157+ routing->entries[i].flags = 0;
2158+ routing->entries[i].u.irqchip.irqchip = 0;
2159+ routing->entries[i].u.irqchip.pin = pin;
2160+ routing->nr++;
2161+}
2162+
2163+int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
2164+{
2165+ int ret;
2166+
2167+ assert(routing);
2168+ ret = ioctl(vm_get_fd(vm), KVM_SET_GSI_ROUTING, routing);
2169+ free(routing);
2170+
2171+ return ret;
2172+}
2173+
2174+void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing)
2175+{
2176+ int ret;
2177+
2178+ ret = _kvm_gsi_routing_write(vm, routing);
2179+ TEST_ASSERT(ret == 0, "KVM_SET_GSI_ROUTING failed, rc: %i errno: %i",
2180+ ret, errno);
2181+}
2182+
20892183 /*
20902184 * VM Dump
20912185 *

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t