History

Merge tag 'kvmarm-6.2' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 updates for 6.2 - Enable the per-vcpu dirty-ring tracking mechanism, together with an option to keep the good old dirty log around for pages that are dirtied by something other than a vcpu. - Switch to the relaxed parallel fault handling, using RCU to delay page table reclaim and giving better performance under load. - Relax the MTE ABI, allowing a VMM to use the MAP_SHARED mapping option, which multi-process VMMs such as crosvm rely on. - Merge the pKVM shadow vcpu state tracking that allows the hypervisor to have its own view of a vcpu, keeping that state private. - Add support for the PMUv3p5 architecture revision, bringing support for 64bit counters on systems that support it, and fix the no-quite-compliant CHAIN-ed counter support for the machines that actually exist out there. - Fix a handful of minor issues around 52bit VA/PA support (64kB pages only) as a prefix of the oncoming support for 4kB and 16kB pages. - Add/Enable/Fix a bunch of selftests covering memslots, breakpoints, stage-2 faults and access tracking. You name it, we got it, we probably broke it. - Pick a small set of documentation and spelling fixes, because no good merge window would be complete without those. As a side effect, this tag also drags: - The 'kvmarm-fixes-6.1-3' tag as a dependency to the dirty-ring series - A shared branch with the arm64 tree that repaints all the system registers to match the ARM ARM's naming, and resulting in interesting conflicts

Paolo Bonzini committed 3 years ago

Commit eb561892 parents 1e79a9e 753d73491 files · +6077 −1770Browse files

Unified · Split

91 files
  1. Documentation/virt/kvm/api.rst +31 −10
  2. Documentation/virt/kvm/arm/pvtime.rst +8 −6
  3. Documentation/virt/kvm/devices/arm-vgic-its.rst +4 −1
  4. Documentation/virt/kvm/devices/vcpu.rst +2 −0
  5. arch/arm64/Kconfig +1 −0
  6. arch/arm64/include/asm/kvm_arm.h +6 −2
  7. arch/arm64/include/asm/kvm_asm.h +5 −2
  8. arch/arm64/include/asm/kvm_host.h +74 −2
  9. arch/arm64/include/asm/kvm_hyp.h +3 −0
  10. arch/arm64/include/asm/kvm_mmu.h +1 −1
  11. arch/arm64/include/asm/kvm_pgtable.h +147 −28
  12. arch/arm64/include/asm/kvm_pkvm.h +38 −0
  13. arch/arm64/include/asm/mte.h +64 −1
  14. arch/arm64/include/asm/pgtable.h +2 −2
  15. arch/arm64/include/asm/sysreg.h +0 −138
  16. arch/arm64/include/uapi/asm/kvm.h +1 −0
  17. arch/arm64/kernel/cpufeature.c +107 −105
  18. arch/arm64/kernel/elfcore.c +1 −1
  19. arch/arm64/kernel/hibernate.c +1 −1
  20. arch/arm64/kernel/image-vars.h +0 −15
  21. arch/arm64/kernel/mte.c +10 −11
  22. arch/arm64/kvm/Kconfig +2 −0
  23. arch/arm64/kvm/arm.c +51 −39
  24. arch/arm64/kvm/guest.c +11 −7
  25. arch/arm64/kvm/hyp/hyp-constants.c +3 −0
  26. arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +21 −4
  27. arch/arm64/kvm/hyp/include/nvhe/memory.h +27 −0
  28. arch/arm64/kvm/hyp/include/nvhe/mm.h +5 −13
  29. arch/arm64/kvm/hyp/include/nvhe/pkvm.h +68 −0
  30. arch/arm64/kvm/hyp/include/nvhe/spinlock.h +9 −1
  31. arch/arm64/kvm/hyp/nvhe/cache.S +11 −0
  32. arch/arm64/kvm/hyp/nvhe/hyp-main.c +108 −2
  33. arch/arm64/kvm/hyp/nvhe/hyp-smp.c +2 −0
  34. arch/arm64/kvm/hyp/nvhe/mem_protect.c +476 −45
  35. arch/arm64/kvm/hyp/nvhe/mm.c +163 −4
  36. arch/arm64/kvm/hyp/nvhe/page_alloc.c +7 −22
  37. arch/arm64/kvm/hyp/nvhe/pkvm.c +436 −0
  38. arch/arm64/kvm/hyp/nvhe/setup.c +61 −37
  39. arch/arm64/kvm/hyp/pgtable.c +343 −309
  40. arch/arm64/kvm/hyp/vhe/Makefile +1 −1
  41. arch/arm64/kvm/mmu.c +132 −61
  42. arch/arm64/kvm/pkvm.c +122 −16
  43. arch/arm64/kvm/pmu-emul.c +196 −286
  44. arch/arm64/kvm/reset.c +0 −29
  45. arch/arm64/kvm/sys_regs.c +135 −22
  46. arch/arm64/kvm/vgic/vgic-its.c +20 −0
  47. arch/arm64/lib/mte.S +1 −1
  48. arch/arm64/mm/copypage.c +5 −2
  49. arch/arm64/mm/fault.c +3 −1
  50. arch/arm64/mm/mteswap.c +6 −10
  51. arch/arm64/tools/gen-sysreg.awk +1 −1
  52. arch/arm64/tools/sysreg +754 −0
  53. arch/x86/include/asm/kvm_host.h +0 −2
  54. arch/x86/kvm/x86.c +6 −9
  55. fs/proc/page.c +2 −1
  56. include/kvm/arm_pmu.h +13 −2
  57. include/kvm/arm_vgic.h +1 −0
  58. include/linux/kernel-page-flags.h +1 −0
  59. include/linux/kvm_dirty_ring.h +12 −8
  60. include/linux/kvm_host.h +6 −4
  61. include/linux/page-flags.h +2 −1
  62. include/trace/events/mmflags.h +5 −4
  63. include/uapi/linux/kvm.h +1 −0
  64. mm/Kconfig +8 −0
  65. mm/huge_memory.c +2 −1
  66. tools/include/linux/bitfield.h +176 −0
  67. tools/testing/selftests/kvm/.gitignore +1 −0
  68. tools/testing/selftests/kvm/Makefile +3 −0
  69. tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c +2 −1
  70. tools/testing/selftests/kvm/aarch64/debug-exceptions.c +239 −72
  71. tools/testing/selftests/kvm/aarch64/page_fault_test.c +1117 −0
  72. tools/testing/selftests/kvm/access_tracking_perf_test.c +1 −7
  73. tools/testing/selftests/kvm/demand_paging_test.c +30 −198
  74. tools/testing/selftests/kvm/dirty_log_test.c +39 −14
  75. tools/testing/selftests/kvm/include/aarch64/processor.h +29 −6
  76. tools/testing/selftests/kvm/include/kvm_util_base.h +29 −2
  77. tools/testing/selftests/kvm/include/memstress.h +3 −0
  78. tools/testing/selftests/kvm/include/userfaultfd_util.h +45 −0
  79. tools/testing/selftests/kvm/lib/aarch64/processor.c +34 −21
  80. tools/testing/selftests/kvm/lib/elf.c +2 −1
  81. tools/testing/selftests/kvm/lib/kvm_util.c +53 −31
  82. tools/testing/selftests/kvm/lib/memstress.c +3 −0
  83. tools/testing/selftests/kvm/lib/riscv/processor.c +17 −12
  84. tools/testing/selftests/kvm/lib/s390x/processor.c +5 −3
  85. tools/testing/selftests/kvm/lib/userfaultfd_util.c +186 −0
  86. tools/testing/selftests/kvm/lib/x86_64/processor.c +7 −6
  87. tools/testing/selftests/kvm/memslot_modification_stress_test.c +1 −5
  88. tools/testing/selftests/kvm/memslot_perf_test.c +207 −106
  89. virt/kvm/Kconfig +6 −0
  90. virt/kvm/dirty_ring.c +44 −2
  91. virt/kvm/kvm_main.c +53 −12
Documentation/virt/kvm/api.rst+31−10
@@ -7418,8 +7418,9 @@ hibernation of the host; however the VMM needs to manually save/restore the · 7417 unmodified lines
74187418 tags as appropriate if the VM is migrated.
74197419
74207420 When this capability is enabled all memory in memslots must be mapped as
7421−not-shareable (no MAP_SHARED), attempts to create a memslot with a
7422−MAP_SHARED mmap will result in an -EINVAL return.
7421+``MAP_ANONYMOUS`` or with a RAM-based file mapping (``tmpfs``, ``memfd``),
7422+attempts to create a memslot with an invalid mmap will result in an
7423+-EINVAL return.
74237424
74247425 When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to
74257426 perform a bulk copy of tags to/from the guest.
@@ -7954,7 +7955,7 @@ regardless of what has actually been exposed through the CPUID leaf. · 528 unmodified lines
79547955 8.29 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL
79557956 ----------------------------------------------------------
79567957
7957−:Architectures: x86
7958+:Architectures: x86, arm64
79587959 :Parameters: args[0] - size of the dirty log ring
79597960
79607961 KVM is capable of tracking dirty memory using ring buffers that are
@@ -8036,13 +8037,6 @@ flushing is done by the KVM_GET_DIRTY_LOG ioctl). To achieve that, one · 75 unmodified lines
80368037 needs to kick the vcpu out of KVM_RUN using a signal. The resulting
80378038 vmexit ensures that all dirty GFNs are flushed to the dirty rings.
80388039
8039−NOTE: the capability KVM_CAP_DIRTY_LOG_RING and the corresponding
8040−ioctl KVM_RESET_DIRTY_RINGS are mutual exclusive to the existing ioctls
8041−KVM_GET_DIRTY_LOG and KVM_CLEAR_DIRTY_LOG. After enabling
8042−KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual
8043−machine will switch to ring-buffer dirty page tracking and further
8044−KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.
8045−
80468040 NOTE: KVM_CAP_DIRTY_LOG_RING_ACQ_REL is the only capability that
80478041 should be exposed by weakly ordered architecture, in order to indicate
80488042 the additional memory ordering requirements imposed on userspace when
@@ -8051,6 +8045,33 @@ Architecture with TSO-like ordering (such as x86) are allowed to · 2 unmodified lines
80518045 expose both KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL
80528046 to userspace.
80538047
8048+After enabling the dirty rings, the userspace needs to detect the
8049+capability of KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP to see whether the
8050+ring structures can be backed by per-slot bitmaps. With this capability
8051+advertised, it means the architecture can dirty guest pages without
8052+vcpu/ring context, so that some of the dirty information will still be
8053+maintained in the bitmap structure. KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP
8054+can't be enabled if the capability of KVM_CAP_DIRTY_LOG_RING_ACQ_REL
8055+hasn't been enabled, or any memslot has been existing.
8056+
8057+Note that the bitmap here is only a backup of the ring structure. The
8058+use of the ring and bitmap combination is only beneficial if there is
8059+only a very small amount of memory that is dirtied out of vcpu/ring
8060+context. Otherwise, the stand-alone per-slot bitmap mechanism needs to
8061+be considered.
8062+
8063+To collect dirty bits in the backup bitmap, userspace can use the same
8064+KVM_GET_DIRTY_LOG ioctl. KVM_CLEAR_DIRTY_LOG isn't needed as long as all
8065+the generation of the dirty bits is done in a single pass. Collecting
8066+the dirty bitmap should be the very last thing that the VMM does before
8067+considering the state as complete. VMM needs to ensure that the dirty
8068+state is final and avoid missing dirty pages from another ioctl ordered
8069+after the bitmap collection.
8070+
8071+NOTE: One example of using the backup bitmap is saving arm64 vgic/its
8072+tables through KVM_DEV_ARM_{VGIC_GRP_CTRL, ITS_SAVE_TABLES} command on
8073+KVM device "kvm-arm-vgic-its" when dirty ring is enabled.
8074+
80548075 8.30 KVM_CAP_XEN_HVM
80558076 --------------------
80568077
Documentation/virt/kvm/arm/pvtime.rst+8−6
@@ -23,21 +23,23 @@ the PV_TIME_FEATURES hypercall should be probed using the SMCCC 1.1 · 22 unmodified lines
2323 ARCH_FEATURES mechanism before calling it.
2424
2525 PV_TIME_FEATURES
26− ============= ======== ==========
26+
27+ ============= ======== =================================================
2728 Function ID: (uint32) 0xC5000020
2829 PV_call_id: (uint32) The function to query for support.
2930 Currently only PV_TIME_ST is supported.
3031 Return value: (int64) NOT_SUPPORTED (-1) or SUCCESS (0) if the relevant
3132 PV-time feature is supported by the hypervisor.
32− ============= ======== ==========
33+ ============= ======== =================================================
3334
3435 PV_TIME_ST
35− ============= ======== ==========
36+
37+ ============= ======== ==============================================
3638 Function ID: (uint32) 0xC5000021
3739 Return value: (int64) IPA of the stolen time data structure for this
3840 VCPU. On failure:
3941 NOT_SUPPORTED (-1)
40− ============= ======== ==========
42+ ============= ======== ==============================================
4143
4244 The IPA returned by PV_TIME_ST should be mapped by the guest as normal memory
4345 with inner and outer write back caching attributes, in the inner shareable
@@ -76,5 +78,5 @@ It is advisable that one or more 64k pages are set aside for the purpose of · 32 unmodified lines
7678 these structures and not used for other purposes, this enables the guest to map
7779 the region using 64k pages and avoids conflicting attributes with other memory.
7880
79−For the user space interface see Documentation/virt/kvm/devices/vcpu.rst
80−section "3. GROUP: KVM_ARM_VCPU_PVTIME_CTRL".
81+For the user space interface see
82+:ref:`Documentation/virt/kvm/devices/vcpu.rst <kvm_arm_vcpu_pvtime_ctrl>`.
Documentation/virt/kvm/devices/arm-vgic-its.rst+4−1
@@ -52,7 +52,10 @@ KVM_DEV_ARM_VGIC_GRP_CTRL · 51 unmodified lines
5252
5353 KVM_DEV_ARM_ITS_SAVE_TABLES
5454 save the ITS table data into guest RAM, at the location provisioned
55− by the guest in corresponding registers/table entries.
55+ by the guest in corresponding registers/table entries. Should userspace
56+ require a form of dirty tracking to identify which pages are modified
57+ by the saving process, it should use a bitmap even if using another
58+ mechanism to track the memory dirtied by the vCPUs.
5659
5760 The layout of the tables in guest memory defines an ABI. The entries
5861 are laid out in little endian format as described in the last paragraph.
Documentation/virt/kvm/devices/vcpu.rst+2−0
@@ -171,6 +171,8 @@ configured values on other VCPUs. Userspace should configure the interrupt · 170 unmodified lines
171171 numbers on at least one VCPU after creating all VCPUs and before running any
172172 VCPUs.
173173
174+.. _kvm_arm_vcpu_pvtime_ctrl:
175+
174176 3. GROUP: KVM_ARM_VCPU_PVTIME_CTRL
175177 ==================================
176178
arch/arm64/Kconfig+1−0
@@ -1965,6 +1965,7 @@ config ARM64_MTE · 1964 unmodified lines
19651965 depends on ARM64_PAN
19661966 select ARCH_HAS_SUBPAGE_FAULTS
19671967 select ARCH_USES_HIGH_VMA_FLAGS
1968+ select ARCH_USES_PG_ARCH_X
19681969 help
19691970 Memory Tagging (part of the ARMv8.5 Extensions) provides
19701971 architectural support for run-time, always-on detection of
arch/arm64/include/asm/kvm_arm.h+6−2
@@ -135,7 +135,7 @@ · 134 unmodified lines
135135 * 40 bits wide (T0SZ = 24). Systems with a PARange smaller than 40 bits are
136136 * not known to exist and will break with this configuration.
137137 *
138− * The VTCR_EL2 is configured per VM and is initialised in kvm_arm_setup_stage2().
138+ * The VTCR_EL2 is configured per VM and is initialised in kvm_init_stage2_mmu.
139139 *
140140 * Note that when using 4K pages, we concatenate two first level page tables
141141 * together. With 16K pages, we concatenate 16 first level page tables.
@@ -340,9 +340,13 @@ · 198 unmodified lines
340340 * We have
341341 * PAR [PA_Shift - 1 : 12] = PA [PA_Shift - 1 : 12]
342342 * HPFAR [PA_Shift - 9 : 4] = FIPA [PA_Shift - 1 : 12]
343+ *
344+ * Always assume 52 bit PA since at this point, we don't know how many PA bits
345+ * the page table has been set up for. This should be safe since unused address
346+ * bits in PAR are res0.
343347 */
344348 #define PAR_TO_HPFAR(par) \
345− (((par) & GENMASK_ULL(PHYS_MASK_SHIFT - 1, 12)) >> 8)
349+ (((par) & GENMASK_ULL(52 - 1, 12)) >> 8)
346350
347351 #define ECN(x) { ESR_ELx_EC_##x, #x }
348352
arch/arm64/include/asm/kvm_asm.h+5−2
@@ -76,6 +76,9 @@ enum __kvm_host_smccc_func { · 75 unmodified lines
7676 __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
7777 __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs,
7878 __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps,
79+ __KVM_HOST_SMCCC_FUNC___pkvm_init_vm,
80+ __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu,
81+ __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm,
7982 };
8083
8184 #define DECLARE_KVM_VHE_SYM(sym) extern char sym[]
@@ -106,7 +109,7 @@ enum __kvm_host_smccc_func { · 24 unmodified lines
106109 #define per_cpu_ptr_nvhe_sym(sym, cpu) \
107110 ({ \
108111 unsigned long base, off; \
109− base = kvm_arm_hyp_percpu_base[cpu]; \
112+ base = kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; \
110113 off = (unsigned long)&CHOOSE_NVHE_SYM(sym) - \
111114 (unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start); \
112115 base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL; \
@@ -211,7 +214,7 @@ DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); · 98 unmodified lines
211214 #define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
212215 #define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
213216
214−extern unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
217+extern unsigned long kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[];
215218 DECLARE_KVM_NVHE_SYM(__per_cpu_start);
216219 DECLARE_KVM_NVHE_SYM(__per_cpu_end);
217220
arch/arm64/include/asm/kvm_host.h+74−2
@@ -73,6 +73,63 @@ u32 __attribute_const__ kvm_target_cpu(void); · 72 unmodified lines
7373 int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
7474 void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu);
7575
76+struct kvm_hyp_memcache {
77+ phys_addr_t head;
78+ unsigned long nr_pages;
79+};
80+
81+static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc,
82+ phys_addr_t *p,
83+ phys_addr_t (*to_pa)(void *virt))
84+{
85+ *p = mc->head;
86+ mc->head = to_pa(p);
87+ mc->nr_pages++;
88+}
89+
90+static inline void *pop_hyp_memcache(struct kvm_hyp_memcache *mc,
91+ void *(*to_va)(phys_addr_t phys))
92+{
93+ phys_addr_t *p = to_va(mc->head);
94+
95+ if (!mc->nr_pages)
96+ return NULL;
97+
98+ mc->head = *p;
99+ mc->nr_pages--;
100+
101+ return p;
102+}
103+
104+static inline int __topup_hyp_memcache(struct kvm_hyp_memcache *mc,
105+ unsigned long min_pages,
106+ void *(*alloc_fn)(void *arg),
107+ phys_addr_t (*to_pa)(void *virt),
108+ void *arg)
109+{
110+ while (mc->nr_pages < min_pages) {
111+ phys_addr_t *p = alloc_fn(arg);
112+
113+ if (!p)
114+ return -ENOMEM;
115+ push_hyp_memcache(mc, p, to_pa);
116+ }
117+
118+ return 0;
119+}
120+
121+static inline void __free_hyp_memcache(struct kvm_hyp_memcache *mc,
122+ void (*free_fn)(void *virt, void *arg),
123+ void *(*to_va)(phys_addr_t phys),
124+ void *arg)
125+{
126+ while (mc->nr_pages)
127+ free_fn(pop_hyp_memcache(mc, to_va), arg);
128+}
129+
130+void free_hyp_memcache(struct kvm_hyp_memcache *mc);
131+int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages);
132+
76133 struct kvm_vmid {
77134 atomic64_t id;
78135 };
@@ -115,6 +172,13 @@ struct kvm_smccc_features { · 36 unmodified lines
115172 unsigned long vendor_hyp_bmap;
116173 };
117174
175+typedef unsigned int pkvm_handle_t;
176+
177+struct kvm_protected_vm {
178+ pkvm_handle_t handle;
179+ struct kvm_hyp_memcache teardown_mc;
180+};
181+
118182 struct kvm_arch {
119183 struct kvm_s2_mmu mmu;
120184
@@ -163,9 +227,19 @@ struct kvm_arch { · 42 unmodified lines
163227
164228 u8 pfr0_csv2;
165229 u8 pfr0_csv3;
230+ struct {
231+ u8 imp:4;
232+ u8 unimp:4;
233+ } dfr0_pmuver;
166234
167235 /* Hypercall features firmware registers' descriptor */
168236 struct kvm_smccc_features smccc_feat;
237+
238+ /*
239+ * For an untrusted host VM, 'pkvm.handle' is used to lookup
240+ * the associated pKVM instance in the hypervisor.
241+ */
242+ struct kvm_protected_vm pkvm;
169243 };
170244
171245 struct kvm_vcpu_fault_info {
@@ -915,8 +989,6 @@ int kvm_set_ipa_limit(void); · 743 unmodified lines
915989 #define __KVM_HAVE_ARCH_VM_ALLOC
916990 struct kvm *kvm_arch_alloc_vm(void);
917991
918−int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type);
919−
920992 static inline bool kvm_vm_is_protected(struct kvm *kvm)
921993 {
922994 return false;
arch/arm64/include/asm/kvm_hyp.h+3−0
@@ -123,4 +123,7 @@ extern u64 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val); · 122 unmodified lines
123123 extern u64 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val);
124124 extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val);
125125
126+extern unsigned long kvm_nvhe_sym(__icache_flags);
127+extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
128+
126129 #endif /* __ARM64_KVM_HYP_H__ */
arch/arm64/include/asm/kvm_mmu.h+1−1
@@ -166,7 +166,7 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, · 165 unmodified lines
166166 void free_hyp_pgds(void);
167167
168168 void stage2_unmap_vm(struct kvm *kvm);
169−int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu);
169+int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type);
170170 void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
171171 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
172172 phys_addr_t pa, unsigned long size, bool writable);
arch/arm64/include/asm/kvm_pgtable.h+147−28
@@ -42,6 +42,8 @@ typedef u64 kvm_pte_t; · 41 unmodified lines
4242 #define KVM_PTE_ADDR_MASK GENMASK(47, PAGE_SHIFT)
4343 #define KVM_PTE_ADDR_51_48 GENMASK(15, 12)
4444
45+#define KVM_PHYS_INVALID (-1ULL)
46+
4547 static inline bool kvm_pte_valid(kvm_pte_t pte)
4648 {
4749 return pte & KVM_PTE_VALID;
@@ -57,6 +59,18 @@ static inline u64 kvm_pte_to_phys(kvm_pte_t pte) · 9 unmodified lines
5759 return pa;
5860 }
5961
62+static inline kvm_pte_t kvm_phys_to_pte(u64 pa)
63+{
64+ kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK;
65+
66+ if (PAGE_SHIFT == 16) {
67+ pa &= GENMASK(51, 48);
68+ pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48);
69+ }
70+
71+ return pte;
72+}
73+
6074 static inline u64 kvm_granule_shift(u32 level)
6175 {
6276 /* Assumes KVM_PGTABLE_MAX_LEVELS is 4 */
@@ -85,6 +99,8 @@ static inline bool kvm_level_supports_block_mapping(u32 level) · 22 unmodified lines
8599 * allocation is physically contiguous.
86100 * @free_pages_exact: Free an exact number of memory pages previously
87101 * allocated by zalloc_pages_exact.
102+ * @free_removed_table: Free a removed paging structure by unlinking and
103+ * dropping references.
88104 * @get_page: Increment the refcount on a page.
89105 * @put_page: Decrement the refcount on a page. When the
90106 * refcount reaches 0 the page is automatically
@@ -103,6 +119,7 @@ struct kvm_pgtable_mm_ops { · 12 unmodified lines
103119 void* (*zalloc_page)(void *arg);
104120 void* (*zalloc_pages_exact)(size_t size);
105121 void (*free_pages_exact)(void *addr, size_t size);
122+ void (*free_removed_table)(void *addr, u32 level);
106123 void (*get_page)(void *addr);
107124 void (*put_page)(void *addr);
108125 int (*page_count)(void *addr);
@@ -161,29 +178,6 @@ enum kvm_pgtable_prot { · 52 unmodified lines
161178 typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
162179 enum kvm_pgtable_prot prot);
163180
164−/**
165− * struct kvm_pgtable - KVM page-table.
166− * @ia_bits: Maximum input address size, in bits.
167− * @start_level: Level at which the page-table walk starts.
168− * @pgd: Pointer to the first top-level entry of the page-table.
169− * @mm_ops: Memory management callbacks.
170− * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
171− * @flags: Stage-2 page-table flags.
172− * @force_pte_cb: Function that returns true if page level mappings must
173− * be used instead of block mappings.
174− */
175−struct kvm_pgtable {
176− u32 ia_bits;
177− u32 start_level;
178− kvm_pte_t *pgd;
179− struct kvm_pgtable_mm_ops *mm_ops;
180−
181− /* Stage-2 only */
182− struct kvm_s2_mmu *mmu;
183− enum kvm_pgtable_stage2_flags flags;
184− kvm_pgtable_force_pte_cb_t force_pte_cb;
185−};
186−
187181 /**
188182 * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.
189183 * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid
@@ -192,17 +186,34 @@ struct kvm_pgtable { · 2 unmodified lines
192186 * children.
193187 * @KVM_PGTABLE_WALK_TABLE_POST: Visit table entries after their
194188 * children.
189+ * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared
190+ * with other software walkers.
195191 */
196192 enum kvm_pgtable_walk_flags {
197193 KVM_PGTABLE_WALK_LEAF = BIT(0),
198194 KVM_PGTABLE_WALK_TABLE_PRE = BIT(1),
199195 KVM_PGTABLE_WALK_TABLE_POST = BIT(2),
196+ KVM_PGTABLE_WALK_SHARED = BIT(3),
197+};
198+
199+struct kvm_pgtable_visit_ctx {
200+ kvm_pte_t *ptep;
201+ kvm_pte_t old;
202+ void *arg;
203+ struct kvm_pgtable_mm_ops *mm_ops;
204+ u64 addr;
205+ u64 end;
206+ u32 level;
207+ enum kvm_pgtable_walk_flags flags;
200208 };
201209
202−typedef int (*kvm_pgtable_visitor_fn_t)(u64 addr, u64 end, u32 level,
203− kvm_pte_t *ptep,
204− enum kvm_pgtable_walk_flags flag,
205− void * const arg);
210+typedef int (*kvm_pgtable_visitor_fn_t)(const struct kvm_pgtable_visit_ctx *ctx,
211+ enum kvm_pgtable_walk_flags visit);
212+
213+static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *ctx)
214+{
215+ return ctx->flags & KVM_PGTABLE_WALK_SHARED;
216+}
206217
207218 /**
208219 * struct kvm_pgtable_walker - Hook into a page-table walk.
@@ -217,6 +228,94 @@ struct kvm_pgtable_walker { · 8 unmodified lines
217228 const enum kvm_pgtable_walk_flags flags;
218229 };
219230
231+/*
232+ * RCU cannot be used in a non-kernel context such as the hyp. As such, page
233+ * table walkers used in hyp do not call into RCU and instead use other
234+ * synchronization mechanisms (such as a spinlock).
235+ */
236+#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
237+
238+typedef kvm_pte_t *kvm_pteref_t;
239+
240+static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
241+ kvm_pteref_t pteref)
242+{
243+ return pteref;
244+}
245+
246+static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
247+{
248+ /*
249+ * Due to the lack of RCU (or a similar protection scheme), only
250+ * non-shared table walkers are allowed in the hypervisor.
251+ */
252+ if (walker->flags & KVM_PGTABLE_WALK_SHARED)
253+ return -EPERM;
254+
255+ return 0;
256+}
257+
258+static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker) {}
259+
260+static inline bool kvm_pgtable_walk_lock_held(void)
261+{
262+ return true;
263+}
264+
265+#else
266+
267+typedef kvm_pte_t __rcu *kvm_pteref_t;
268+
269+static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
270+ kvm_pteref_t pteref)
271+{
272+ return rcu_dereference_check(pteref, !(walker->flags & KVM_PGTABLE_WALK_SHARED));
273+}
274+
275+static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
276+{
277+ if (walker->flags & KVM_PGTABLE_WALK_SHARED)
278+ rcu_read_lock();
279+
280+ return 0;
281+}
282+
283+static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker)
284+{
285+ if (walker->flags & KVM_PGTABLE_WALK_SHARED)
286+ rcu_read_unlock();
287+}
288+
289+static inline bool kvm_pgtable_walk_lock_held(void)
290+{
291+ return rcu_read_lock_held();
292+}
293+
294+#endif
295+
296+/**
297+ * struct kvm_pgtable - KVM page-table.
298+ * @ia_bits: Maximum input address size, in bits.
299+ * @start_level: Level at which the page-table walk starts.
300+ * @pgd: Pointer to the first top-level entry of the page-table.
301+ * @mm_ops: Memory management callbacks.
302+ * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
303+ * @flags: Stage-2 page-table flags.
304+ * @force_pte_cb: Function that returns true if page level mappings must
305+ * be used instead of block mappings.
306+ */
307+struct kvm_pgtable {
308+ u32 ia_bits;
309+ u32 start_level;
310+ kvm_pteref_t pgd;
311+ struct kvm_pgtable_mm_ops *mm_ops;
312+
313+ /* Stage-2 only */
314+ struct kvm_s2_mmu *mmu;
315+ enum kvm_pgtable_stage2_flags flags;
316+ kvm_pgtable_force_pte_cb_t force_pte_cb;
317+};
318+
220319 /**
221320 * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
222321 * @pgt: Uninitialised page-table structure to initialise.
@@ -296,6 +395,14 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); · 73 unmodified lines
296395 */
297396 u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift);
298397
398+/**
399+ * kvm_pgtable_stage2_pgd_size() - Helper to compute size of a stage-2 PGD
400+ * @vtcr: Content of the VTCR register.
401+ *
402+ * Return: the size (in bytes) of the stage-2 PGD
403+ */
404+size_t kvm_pgtable_stage2_pgd_size(u64 vtcr);
405+
299406 /**
300407 * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
301408 * @pgt: Uninitialised page-table structure to initialise.
@@ -324,6 +431,17 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 22 unmodified lines
324431 */
325432 void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
326433
434+/**
435+ * kvm_pgtable_stage2_free_removed() - Free a removed stage-2 paging structure.
436+ * @mm_ops: Memory management callbacks.
437+ * @pgtable: Unlinked stage-2 paging structure to be freed.
438+ * @level: Level of the stage-2 paging structure to be freed.
439+ *
440+ * The page-table is assumed to be unreachable by any hardware walkers prior to
441+ * freeing and therefore no TLB invalidation is performed.
442+ */
443+void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level);
444+
327445 /**
328446 * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table.
329447 * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init*().
@@ -333,6 +451,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); · 3 unmodified lines
333451 * @prot: Permissions and attributes for the mapping.
334452 * @mc: Cache of pre-allocated and zeroed memory from which to allocate
335453 * page-table pages.
454+ * @flags: Flags to control the page-table walk (ex. a shared walk)
336455 *
337456 * The offset of @addr within a page is ignored, @size is rounded-up to
338457 * the next page boundary and @phys is rounded-down to the previous page
@@ -354,7 +473,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); · 15 unmodified lines
354473 */
355474 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
356475 u64 phys, enum kvm_pgtable_prot prot,
357− void *mc);
476+ void *mc, enum kvm_pgtable_walk_flags flags);
358477
359478 /**
360479 * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to
arch/arm64/include/asm/kvm_pkvm.h+38−0
@@ -9,11 +9,49 @@ · 8 unmodified lines
99 #include <linux/memblock.h>
1010 #include <asm/kvm_pgtable.h>
1111
12+/* Maximum number of VMs that can co-exist under pKVM. */
13+#define KVM_MAX_PVMS 255
14+
1215 #define HYP_MEMBLOCK_REGIONS 128
1316
17+int pkvm_init_host_vm(struct kvm *kvm);
18+int pkvm_create_hyp_vm(struct kvm *kvm);
19+void pkvm_destroy_hyp_vm(struct kvm *kvm);
20+
1421 extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
1522 extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
1623
24+static inline unsigned long
25+hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size)
26+{
27+ unsigned long nr_pages = reg->size >> PAGE_SHIFT;
28+ unsigned long start, end;
29+
30+ start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size;
31+ end = start + nr_pages * vmemmap_entry_size;
32+ start = ALIGN_DOWN(start, PAGE_SIZE);
33+ end = ALIGN(end, PAGE_SIZE);
34+
35+ return end - start;
36+}
37+
38+static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size)
39+{
40+ unsigned long res = 0, i;
41+
42+ for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
43+ res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i],
44+ vmemmap_entry_size);
45+ }
46+
47+ return res >> PAGE_SHIFT;
48+}
49+
50+static inline unsigned long hyp_vm_table_pages(void)
51+{
52+ return PAGE_ALIGN(KVM_MAX_PVMS * sizeof(void *)) >> PAGE_SHIFT;
53+}
54+
1755 static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
1856 {
1957 unsigned long total = 0, i;
arch/arm64/include/asm/mte.h+64−1
@@ -25,7 +25,7 @@ unsigned long mte_copy_tags_to_user(void __user *to, void *from, · 24 unmodified lines
2525 unsigned long n);
2626 int mte_save_tags(struct page *page);
2727 void mte_save_page_tags(const void *page_addr, void *tag_storage);
28−bool mte_restore_tags(swp_entry_t entry, struct page *page);
28+void mte_restore_tags(swp_entry_t entry, struct page *page);
2929 void mte_restore_page_tags(void *page_addr, const void *tag_storage);
3030 void mte_invalidate_tags(int type, pgoff_t offset);
3131 void mte_invalidate_tags_area(int type);
@@ -36,6 +36,58 @@ void mte_free_tag_storage(char *storage); · 4 unmodified lines
3636
3737 /* track which pages have valid allocation tags */
3838 #define PG_mte_tagged PG_arch_2
39+/* simple lock to avoid multiple threads tagging the same page */
40+#define PG_mte_lock PG_arch_3
41+
42+static inline void set_page_mte_tagged(struct page *page)
43+{
44+ /*
45+ * Ensure that the tags written prior to this function are visible
46+ * before the page flags update.
47+ */
48+ smp_wmb();
49+ set_bit(PG_mte_tagged, &page->flags);
50+}
51+
52+static inline bool page_mte_tagged(struct page *page)
53+{
54+ bool ret = test_bit(PG_mte_tagged, &page->flags);
55+
56+ /*
57+ * If the page is tagged, ensure ordering with a likely subsequent
58+ * read of the tags.
59+ */
60+ if (ret)
61+ smp_rmb();
62+ return ret;
63+}
64+
65+/*
66+ * Lock the page for tagging and return 'true' if the page can be tagged,
67+ * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the
68+ * locking only happens once for page initialisation.
69+ *
70+ * The page MTE lock state:
71+ *
72+ * Locked: PG_mte_lock && !PG_mte_tagged
73+ * Unlocked: !PG_mte_lock || PG_mte_tagged
74+ *
75+ * Acquire semantics only if the page is tagged (returning 'false').
76+ */
77+static inline bool try_page_mte_tagging(struct page *page)
78+{
79+ if (!test_and_set_bit(PG_mte_lock, &page->flags))
80+ return true;
81+
82+ /*
83+ * The tags are either being initialised or may have been initialised
84+ * already. Check if the PG_mte_tagged flag has been set or wait
85+ * otherwise.
86+ */
87+ smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged));
88+
89+ return false;
90+}
3991
4092 void mte_zero_clear_page_tags(void *addr);
4193 void mte_sync_tags(pte_t old_pte, pte_t pte);
@@ -56,6 +108,17 @@ size_t mte_probe_user_range(const char __user *uaddr, size_t size); · 14 unmodified lines
56108 /* unused if !CONFIG_ARM64_MTE, silence the compiler */
57109 #define PG_mte_tagged 0
58110
111+static inline void set_page_mte_tagged(struct page *page)
112+{
113+}
114+static inline bool page_mte_tagged(struct page *page)
115+{
116+ return false;
117+}
118+static inline bool try_page_mte_tagging(struct page *page)
119+{
120+ return false;
121+}
59122 static inline void mte_zero_clear_page_tags(void *addr)
60123 {
61124 }
arch/arm64/include/asm/pgtable.h+2−2
@@ -1049,8 +1049,8 @@ static inline void arch_swap_invalidate_area(int type) · 1048 unmodified lines
10491049 #define __HAVE_ARCH_SWAP_RESTORE
10501050 static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
10511051 {
1052− if (system_supports_mte() && mte_restore_tags(entry, &folio->page))
1053− set_bit(PG_mte_tagged, &folio->flags);
1052+ if (system_supports_mte())
1053+ mte_restore_tags(entry, &folio->page);
10541054 }
10551055
10561056 #endif /* CONFIG_ARM64_MTE */
arch/arm64/include/asm/sysreg.h+0−138
@@ -165,31 +165,6 @@ · 164 unmodified lines
165165 #define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
166166 #define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
167167
168−#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
169−#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
170−#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
171−#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
172−#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
173−#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
174−#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
175−#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
176−#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
177−#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
178−#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
179−#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
180−
181−#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
182−#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
183−#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
184−#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
185−#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
186−#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
187−#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
188−
189−#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
190−#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
191−#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
192−
193168 #define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
194169 #define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
195170 #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
@@ -692,112 +667,6 @@ · 496 unmodified lines
692667 #define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
693668 #endif
694669
695−#define ID_DFR0_PERFMON_SHIFT 24
696−
697−#define ID_DFR0_PERFMON_8_0 0x3
698−#define ID_DFR0_PERFMON_8_1 0x4
699−#define ID_DFR0_PERFMON_8_4 0x5
700−#define ID_DFR0_PERFMON_8_5 0x6
701−
702−#define ID_ISAR4_SWP_FRAC_SHIFT 28
703−#define ID_ISAR4_PSR_M_SHIFT 24
704−#define ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT 20
705−#define ID_ISAR4_BARRIER_SHIFT 16
706−#define ID_ISAR4_SMC_SHIFT 12
707−#define ID_ISAR4_WRITEBACK_SHIFT 8
708−#define ID_ISAR4_WITHSHIFTS_SHIFT 4
709−#define ID_ISAR4_UNPRIV_SHIFT 0
710−
711−#define ID_DFR1_MTPMU_SHIFT 0
712−
713−#define ID_ISAR0_DIVIDE_SHIFT 24
714−#define ID_ISAR0_DEBUG_SHIFT 20
715−#define ID_ISAR0_COPROC_SHIFT 16
716−#define ID_ISAR0_CMPBRANCH_SHIFT 12
717−#define ID_ISAR0_BITFIELD_SHIFT 8
718−#define ID_ISAR0_BITCOUNT_SHIFT 4
719−#define ID_ISAR0_SWAP_SHIFT 0
720−
721−#define ID_ISAR5_RDM_SHIFT 24
722−#define ID_ISAR5_CRC32_SHIFT 16
723−#define ID_ISAR5_SHA2_SHIFT 12
724−#define ID_ISAR5_SHA1_SHIFT 8
725−#define ID_ISAR5_AES_SHIFT 4
726−#define ID_ISAR5_SEVL_SHIFT 0
727−
728−#define ID_ISAR6_I8MM_SHIFT 24
729−#define ID_ISAR6_BF16_SHIFT 20
730−#define ID_ISAR6_SPECRES_SHIFT 16
731−#define ID_ISAR6_SB_SHIFT 12
732−#define ID_ISAR6_FHM_SHIFT 8
733−#define ID_ISAR6_DP_SHIFT 4
734−#define ID_ISAR6_JSCVT_SHIFT 0
735−
736−#define ID_MMFR0_INNERSHR_SHIFT 28
737−#define ID_MMFR0_FCSE_SHIFT 24
738−#define ID_MMFR0_AUXREG_SHIFT 20
739−#define ID_MMFR0_TCM_SHIFT 16
740−#define ID_MMFR0_SHARELVL_SHIFT 12
741−#define ID_MMFR0_OUTERSHR_SHIFT 8
742−#define ID_MMFR0_PMSA_SHIFT 4
743−#define ID_MMFR0_VMSA_SHIFT 0
744−
745−#define ID_MMFR4_EVT_SHIFT 28
746−#define ID_MMFR4_CCIDX_SHIFT 24
747−#define ID_MMFR4_LSM_SHIFT 20
748−#define ID_MMFR4_HPDS_SHIFT 16
749−#define ID_MMFR4_CNP_SHIFT 12
750−#define ID_MMFR4_XNX_SHIFT 8
751−#define ID_MMFR4_AC2_SHIFT 4
752−#define ID_MMFR4_SPECSEI_SHIFT 0
753−
754−#define ID_MMFR5_ETS_SHIFT 0
755−
756−#define ID_PFR0_DIT_SHIFT 24
757−#define ID_PFR0_CSV2_SHIFT 16
758−#define ID_PFR0_STATE3_SHIFT 12
759−#define ID_PFR0_STATE2_SHIFT 8
760−#define ID_PFR0_STATE1_SHIFT 4
761−#define ID_PFR0_STATE0_SHIFT 0
762−
763−#define ID_DFR0_PERFMON_SHIFT 24
764−#define ID_DFR0_MPROFDBG_SHIFT 20
765−#define ID_DFR0_MMAPTRC_SHIFT 16
766−#define ID_DFR0_COPTRC_SHIFT 12
767−#define ID_DFR0_MMAPDBG_SHIFT 8
768−#define ID_DFR0_COPSDBG_SHIFT 4
769−#define ID_DFR0_COPDBG_SHIFT 0
770−
771−#define ID_PFR2_SSBS_SHIFT 4
772−#define ID_PFR2_CSV3_SHIFT 0
773−
774−#define MVFR0_FPROUND_SHIFT 28
775−#define MVFR0_FPSHVEC_SHIFT 24
776−#define MVFR0_FPSQRT_SHIFT 20
777−#define MVFR0_FPDIVIDE_SHIFT 16
778−#define MVFR0_FPTRAP_SHIFT 12
779−#define MVFR0_FPDP_SHIFT 8
780−#define MVFR0_FPSP_SHIFT 4
781−#define MVFR0_SIMD_SHIFT 0
782−
783−#define MVFR1_SIMDFMAC_SHIFT 28
784−#define MVFR1_FPHP_SHIFT 24
785−#define MVFR1_SIMDHP_SHIFT 20
786−#define MVFR1_SIMDSP_SHIFT 16
787−#define MVFR1_SIMDINT_SHIFT 12
788−#define MVFR1_SIMDLS_SHIFT 8
789−#define MVFR1_FPDNAN_SHIFT 4
790−#define MVFR1_FPFTZ_SHIFT 0
791−
792−#define ID_PFR1_GIC_SHIFT 28
793−#define ID_PFR1_VIRT_FRAC_SHIFT 24
794−#define ID_PFR1_SEC_FRAC_SHIFT 20
795−#define ID_PFR1_GENTIMER_SHIFT 16
796−#define ID_PFR1_VIRTUALIZATION_SHIFT 12
797−#define ID_PFR1_MPROGMOD_SHIFT 8
798−#define ID_PFR1_SECURITY_SHIFT 4
799−#define ID_PFR1_PROGMOD_SHIFT 0
800−
801670 #if defined(CONFIG_ARM64_4K_PAGES)
802671 #define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
803672 #define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
@@ -815,9 +684,6 @@ · 11 unmodified lines
815684 #define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
816685 #endif
817686
818−#define MVFR2_FPMISC_SHIFT 4
819−#define MVFR2_SIMDMISC_SHIFT 0
820−
821687 #define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */
822688 #define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */
823689
@@ -851,10 +717,6 @@ · 27 unmodified lines
851717 #define SYS_RGSR_EL1_SEED_SHIFT 8
852718 #define SYS_RGSR_EL1_SEED_MASK 0xffffUL
853719
854−/* GMID_EL1 field definitions */
855−#define GMID_EL1_BS_SHIFT 0
856−#define GMID_EL1_BS_SIZE 4
857−
858720 /* TFSR{,E0}_EL1 bit definitions */
859721 #define SYS_TFSR_EL1_TF0_SHIFT 0
860722 #define SYS_TFSR_EL1_TF1_SHIFT 1
arch/arm64/include/uapi/asm/kvm.h+1−0
@@ -43,6 +43,7 @@ · 42 unmodified lines
4343 #define __KVM_HAVE_VCPU_EVENTS
4444
4545 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
46+#define KVM_DIRTY_LOG_PAGE_OFFSET 64
4647
4748 #define KVM_REG_SIZE(id) \
4849 (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
arch/arm64/kernel/cpufeature.c+107−105
@@ -402,14 +402,14 @@ struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { · 401 unmodified lines
402402 };
403403
404404 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
405− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_INNERSHR_SHIFT, 4, 0xf),
406− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_FCSE_SHIFT, 4, 0),
407− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_AUXREG_SHIFT, 4, 0),
408− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_TCM_SHIFT, 4, 0),
409− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_SHARELVL_SHIFT, 4, 0),
410− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_OUTERSHR_SHIFT, 4, 0xf),
411− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_PMSA_SHIFT, 4, 0),
412− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_VMSA_SHIFT, 4, 0),
405+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
406+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
407+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
408+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
409+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
410+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
411+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
412+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
413413 ARM64_FTR_END,
414414 };
415415
@@ -429,32 +429,32 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { · 13 unmodified lines
429429 };
430430
431431 static const struct arm64_ftr_bits ftr_mvfr0[] = {
432− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPROUND_SHIFT, 4, 0),
433− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSHVEC_SHIFT, 4, 0),
434− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSQRT_SHIFT, 4, 0),
435− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDIVIDE_SHIFT, 4, 0),
436− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPTRAP_SHIFT, 4, 0),
437− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDP_SHIFT, 4, 0),
438− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSP_SHIFT, 4, 0),
439− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_SIMD_SHIFT, 4, 0),
432+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
433+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
434+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
435+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
436+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
437+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
438+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
439+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
440440 ARM64_FTR_END,
441441 };
442442
443443 static const struct arm64_ftr_bits ftr_mvfr1[] = {
444− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDFMAC_SHIFT, 4, 0),
445− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPHP_SHIFT, 4, 0),
446− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDHP_SHIFT, 4, 0),
447− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDSP_SHIFT, 4, 0),
448− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDINT_SHIFT, 4, 0),
449− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDLS_SHIFT, 4, 0),
450− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPDNAN_SHIFT, 4, 0),
451− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPFTZ_SHIFT, 4, 0),
444+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
445+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
446+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
447+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
448+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
449+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
450+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
451+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
452452 ARM64_FTR_END,
453453 };
454454
455455 static const struct arm64_ftr_bits ftr_mvfr2[] = {
456− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0),
457− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0),
456+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
457+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
458458 ARM64_FTR_END,
459459 };
460460
@@ -470,34 +470,34 @@ static const struct arm64_ftr_bits ftr_gmid[] = { · 9 unmodified lines
470470 };
471471
472472 static const struct arm64_ftr_bits ftr_id_isar0[] = {
473− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),
474− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),
475− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_COPROC_SHIFT, 4, 0),
476− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_CMPBRANCH_SHIFT, 4, 0),
477− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITFIELD_SHIFT, 4, 0),
478− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITCOUNT_SHIFT, 4, 0),
479− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_SWAP_SHIFT, 4, 0),
473+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
474+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
475+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
476+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
477+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
478+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
479+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
480480 ARM64_FTR_END,
481481 };
482482
483483 static const struct arm64_ftr_bits ftr_id_isar5[] = {
484− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
485− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
486− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
487− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
488− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
489− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
484+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
485+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
486+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
487+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
488+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
489+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
490490 ARM64_FTR_END,
491491 };
492492
493493 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
494− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EVT_SHIFT, 4, 0),
495− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CCIDX_SHIFT, 4, 0),
496− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_LSM_SHIFT, 4, 0),
497− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_HPDS_SHIFT, 4, 0),
498− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CNP_SHIFT, 4, 0),
499− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_XNX_SHIFT, 4, 0),
500− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_AC2_SHIFT, 4, 0),
494+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
495+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
496+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
497+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
498+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
499+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
500+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
501501
502502 /*
503503 * SpecSEI = 1 indicates that the PE might generate an SError on an
@@ -505,80 +505,80 @@ static const struct arm64_ftr_bits ftr_id_mmfr4[] = { · 1 unmodified line
505505 * SError might be generated than it will not be. Hence it has been
506506 * classified as FTR_HIGHER_SAFE.
507507 */
508− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_SPECSEI_SHIFT, 4, 0),
508+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
509509 ARM64_FTR_END,
510510 };
511511
512512 static const struct arm64_ftr_bits ftr_id_isar4[] = {
513− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SWP_FRAC_SHIFT, 4, 0),
514− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_PSR_M_SHIFT, 4, 0),
515− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT, 4, 0),
516− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_BARRIER_SHIFT, 4, 0),
517− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SMC_SHIFT, 4, 0),
518− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WRITEBACK_SHIFT, 4, 0),
519− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WITHSHIFTS_SHIFT, 4, 0),
520− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_UNPRIV_SHIFT, 4, 0),
513+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
514+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
515+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
516+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
517+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
518+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
519+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
520+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
521521 ARM64_FTR_END,
522522 };
523523
524524 static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
525− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_ETS_SHIFT, 4, 0),
525+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
526526 ARM64_FTR_END,
527527 };
528528
529529 static const struct arm64_ftr_bits ftr_id_isar6[] = {
530− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_I8MM_SHIFT, 4, 0),
531− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_BF16_SHIFT, 4, 0),
532− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SPECRES_SHIFT, 4, 0),
533− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SB_SHIFT, 4, 0),
534− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_FHM_SHIFT, 4, 0),
535− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_DP_SHIFT, 4, 0),
536− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_JSCVT_SHIFT, 4, 0),
530+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
531+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
532+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
533+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
534+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
535+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
536+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
537537 ARM64_FTR_END,
538538 };
539539
540540 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
541− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_DIT_SHIFT, 4, 0),
542− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_CSV2_SHIFT, 4, 0),
543− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE3_SHIFT, 4, 0),
544− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE2_SHIFT, 4, 0),
545− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE1_SHIFT, 4, 0),
546− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE0_SHIFT, 4, 0),
541+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
542+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
543+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
544+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
545+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
546+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
547547 ARM64_FTR_END,
548548 };
549549
550550 static const struct arm64_ftr_bits ftr_id_pfr1[] = {
551− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GIC_SHIFT, 4, 0),
552− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRT_FRAC_SHIFT, 4, 0),
553− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SEC_FRAC_SHIFT, 4, 0),
554− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GENTIMER_SHIFT, 4, 0),
555− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRTUALIZATION_SHIFT, 4, 0),
556− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_MPROGMOD_SHIFT, 4, 0),
557− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SECURITY_SHIFT, 4, 0),
558− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_PROGMOD_SHIFT, 4, 0),
551+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
552+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
553+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
554+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
555+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
556+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
557+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
558+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
559559 ARM64_FTR_END,
560560 };
561561
562562 static const struct arm64_ftr_bits ftr_id_pfr2[] = {
563− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
564− ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),
563+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
564+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
565565 ARM64_FTR_END,
566566 };
567567
568568 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
569569 /* [31:28] TraceFilt */
570− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_PERFMON_SHIFT, 4, 0),
571− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MPROFDBG_SHIFT, 4, 0),
572− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPTRC_SHIFT, 4, 0),
573− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPTRC_SHIFT, 4, 0),
574− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPDBG_SHIFT, 4, 0),
575− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPSDBG_SHIFT, 4, 0),
576− ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPDBG_SHIFT, 4, 0),
570+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
571+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
572+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
573+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
574+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
575+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
576+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
577577 ARM64_FTR_END,
578578 };
579579
580580 static const struct arm64_ftr_bits ftr_id_dfr1[] = {
581− S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_MTPMU_SHIFT, 4, 0),
581+ S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
582582 ARM64_FTR_END,
583583 };
584584
@@ -1119,12 +1119,12 @@ static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, · 534 unmodified lines
11191119 * EL1-dependent register fields to avoid spurious sanity check fails.
11201120 */
11211121 if (!id_aa64pfr0_32bit_el1(pfr0)) {
1122− relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_SMC_SHIFT);
1123− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);
1124− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);
1125− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);
1126− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SECURITY_SHIFT);
1127− relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_PROGMOD_SHIFT);
1122+ relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
1123+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
1124+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
1125+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
1126+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
1127+ relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
11281128 }
11291129
11301130 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
@@ -2074,8 +2074,10 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) · 943 unmodified lines
20742074 * Clear the tags in the zero page. This needs to be done via the
20752075 * linear map which has the Tagged attribute.
20762076 */
2077− if (!test_and_set_bit(PG_mte_tagged, &ZERO_PAGE(0)->flags))
2077+ if (try_page_mte_tagging(ZERO_PAGE(0))) {
20782078 mte_clear_page_tags(lm_alias(empty_zero_page));
2079+ set_page_mte_tagged(ZERO_PAGE(0));
2080+ }
20792081
20802082 kasan_init_hw_tags_cpu();
20812083 }
@@ -2829,24 +2831,24 @@ static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope) · 747 unmodified lines
28292831 else
28302832 mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
28312833
2832− return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDSP_SHIFT) &&
2833− cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDINT_SHIFT) &&
2834− cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDLS_SHIFT);
2834+ return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
2835+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
2836+ cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
28352837 }
28362838 #endif
28372839
28382840 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
28392841 #ifdef CONFIG_COMPAT
28402842 HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
2841− HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
2843+ HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_EL1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
28422844 /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
2843− HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
2844− HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
2845− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
2846− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
2847− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
2848− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
2849− HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
2845+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
2846+ HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
2847+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
2848+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
2849+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
2850+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
2851+ HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
28502852 #endif
28512853 {},
28522854 };
arch/arm64/kernel/elfcore.c+1−1
@@ -47,7 +47,7 @@ static int mte_dump_tag_range(struct coredump_params *cprm, · 46 unmodified lines
4747 * Pages mapped in user space as !pte_access_permitted() (e.g.
4848 * PROT_EXEC only) may not have the PG_mte_tagged flag set.
4949 */
50− if (!test_bit(PG_mte_tagged, &page->flags)) {
50+ if (!page_mte_tagged(page)) {
5151 put_page(page);
5252 dump_skip(cprm, MTE_PAGE_TAG_STORAGE);
5353 continue;
arch/arm64/kernel/hibernate.c+1−1
@@ -271,7 +271,7 @@ static int swsusp_mte_save_tags(void) · 270 unmodified lines
271271 if (!page)
272272 continue;
273273
274− if (!test_bit(PG_mte_tagged, &page->flags))
274+ if (!page_mte_tagged(page))
275275 continue;
276276
277277 ret = save_tags(page, pfn);
arch/arm64/kernel/image-vars.h+0−15
@@ -71,12 +71,6 @@ KVM_NVHE_ALIAS(nvhe_hyp_panic_handler); · 70 unmodified lines
7171 /* Vectors installed by hyp-init on reset HVC. */
7272 KVM_NVHE_ALIAS(__hyp_stub_vectors);
7373
74−/* Kernel symbol used by icache_is_vpipt(). */
75−KVM_NVHE_ALIAS(__icache_flags);
76−
77−/* VMID bits set by the KVM VMID allocator */
78−KVM_NVHE_ALIAS(kvm_arm_vmid_bits);
79−
8074 /* Static keys which are set if a vGIC trap should be handled in hyp. */
8175 KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
8276 KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
@@ -92,9 +86,6 @@ KVM_NVHE_ALIAS(gic_nonsecure_priorities); · 9 unmodified lines
9286 KVM_NVHE_ALIAS(__start___kvm_ex_table);
9387 KVM_NVHE_ALIAS(__stop___kvm_ex_table);
9488
95−/* Array containing bases of nVHE per-CPU memory regions. */
96−KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base);
97−
9889 /* PMU available static key */
9990 #ifdef CONFIG_HW_PERF_EVENTS
10091 KVM_NVHE_ALIAS(kvm_arm_pmu_available);
@@ -111,12 +102,6 @@ KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy); · 10 unmodified lines
111102 KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);
112103 #endif
113104
114−/* Kernel memory sections */
115−KVM_NVHE_ALIAS(__start_rodata);
116−KVM_NVHE_ALIAS(__end_rodata);
117−KVM_NVHE_ALIAS(__bss_start);
118−KVM_NVHE_ALIAS(__bss_stop);
119−
120105 /* Hyp memory sections */
121106 KVM_NVHE_ALIAS(__hyp_idmap_text_start);
122107 KVM_NVHE_ALIAS(__hyp_idmap_text_end);
arch/arm64/kernel/mte.c+10−11
@@ -41,19 +41,17 @@ static void mte_sync_page_tags(struct page *page, pte_t old_pte, · 40 unmodified lines
4141 if (check_swap && is_swap_pte(old_pte)) {
4242 swp_entry_t entry = pte_to_swp_entry(old_pte);
4343
44− if (!non_swap_entry(entry) && mte_restore_tags(entry, page))
45− return;
44+ if (!non_swap_entry(entry))
45+ mte_restore_tags(entry, page);
4646 }
4747
4848 if (!pte_is_tagged)
4949 return;
5050
51− /*
52− * Test PG_mte_tagged again in case it was racing with another
53− * set_pte_at().
54− */
55− if (!test_and_set_bit(PG_mte_tagged, &page->flags))
51+ if (try_page_mte_tagging(page)) {
5652 mte_clear_page_tags(page_address(page));
53+ set_page_mte_tagged(page);
54+ }
5755 }
5856
5957 void mte_sync_tags(pte_t old_pte, pte_t pte)
@@ -69,9 +67,11 @@ void mte_sync_tags(pte_t old_pte, pte_t pte) · 9 unmodified lines
6967
7068 /* if PG_mte_tagged is set, tags have already been initialised */
7169 for (i = 0; i < nr_pages; i++, page++) {
72− if (!test_bit(PG_mte_tagged, &page->flags))
70+ if (!page_mte_tagged(page)) {
7371 mte_sync_page_tags(page, old_pte, check_swap,
7472 pte_is_tagged);
73+ set_page_mte_tagged(page);
74+ }
7575 }
7676
7777 /* ensure the tags are visible before the PTE is set */
@@ -96,8 +96,7 @@ int memcmp_pages(struct page *page1, struct page *page2) · 18 unmodified lines
9696 * pages is tagged, set_pte_at() may zero or change the tags of the
9797 * other page via mte_sync_tags().
9898 */
99− if (test_bit(PG_mte_tagged, &page1->flags) ||
100− test_bit(PG_mte_tagged, &page2->flags))
99+ if (page_mte_tagged(page1) || page_mte_tagged(page2))
101100 return addr1 != addr2;
102101
103102 return ret;
@@ -454,7 +453,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, · 350 unmodified lines
454453 put_page(page);
455454 break;
456455 }
457− WARN_ON_ONCE(!test_bit(PG_mte_tagged, &page->flags));
456+ WARN_ON_ONCE(!page_mte_tagged(page));
458457
459458 /* limit access to the end of the page */
460459 offset = offset_in_page(addr);
arch/arm64/kvm/Kconfig+2−0
@@ -32,6 +32,8 @@ menuconfig KVM · 31 unmodified lines
3232 select KVM_VFIO
3333 select HAVE_KVM_EVENTFD
3434 select HAVE_KVM_IRQFD
35+ select HAVE_KVM_DIRTY_RING_ACQ_REL
36+ select NEED_KVM_DIRTY_RING_WITH_BITMAP
3537 select HAVE_KVM_MSI
3638 select HAVE_KVM_IRQCHIP
3739 select HAVE_KVM_IRQ_ROUTING
arch/arm64/kvm/arm.c+51−39
@@ -37,6 +37,7 @@ · 36 unmodified lines
3737 #include <asm/kvm_arm.h>
3838 #include <asm/kvm_asm.h>
3939 #include <asm/kvm_mmu.h>
40+#include <asm/kvm_pkvm.h>
4041 #include <asm/kvm_emulate.h>
4142 #include <asm/sections.h>
4243
@@ -50,7 +51,6 @@ DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); · 7 unmodified lines
5051 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
5152
5253 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
53−unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
5454 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
5555
5656 static bool vgic_present;
@@ -138,24 +138,24 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 81 unmodified lines
138138 {
139139 int ret;
140140
141− ret = kvm_arm_setup_stage2(kvm, type);
142− if (ret)
143− return ret;
144−
145− ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu);
141+ ret = kvm_share_hyp(kvm, kvm + 1);
146142 if (ret)
147143 return ret;
148144
149− ret = kvm_share_hyp(kvm, kvm + 1);
145+ ret = pkvm_init_host_vm(kvm);
150146 if (ret)
151− goto out_free_stage2_pgd;
147+ goto err_unshare_kvm;
152148
153149 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL)) {
154150 ret = -ENOMEM;
155− goto out_free_stage2_pgd;
151+ goto err_unshare_kvm;
156152 }
157153 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
158154
155+ ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
156+ if (ret)
157+ goto err_free_cpumask;
158+
159159 kvm_vgic_early_init(kvm);
160160
161161 /* The maximum number of VCPUs is limited by the host's GIC model */
@@ -164,9 +164,18 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 2 unmodified lines
164164 set_default_spectre(kvm);
165165 kvm_arm_init_hypercalls(kvm);
166166
167− return ret;
168−out_free_stage2_pgd:
169− kvm_free_stage2_pgd(&kvm->arch.mmu);
167+ /*
168+ * Initialise the default PMUver before there is a chance to
169+ * create an actual PMU.
170+ */
171+ kvm->arch.dfr0_pmuver.imp = kvm_arm_pmu_get_pmuver_limit();
172+
173+ return 0;
174+
175+err_free_cpumask:
176+ free_cpumask_var(kvm->arch.supported_cpus);
177+err_unshare_kvm:
178+ kvm_unshare_hyp(kvm, kvm + 1);
170179 return ret;
171180 }
172181
@@ -187,6 +196,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 14 unmodified lines
187196
188197 kvm_vgic_destroy(kvm);
189198
199+ if (is_protected_kvm_enabled())
200+ pkvm_destroy_hyp_vm(kvm);
201+
190202 kvm_destroy_vcpus(kvm);
191203
192204 kvm_unshare_hyp(kvm, kvm + 1);
@@ -569,6 +581,12 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 376 unmodified lines
569581 if (ret)
570582 return ret;
571583
584+ if (is_protected_kvm_enabled()) {
585+ ret = pkvm_create_hyp_vm(kvm);
586+ if (ret)
587+ return ret;
588+ }
589+
572590 if (!irqchip_in_kernel(kvm)) {
573591 /*
574592 * Tell the rest of the code that there are userspace irqchip
@@ -746,6 +764,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) · 171 unmodified lines
746764
747765 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
748766 return kvm_vcpu_suspend(vcpu);
767+
768+ if (kvm_dirty_ring_check_request(vcpu))
769+ return 0;
749770 }
750771
751772 return 1;
@@ -1518,7 +1539,7 @@ static int kvm_init_vector_slots(void) · 766 unmodified lines
15181539 return 0;
15191540 }
15201541
1521−static void cpu_prepare_hyp_mode(int cpu)
1542+static void cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
15221543 {
15231544 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
15241545 unsigned long tcr;
@@ -1534,23 +1555,9 @@ static void cpu_prepare_hyp_mode(int cpu) · 9 unmodified lines
15341555
15351556 params->mair_el2 = read_sysreg(mair_el1);
15361557
1537− /*
1538− * The ID map may be configured to use an extended virtual address
1539− * range. This is only the case if system RAM is out of range for the
1540− * currently configured page size and VA_BITS, in which case we will
1541− * also need the extended virtual range for the HYP ID map, or we won't
1542− * be able to enable the EL2 MMU.
1543− *
1544− * However, at EL2, there is only one TTBR register, and we can't switch
1545− * between translation tables *and* update TCR_EL2.T0SZ at the same
1546− * time. Bottom line: we need to use the extended range with *both* our
1547− * translation tables.
1548− *
1549− * So use the same T0SZ value we use for the ID map.
1550− */
15511558 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
15521559 tcr &= ~TCR_T0SZ_MASK;
1553− tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
1560+ tcr |= TCR_T0SZ(hyp_va_bits);
15541561 params->tcr_el2 = tcr;
15551562
15561563 params->pgd_pa = kvm_mmu_get_httbr();
@@ -1844,13 +1851,13 @@ static void teardown_hyp_mode(void) · 287 unmodified lines
18441851 free_hyp_pgds();
18451852 for_each_possible_cpu(cpu) {
18461853 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1847− free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order());
1854+ free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
18481855 }
18491856 }
18501857
18511858 static int do_pkvm_init(u32 hyp_va_bits)
18521859 {
1853− void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);
1860+ void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
18541861 int ret;
18551862
18561863 preempt_disable();
@@ -1870,11 +1877,8 @@ static int do_pkvm_init(u32 hyp_va_bits) · 13 unmodified lines
18701877 return ret;
18711878 }
18721879
1873−static int kvm_hyp_init_protection(u32 hyp_va_bits)
1880+static void kvm_hyp_init_symbols(void)
18741881 {
1875− void *addr = phys_to_virt(hyp_mem_base);
1876− int ret;
1877−
18781882 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
18791883 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
18801884 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
@@ -1883,6 +1887,14 @@ static int kvm_hyp_init_protection(u32 hyp_va_bits) · 2 unmodified lines
18831887 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
18841888 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
18851889 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
1890+ kvm_nvhe_sym(__icache_flags) = __icache_flags;
1891+ kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
1892+}
1893+
1894+static int kvm_hyp_init_protection(u32 hyp_va_bits)
1895+{
1896+ void *addr = phys_to_virt(hyp_mem_base);
1897+ int ret;
18861898
18871899 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
18881900 if (ret)
@@ -1950,7 +1962,7 @@ static int init_hyp_mode(void) · 61 unmodified lines
19501962
19511963 page_addr = page_address(page);
19521964 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
1953− kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr;
1965+ kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
19541966 }
19551967
19561968 /*
@@ -2043,7 +2055,7 @@ static int init_hyp_mode(void) · 86 unmodified lines
20432055 }
20442056
20452057 for_each_possible_cpu(cpu) {
2046− char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];
2058+ char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
20472059 char *percpu_end = percpu_begin + nvhe_percpu_size();
20482060
20492061 /* Map Hyp percpu pages */
@@ -2054,9 +2066,11 @@ static int init_hyp_mode(void) · 4 unmodified lines
20542066 }
20552067
20562068 /* Prepare the CPU initialization parameters */
2057− cpu_prepare_hyp_mode(cpu);
2069+ cpu_prepare_hyp_mode(cpu, hyp_va_bits);
20582070 }
20592071
2072+ kvm_hyp_init_symbols();
2073+
20602074 if (is_protected_kvm_enabled()) {
20612075 init_cpu_logical_map();
20622076
@@ -2064,9 +2078,7 @@ static int init_hyp_mode(void) · 1 unmodified line
20642078 err = -ENODEV;
20652079 goto out_err;
20662080 }
2067− }
20682081
2069− if (is_protected_kvm_enabled()) {
20702082 err = kvm_hyp_init_protection(hyp_va_bits);
20712083 if (err) {
20722084 kvm_err("Failed to init hyp memory protection\n");
arch/arm64/kvm/guest.c+11−7
@@ -1059,7 +1059,7 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 1058 unmodified lines
10591059 maddr = page_address(page);
10601060
10611061 if (!write) {
1062− if (test_bit(PG_mte_tagged, &page->flags))
1062+ if (page_mte_tagged(page))
10631063 num_tags = mte_copy_tags_to_user(tags, maddr,
10641064 MTE_GRANULES_PER_PAGE);
10651065 else
@@ -1068,15 +1068,19 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 2 unmodified lines
10681068 clear_user(tags, MTE_GRANULES_PER_PAGE);
10691069 kvm_release_pfn_clean(pfn);
10701070 } else {
1071+ /*
1072+ * Only locking to serialise with a concurrent
1073+ * set_pte_at() in the VMM but still overriding the
1074+ * tags, hence ignoring the return value.
1075+ */
1076+ try_page_mte_tagging(page);
10711077 num_tags = mte_copy_tags_from_user(maddr, tags,
10721078 MTE_GRANULES_PER_PAGE);
10731079
1074− /*
1075− * Set the flag after checking the write
1076− * completed fully
1077− */
1078− if (num_tags == MTE_GRANULES_PER_PAGE)
1079− set_bit(PG_mte_tagged, &page->flags);
1080+ /* uaccess failed, don't leave stale tags */
1081+ if (num_tags != MTE_GRANULES_PER_PAGE)
1082+ mte_clear_page_tags(page);
1083+ set_page_mte_tagged(page);
10801084
10811085 kvm_release_pfn_dirty(pfn);
10821086 }
arch/arm64/kvm/hyp/hyp-constants.c+3−0
@@ -2,9 +2,12 @@ · 1 unmodified line
22
33 #include <linux/kbuild.h>
44 #include <nvhe/memory.h>
5+#include <nvhe/pkvm.h>
56
67 int main(void)
78 {
89 DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));
10+ DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm));
11+ DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu));
912 return 0;
1013 }
arch/arm64/kvm/hyp/include/nvhe/mem_protect.h+21−4
@@ -8,8 +8,10 @@ · 7 unmodified lines
88 #define __KVM_NVHE_MEM_PROTECT__
99 #include <linux/kvm_host.h>
1010 #include <asm/kvm_hyp.h>
11+#include <asm/kvm_mmu.h>
1112 #include <asm/kvm_pgtable.h>
1213 #include <asm/virt.h>
14+#include <nvhe/pkvm.h>
1315 #include <nvhe/spinlock.h>
1416
1517 /*
@@ -43,30 +45,45 @@ static inline enum pkvm_page_state pkvm_getstate(enum kvm_pgtable_prot prot) · 27 unmodified lines
4345 return prot & PKVM_PAGE_STATE_PROT_MASK;
4446 }
4547
46−struct host_kvm {
48+struct host_mmu {
4749 struct kvm_arch arch;
4850 struct kvm_pgtable pgt;
4951 struct kvm_pgtable_mm_ops mm_ops;
5052 hyp_spinlock_t lock;
5153 };
52−extern struct host_kvm host_kvm;
54+extern struct host_mmu host_mmu;
5355
54−extern const u8 pkvm_hyp_id;
56+/* This corresponds to page-table locking order */
57+enum pkvm_component_id {
58+ PKVM_ID_HOST,
59+ PKVM_ID_HYP,
60+};
61+
62+extern unsigned long hyp_nr_cpus;
5563
5664 int __pkvm_prot_finalize(void);
5765 int __pkvm_host_share_hyp(u64 pfn);
5866 int __pkvm_host_unshare_hyp(u64 pfn);
67+int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages);
68+int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages);
5969
6070 bool addr_is_memory(phys_addr_t phys);
6171 int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);
6272 int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id);
6373 int kvm_host_prepare_stage2(void *pgt_pool_base);
74+int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd);
6475 void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt);
6576
77+int hyp_pin_shared_mem(void *from, void *to);
78+void hyp_unpin_shared_mem(void *from, void *to);
79+void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc);
80+int refill_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages,
81+ struct kvm_hyp_memcache *host_mc);
82+
6683 static __always_inline void __load_host_stage2(void)
6784 {
6885 if (static_branch_likely(&kvm_protected_mode_initialized))
69− __load_stage2(&host_kvm.arch.mmu, &host_kvm.arch);
86+ __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
7087 else
7188 write_sysreg(0, vttbr_el2);
7289 }
arch/arm64/kvm/hyp/include/nvhe/memory.h+27−0
@@ -38,6 +38,10 @@ static inline phys_addr_t hyp_virt_to_phys(void *addr) · 37 unmodified lines
3838 #define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))
3939 #define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)
4040
41+/*
42+ * Refcounting for 'struct hyp_page'.
43+ * hyp_pool::lock must be held if atomic access to the refcount is required.
44+ */
4145 static inline int hyp_page_count(void *addr)
4246 {
4347 struct hyp_page *p = hyp_virt_to_page(addr);
@@ -45,4 +49,27 @@ static inline int hyp_page_count(void *addr) · 1 unmodified line
4549 return p->refcount;
4650 }
4751
52+static inline void hyp_page_ref_inc(struct hyp_page *p)
53+{
54+ BUG_ON(p->refcount == USHRT_MAX);
55+ p->refcount++;
56+}
57+
58+static inline void hyp_page_ref_dec(struct hyp_page *p)
59+{
60+ BUG_ON(!p->refcount);
61+ p->refcount--;
62+}
63+
64+static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
65+{
66+ hyp_page_ref_dec(p);
67+ return (p->refcount == 0);
68+}
69+
70+static inline void hyp_set_page_refcounted(struct hyp_page *p)
71+{
72+ BUG_ON(p->refcount);
73+ p->refcount = 1;
74+}
4875 #endif /* __KVM_HYP_MEMORY_H */
arch/arm64/kvm/hyp/include/nvhe/mm.h+5−13
@@ -13,9 +13,13 @@ · 12 unmodified lines
1313 extern struct kvm_pgtable pkvm_pgtable;
1414 extern hyp_spinlock_t pkvm_pgd_lock;
1515
16+int hyp_create_pcpu_fixmap(void);
17+void *hyp_fixmap_map(phys_addr_t phys);
18+void hyp_fixmap_unmap(void);
19+
1620 int hyp_create_idmap(u32 hyp_va_bits);
1721 int hyp_map_vectors(void);
18−int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back);
22+int hyp_back_vmemmap(phys_addr_t back);
1923 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
2024 int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
2125 int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);
@@ -24,16 +28,4 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, · 2 unmodified lines
2428 unsigned long *haddr);
2529 int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);
2630
27−static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size,
28− unsigned long *start, unsigned long *end)
29−{
30− unsigned long nr_pages = size >> PAGE_SHIFT;
31− struct hyp_page *p = hyp_phys_to_page(phys);
32−
33− *start = (unsigned long)p;
34− *end = *start + nr_pages * sizeof(struct hyp_page);
35− *start = ALIGN_DOWN(*start, PAGE_SIZE);
36− *end = ALIGN(*end, PAGE_SIZE);
37−}
38−
3931 #endif /* __KVM_HYP_MM_H */
arch/arm64/kvm/hyp/include/nvhe/pkvm.h+68−0
@@ -0,0 +1,68 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2021 Google LLC
4+ * Author: Fuad Tabba <tabba@google.com>
5+ */
6+
7+#ifndef __ARM64_KVM_NVHE_PKVM_H__
8+#define __ARM64_KVM_NVHE_PKVM_H__
9+
10+#include <asm/kvm_pkvm.h>
11+
12+#include <nvhe/gfp.h>
13+#include <nvhe/spinlock.h>
14+
15+/*
16+ * Holds the relevant data for maintaining the vcpu state completely at hyp.
17+ */
18+struct pkvm_hyp_vcpu {
19+ struct kvm_vcpu vcpu;
20+
21+ /* Backpointer to the host's (untrusted) vCPU instance. */
22+ struct kvm_vcpu *host_vcpu;
23+};
24+
25+/*
26+ * Holds the relevant data for running a protected vm.
27+ */
28+struct pkvm_hyp_vm {
29+ struct kvm kvm;
30+
31+ /* Backpointer to the host's (untrusted) KVM instance. */
32+ struct kvm *host_kvm;
33+
34+ /* The guest's stage-2 page-table managed by the hypervisor. */
35+ struct kvm_pgtable pgt;
36+ struct kvm_pgtable_mm_ops mm_ops;
37+ struct hyp_pool pool;
38+ hyp_spinlock_t lock;
39+
40+ /*
41+ * The number of vcpus initialized and ready to run.
42+ * Modifying this is protected by 'vm_table_lock'.
43+ */
44+ unsigned int nr_vcpus;
45+
46+ /* Array of the hyp vCPU structures for this VM. */
47+ struct pkvm_hyp_vcpu *vcpus[];
48+};
49+
50+static inline struct pkvm_hyp_vm *
51+pkvm_hyp_vcpu_to_hyp_vm(struct pkvm_hyp_vcpu *hyp_vcpu)
52+{
53+ return container_of(hyp_vcpu->vcpu.kvm, struct pkvm_hyp_vm, kvm);
54+}
55+
56+void pkvm_hyp_vm_table_init(void *tbl);
57+
58+int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
59+ unsigned long pgd_hva);
60+int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
61+ unsigned long vcpu_hva);
62+int __pkvm_teardown_vm(pkvm_handle_t handle);
63+
64+struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
65+ unsigned int vcpu_idx);
66+void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu);
67+
68+#endif /* __ARM64_KVM_NVHE_PKVM_H__ */
arch/arm64/kvm/hyp/include/nvhe/spinlock.h+9−1
@@ -28,9 +28,17 @@ typedef union hyp_spinlock { · 27 unmodified lines
2828 };
2929 } hyp_spinlock_t;
3030
31+#define __HYP_SPIN_LOCK_INITIALIZER \
32+ { .__val = 0 }
33+
34+#define __HYP_SPIN_LOCK_UNLOCKED \
35+ ((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER)
36+
37+#define DEFINE_HYP_SPINLOCK(x) hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED
38+
3139 #define hyp_spin_lock_init(l) \
3240 do { \
33− *(l) = (hyp_spinlock_t){ .__val = 0 }; \
41+ *(l) = __HYP_SPIN_LOCK_UNLOCKED; \
3442 } while (0)
3543
3644 static inline void hyp_spin_lock(hyp_spinlock_t *lock)
arch/arm64/kvm/hyp/nvhe/cache.S+11−0
@@ -12,3 +12,14 @@ SYM_FUNC_START(__pi_dcache_clean_inval_poc) · 11 unmodified lines
1212 ret
1313 SYM_FUNC_END(__pi_dcache_clean_inval_poc)
1414 SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)
15+
16+SYM_FUNC_START(__pi_icache_inval_pou)
17+alternative_if ARM64_HAS_CACHE_DIC
18+ isb
19+ ret
20+alternative_else_nop_endif
21+
22+ invalidate_icache_by_line x0, x1, x2, x3
23+ ret
24+SYM_FUNC_END(__pi_icache_inval_pou)
25+SYM_FUNC_ALIAS(icache_inval_pou, __pi_icache_inval_pou)
arch/arm64/kvm/hyp/nvhe/hyp-main.c+108−2
@@ -15,17 +15,93 @@ · 14 unmodified lines
1515
1616 #include <nvhe/mem_protect.h>
1717 #include <nvhe/mm.h>
18+#include <nvhe/pkvm.h>
1819 #include <nvhe/trap_handler.h>
1920
2021 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
2122
2223 void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
2324
25+static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
26+{
27+ struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
28+
29+ hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
30+
31+ hyp_vcpu->vcpu.arch.sve_state = kern_hyp_va(host_vcpu->arch.sve_state);
32+ hyp_vcpu->vcpu.arch.sve_max_vl = host_vcpu->arch.sve_max_vl;
33+
34+ hyp_vcpu->vcpu.arch.hw_mmu = host_vcpu->arch.hw_mmu;
35+
36+ hyp_vcpu->vcpu.arch.hcr_el2 = host_vcpu->arch.hcr_el2;
37+ hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2;
38+ hyp_vcpu->vcpu.arch.cptr_el2 = host_vcpu->arch.cptr_el2;
39+
40+ hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags;
41+ hyp_vcpu->vcpu.arch.fp_state = host_vcpu->arch.fp_state;
42+
43+ hyp_vcpu->vcpu.arch.debug_ptr = kern_hyp_va(host_vcpu->arch.debug_ptr);
44+ hyp_vcpu->vcpu.arch.host_fpsimd_state = host_vcpu->arch.host_fpsimd_state;
45+
46+ hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2;
47+
48+ hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3;
49+}
50+
51+static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
52+{
53+ struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
54+ struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
55+ struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
56+ unsigned int i;
57+
58+ host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
59+
60+ host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2;
61+ host_vcpu->arch.cptr_el2 = hyp_vcpu->vcpu.arch.cptr_el2;
62+
63+ host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault;
64+
65+ host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags;
66+ host_vcpu->arch.fp_state = hyp_vcpu->vcpu.arch.fp_state;
67+
68+ host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr;
69+ for (i = 0; i < hyp_cpu_if->used_lrs; ++i)
70+ host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
71+}
72+
2473 static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
2574 {
26− DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
75+ DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1);
76+ int ret;
2777
28− cpu_reg(host_ctxt, 1) = __kvm_vcpu_run(kern_hyp_va(vcpu));
78+ host_vcpu = kern_hyp_va(host_vcpu);
79+
80+ if (unlikely(is_protected_kvm_enabled())) {
81+ struct pkvm_hyp_vcpu *hyp_vcpu;
82+ struct kvm *host_kvm;
83+
84+ host_kvm = kern_hyp_va(host_vcpu->kvm);
85+ hyp_vcpu = pkvm_load_hyp_vcpu(host_kvm->arch.pkvm.handle,
86+ host_vcpu->vcpu_idx);
87+ if (!hyp_vcpu) {
88+ ret = -EINVAL;
89+ goto out;
90+ }
91+
92+ flush_hyp_vcpu(hyp_vcpu);
93+
94+ ret = __kvm_vcpu_run(&hyp_vcpu->vcpu);
95+
96+ sync_hyp_vcpu(hyp_vcpu);
97+ pkvm_put_hyp_vcpu(hyp_vcpu);
98+ } else {
99+ /* The host is fully trusted, run its vCPU directly. */
100+ ret = __kvm_vcpu_run(host_vcpu);
101+ }
102+
103+out:
104+ cpu_reg(host_ctxt, 1) = ret;
29105 }
30106
31107 static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)
@@ -191,6 +267,33 @@ static void handle___pkvm_vcpu_init_traps(struct kvm_cpu_context *host_ctxt) · 159 unmodified lines
191267 __pkvm_vcpu_init_traps(kern_hyp_va(vcpu));
192268 }
193269
270+static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt)
271+{
272+ DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1);
273+ DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2);
274+ DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3);
275+
276+ host_kvm = kern_hyp_va(host_kvm);
277+ cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva);
278+}
279+
280+static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt)
281+{
282+ DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
283+ DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2);
284+ DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3);
285+
286+ host_vcpu = kern_hyp_va(host_vcpu);
287+ cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva);
288+}
289+
290+static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt)
291+{
292+ DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
293+
294+ cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle);
295+}
296+
194297 typedef void (*hcall_t)(struct kvm_cpu_context *);
195298
196299 #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -220,6 +323,9 @@ static const hcall_t host_hcall[] = { · 23 unmodified lines
220323 HANDLE_FUNC(__vgic_v3_save_aprs),
221324 HANDLE_FUNC(__vgic_v3_restore_aprs),
222325 HANDLE_FUNC(__pkvm_vcpu_init_traps),
326+ HANDLE_FUNC(__pkvm_init_vm),
327+ HANDLE_FUNC(__pkvm_init_vcpu),
328+ HANDLE_FUNC(__pkvm_teardown_vm),
223329 };
224330
225331 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
arch/arm64/kvm/hyp/nvhe/hyp-smp.c+2−0
@@ -23,6 +23,8 @@ u64 cpu_logical_map(unsigned int cpu) · 22 unmodified lines
2323 return hyp_cpu_logical_map[cpu];
2424 }
2525
26+unsigned long __ro_after_init kvm_arm_hyp_percpu_base[NR_CPUS];
27+
2628 unsigned long __hyp_per_cpu_offset(unsigned int cpu)
2729 {
2830 unsigned long *cpu_base_array;
arch/arm64/kvm/hyp/nvhe/mem_protect.c+476−45
@@ -21,21 +21,33 @@ · 20 unmodified lines
2121
2222 #define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP)
2323
24−extern unsigned long hyp_nr_cpus;
25−struct host_kvm host_kvm;
24+struct host_mmu host_mmu;
2625
2726 static struct hyp_pool host_s2_pool;
2827
29−const u8 pkvm_hyp_id = 1;
28+static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm);
29+#define current_vm (*this_cpu_ptr(&__current_vm))
30+
31+static void guest_lock_component(struct pkvm_hyp_vm *vm)
32+{
33+ hyp_spin_lock(&vm->lock);
34+ current_vm = vm;
35+}
36+
37+static void guest_unlock_component(struct pkvm_hyp_vm *vm)
38+{
39+ current_vm = NULL;
40+ hyp_spin_unlock(&vm->lock);
41+}
3042
3143 static void host_lock_component(void)
3244 {
33− hyp_spin_lock(&host_kvm.lock);
45+ hyp_spin_lock(&host_mmu.lock);
3446 }
3547
3648 static void host_unlock_component(void)
3749 {
38− hyp_spin_unlock(&host_kvm.lock);
50+ hyp_spin_unlock(&host_mmu.lock);
3951 }
4052
4153 static void hyp_lock_component(void)
@@ -79,6 +91,11 @@ static void host_s2_put_page(void *addr) · 37 unmodified lines
7991 hyp_put_page(&host_s2_pool, addr);
8092 }
8193
94+static void host_s2_free_removed_table(void *addr, u32 level)
95+{
96+ kvm_pgtable_stage2_free_removed(&host_mmu.mm_ops, addr, level);
97+}
98+
8299 static int prepare_s2_pool(void *pgt_pool_base)
83100 {
84101 unsigned long nr_pages, pfn;
@@ -90,9 +107,10 @@ static int prepare_s2_pool(void *pgt_pool_base) · 5 unmodified lines
90107 if (ret)
91108 return ret;
92109
93− host_kvm.mm_ops = (struct kvm_pgtable_mm_ops) {
110+ host_mmu.mm_ops = (struct kvm_pgtable_mm_ops) {
94111 .zalloc_pages_exact = host_s2_zalloc_pages_exact,
95112 .zalloc_page = host_s2_zalloc_page,
113+ .free_removed_table = host_s2_free_removed_table,
96114 .phys_to_virt = hyp_phys_to_virt,
97115 .virt_to_phys = hyp_virt_to_phys,
98116 .page_count = hyp_page_count,
@@ -111,7 +129,7 @@ static void prepare_host_vtcr(void) · 12 unmodified lines
111129 parange = kvm_get_parange(id_aa64mmfr0_el1_sys_val);
112130 phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange);
113131
114− host_kvm.arch.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val,
132+ host_mmu.arch.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val,
115133 id_aa64mmfr1_el1_sys_val, phys_shift);
116134 }
117135
@@ -119,45 +137,170 @@ static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot pr · 1 unmodified line
119137
120138 int kvm_host_prepare_stage2(void *pgt_pool_base)
121139 {
122− struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;
140+ struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
123141 int ret;
124142
125143 prepare_host_vtcr();
126− hyp_spin_lock_init(&host_kvm.lock);
127− mmu->arch = &host_kvm.arch;
144+ hyp_spin_lock_init(&host_mmu.lock);
145+ mmu->arch = &host_mmu.arch;
128146
129147 ret = prepare_s2_pool(pgt_pool_base);
130148 if (ret)
131149 return ret;
132150
133− ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, mmu,
134− &host_kvm.mm_ops, KVM_HOST_S2_FLAGS,
151+ ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,
152+ &host_mmu.mm_ops, KVM_HOST_S2_FLAGS,
135153 host_stage2_force_pte_cb);
136154 if (ret)
137155 return ret;
138156
139− mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);
140− mmu->pgt = &host_kvm.pgt;
157+ mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);
158+ mmu->pgt = &host_mmu.pgt;
141159 atomic64_set(&mmu->vmid.id, 0);
142160
143161 return 0;
144162 }
145163
164+static bool guest_stage2_force_pte_cb(u64 addr, u64 end,
165+ enum kvm_pgtable_prot prot)
166+{
167+ return true;
168+}
169+
170+static void *guest_s2_zalloc_pages_exact(size_t size)
171+{
172+ void *addr = hyp_alloc_pages(&current_vm->pool, get_order(size));
173+
174+ WARN_ON(size != (PAGE_SIZE << get_order(size)));
175+ hyp_split_page(hyp_virt_to_page(addr));
176+
177+ return addr;
178+}
179+
180+static void guest_s2_free_pages_exact(void *addr, unsigned long size)
181+{
182+ u8 order = get_order(size);
183+ unsigned int i;
184+
185+ for (i = 0; i < (1 << order); i++)
186+ hyp_put_page(&current_vm->pool, addr + (i * PAGE_SIZE));
187+}
188+
189+static void *guest_s2_zalloc_page(void *mc)
190+{
191+ struct hyp_page *p;
192+ void *addr;
193+
194+ addr = hyp_alloc_pages(&current_vm->pool, 0);
195+ if (addr)
196+ return addr;
197+
198+ addr = pop_hyp_memcache(mc, hyp_phys_to_virt);
199+ if (!addr)
200+ return addr;
201+
202+ memset(addr, 0, PAGE_SIZE);
203+ p = hyp_virt_to_page(addr);
204+ memset(p, 0, sizeof(*p));
205+ p->refcount = 1;
206+
207+ return addr;
208+}
209+
210+static void guest_s2_get_page(void *addr)
211+{
212+ hyp_get_page(&current_vm->pool, addr);
213+}
214+
215+static void guest_s2_put_page(void *addr)
216+{
217+ hyp_put_page(&current_vm->pool, addr);
218+}
219+
220+static void clean_dcache_guest_page(void *va, size_t size)
221+{
222+ __clean_dcache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
223+ hyp_fixmap_unmap();
224+}
225+
226+static void invalidate_icache_guest_page(void *va, size_t size)
227+{
228+ __invalidate_icache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
229+ hyp_fixmap_unmap();
230+}
231+
232+int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd)
233+{
234+ struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
235+ unsigned long nr_pages;
236+ int ret;
237+
238+ nr_pages = kvm_pgtable_stage2_pgd_size(vm->kvm.arch.vtcr) >> PAGE_SHIFT;
239+ ret = hyp_pool_init(&vm->pool, hyp_virt_to_pfn(pgd), nr_pages, 0);
240+ if (ret)
241+ return ret;
242+
243+ hyp_spin_lock_init(&vm->lock);
244+ vm->mm_ops = (struct kvm_pgtable_mm_ops) {
245+ .zalloc_pages_exact = guest_s2_zalloc_pages_exact,
246+ .free_pages_exact = guest_s2_free_pages_exact,
247+ .zalloc_page = guest_s2_zalloc_page,
248+ .phys_to_virt = hyp_phys_to_virt,
249+ .virt_to_phys = hyp_virt_to_phys,
250+ .page_count = hyp_page_count,
251+ .get_page = guest_s2_get_page,
252+ .put_page = guest_s2_put_page,
253+ .dcache_clean_inval_poc = clean_dcache_guest_page,
254+ .icache_inval_pou = invalidate_icache_guest_page,
255+ };
256+
257+ guest_lock_component(vm);
258+ ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0,
259+ guest_stage2_force_pte_cb);
260+ guest_unlock_component(vm);
261+ if (ret)
262+ return ret;
263+
264+ vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
265+
266+ return 0;
267+}
268+
269+void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc)
270+{
271+ void *addr;
272+
273+ /* Dump all pgtable pages in the hyp_pool */
274+ guest_lock_component(vm);
275+ kvm_pgtable_stage2_destroy(&vm->pgt);
276+ vm->kvm.arch.mmu.pgd_phys = 0ULL;
277+ guest_unlock_component(vm);
278+
279+ /* Drain the hyp_pool into the memcache */
280+ addr = hyp_alloc_pages(&vm->pool, 0);
281+ while (addr) {
282+ memset(hyp_virt_to_page(addr), 0, sizeof(struct hyp_page));
283+ push_hyp_memcache(mc, addr, hyp_virt_to_phys);
284+ WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1));
285+ addr = hyp_alloc_pages(&vm->pool, 0);
286+ }
287+}
288+
146289 int __pkvm_prot_finalize(void)
147290 {
148− struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;
291+ struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
149292 struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
150293
151294 if (params->hcr_el2 & HCR_VM)
152295 return -EPERM;
153296
154297 params->vttbr = kvm_get_vttbr(mmu);
155− params->vtcr = host_kvm.arch.vtcr;
298+ params->vtcr = host_mmu.arch.vtcr;
156299 params->hcr_el2 |= HCR_VM;
157300 kvm_flush_dcache_to_poc(params, sizeof(*params));
158301
159302 write_sysreg(params->hcr_el2, hcr_el2);
160− __load_stage2(&host_kvm.arch.mmu, &host_kvm.arch);
303+ __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
161304
162305 /*
163306 * Make sure to have an ISB before the TLB maintenance below but only
@@ -175,7 +318,7 @@ int __pkvm_prot_finalize(void) · 11 unmodified lines
175318
176319 static int host_stage2_unmap_dev_all(void)
177320 {
178− struct kvm_pgtable *pgt = &host_kvm.pgt;
321+ struct kvm_pgtable *pgt = &host_mmu.pgt;
179322 struct memblock_region *reg;
180323 u64 addr = 0;
181324 int i, ret;
@@ -195,7 +338,7 @@ struct kvm_mem_range { · 13 unmodified lines
195338 u64 end;
196339 };
197340
198−static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)
341+static struct memblock_region *find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)
199342 {
200343 int cur, left = 0, right = hyp_memblock_nr;
201344 struct memblock_region *reg;
@@ -218,18 +361,28 @@ static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range) · 16 unmodified lines
218361 } else {
219362 range->start = reg->base;
220363 range->end = end;
221− return true;
364+ return reg;
222365 }
223366 }
224367
225− return false;
368+ return NULL;
226369 }
227370
228371 bool addr_is_memory(phys_addr_t phys)
229372 {
230373 struct kvm_mem_range range;
231374
232− return find_mem_range(phys, &range);
375+ return !!find_mem_range(phys, &range);
376+}
377+
378+static bool addr_is_allowed_memory(phys_addr_t phys)
379+{
380+ struct memblock_region *reg;
381+ struct kvm_mem_range range;
382+
383+ reg = find_mem_range(phys, &range);
384+
385+ return reg && !(reg->flags & MEMBLOCK_NOMAP);
233386 }
234387
235388 static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range)
@@ -250,8 +403,8 @@ static bool range_is_memory(u64 start, u64 end) · 14 unmodified lines
250403 static inline int __host_stage2_idmap(u64 start, u64 end,
251404 enum kvm_pgtable_prot prot)
252405 {
253− return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start,
254− prot, &host_s2_pool);
406+ return kvm_pgtable_stage2_map(&host_mmu.pgt, start, end - start, start,
407+ prot, &host_s2_pool, 0);
255408 }
256409
257410 /*
@@ -263,7 +416,7 @@ static inline int __host_stage2_idmap(u64 start, u64 end, · 5 unmodified lines
263416 #define host_stage2_try(fn, ...) \
264417 ({ \
265418 int __ret; \
266− hyp_assert_lock_held(&host_kvm.lock); \
419+ hyp_assert_lock_held(&host_mmu.lock); \
267420 __ret = fn(__VA_ARGS__); \
268421 if (__ret == -ENOMEM) { \
269422 __ret = host_stage2_unmap_dev_all(); \
@@ -286,8 +439,8 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) · 16 unmodified lines
286439 u32 level;
287440 int ret;
288441
289− hyp_assert_lock_held(&host_kvm.lock);
290− ret = kvm_pgtable_get_leaf(&host_kvm.pgt, addr, &pte, &level);
442+ hyp_assert_lock_held(&host_mmu.lock);
443+ ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
291444 if (ret)
292445 return ret;
293446
@@ -319,7 +472,7 @@ int host_stage2_idmap_locked(phys_addr_t addr, u64 size, · 25 unmodified lines
319472
320473 int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
321474 {
322− return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_kvm.pgt,
475+ return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
323476 addr, size, &host_s2_pool, owner_id);
324477 }
325478
@@ -348,7 +501,7 @@ static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot pr · 22 unmodified lines
348501 static int host_stage2_idmap(u64 addr)
349502 {
350503 struct kvm_mem_range range;
351− bool is_memory = find_mem_range(addr, &range);
504+ bool is_memory = !!find_mem_range(addr, &range);
352505 enum kvm_pgtable_prot prot;
353506 int ret;
354507
@@ -380,12 +533,6 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) · 25 unmodified lines
380533 BUG_ON(ret && ret != -EAGAIN);
381534 }
382535
383−/* This corresponds to locking order */
384−enum pkvm_component_id {
385− PKVM_ID_HOST,
386− PKVM_ID_HYP,
387−};
388−
389536 struct pkvm_mem_transition {
390537 u64 nr_pages;
391538
@@ -399,6 +546,9 @@ struct pkvm_mem_transition { · 7 unmodified lines
399546 /* Address in the completer's address space */
400547 u64 completer_addr;
401548 } host;
549+ struct {
550+ u64 completer_addr;
551+ } hyp;
402552 };
403553 } initiator;
404554
@@ -412,23 +562,24 @@ struct pkvm_mem_share { · 7 unmodified lines
412562 const enum kvm_pgtable_prot completer_prot;
413563 };
414564
565+struct pkvm_mem_donation {
566+ const struct pkvm_mem_transition tx;
567+};
568+
415569 struct check_walk_data {
416570 enum pkvm_page_state desired;
417571 enum pkvm_page_state (*get_page_state)(kvm_pte_t pte);
418572 };
419573
420−static int __check_page_state_visitor(u64 addr, u64 end, u32 level,
421− kvm_pte_t *ptep,
422− enum kvm_pgtable_walk_flags flag,
423− void * const arg)
574+static int __check_page_state_visitor(const struct kvm_pgtable_visit_ctx *ctx,
575+ enum kvm_pgtable_walk_flags visit)
424576 {
425− struct check_walk_data *d = arg;
426− kvm_pte_t pte = *ptep;
577+ struct check_walk_data *d = ctx->arg;
427578
428− if (kvm_pte_valid(pte) && !addr_is_memory(kvm_pte_to_phys(pte)))
579+ if (kvm_pte_valid(ctx->old) && !addr_is_allowed_memory(kvm_pte_to_phys(ctx->old)))
429580 return -EINVAL;
430581
431− return d->get_page_state(pte) == d->desired ? 0 : -EPERM;
582+ return d->get_page_state(ctx->old) == d->desired ? 0 : -EPERM;
432583 }
433584
434585 static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,
@@ -459,8 +610,8 @@ static int __host_check_page_state_range(u64 addr, u64 size, · 24 unmodified lines
459610 .get_page_state = host_get_page_state,
460611 };
461612
462− hyp_assert_lock_held(&host_kvm.lock);
463− return check_page_state_range(&host_kvm.pgt, addr, size, &d);
613+ hyp_assert_lock_held(&host_mmu.lock);
614+ return check_page_state_range(&host_mmu.pgt, addr, size, &d);
464615 }
465616
466617 static int __host_set_page_state_range(u64 addr, u64 size,
@@ -511,6 +662,46 @@ static int host_initiate_unshare(u64 *completer_addr, · 44 unmodified lines
511662 return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
512663 }
513664
665+static int host_initiate_donation(u64 *completer_addr,
666+ const struct pkvm_mem_transition *tx)
667+{
668+ u8 owner_id = tx->completer.id;
669+ u64 size = tx->nr_pages * PAGE_SIZE;
670+
671+ *completer_addr = tx->initiator.host.completer_addr;
672+ return host_stage2_set_owner_locked(tx->initiator.addr, size, owner_id);
673+}
674+
675+static bool __host_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
676+{
677+ return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
678+ tx->initiator.id != PKVM_ID_HYP);
679+}
680+
681+static int __host_ack_transition(u64 addr, const struct pkvm_mem_transition *tx,
682+ enum pkvm_page_state state)
683+{
684+ u64 size = tx->nr_pages * PAGE_SIZE;
685+
686+ if (__host_ack_skip_pgtable_check(tx))
687+ return 0;
688+
689+ return __host_check_page_state_range(addr, size, state);
690+}
691+
692+static int host_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
693+{
694+ return __host_ack_transition(addr, tx, PKVM_NOPAGE);
695+}
696+
697+static int host_complete_donation(u64 addr, const struct pkvm_mem_transition *tx)
698+{
699+ u64 size = tx->nr_pages * PAGE_SIZE;
700+ u8 host_id = tx->completer.id;
701+
702+ return host_stage2_set_owner_locked(addr, size, host_id);
703+}
704+
514705 static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte)
515706 {
516707 if (!kvm_pte_valid(pte))
@@ -531,6 +722,27 @@ static int __hyp_check_page_state_range(u64 addr, u64 size, · 14 unmodified lines
531722 return check_page_state_range(&pkvm_pgtable, addr, size, &d);
532723 }
533724
725+static int hyp_request_donation(u64 *completer_addr,
726+ const struct pkvm_mem_transition *tx)
727+{
728+ u64 size = tx->nr_pages * PAGE_SIZE;
729+ u64 addr = tx->initiator.addr;
730+
731+ *completer_addr = tx->initiator.hyp.completer_addr;
732+ return __hyp_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
733+}
734+
735+static int hyp_initiate_donation(u64 *completer_addr,
736+ const struct pkvm_mem_transition *tx)
737+{
738+ u64 size = tx->nr_pages * PAGE_SIZE;
739+ int ret;
740+
741+ *completer_addr = tx->initiator.hyp.completer_addr;
742+ ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, tx->initiator.addr, size);
743+ return (ret != size) ? -EFAULT : 0;
744+}
745+
534746 static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
535747 {
536748 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
@@ -555,6 +767,9 @@ static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 18 unmodified lines
555767 {
556768 u64 size = tx->nr_pages * PAGE_SIZE;
557769
770+ if (tx->initiator.id == PKVM_ID_HOST && hyp_page_count((void *)addr))
771+ return -EBUSY;
772+
558773 if (__hyp_ack_skip_pgtable_check(tx))
559774 return 0;
560775
@@ -562,6 +777,16 @@ static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 1 unmodified line
562777 PKVM_PAGE_SHARED_BORROWED);
563778 }
564779
780+static int hyp_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
781+{
782+ u64 size = tx->nr_pages * PAGE_SIZE;
783+
784+ if (__hyp_ack_skip_pgtable_check(tx))
785+ return 0;
786+
787+ return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
788+}
789+
565790 static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
566791 enum kvm_pgtable_prot perms)
567792 {
@@ -580,6 +805,15 @@ static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 12 unmodified lines
580805 return (ret != size) ? -EFAULT : 0;
581806 }
582807
808+static int hyp_complete_donation(u64 addr,
809+ const struct pkvm_mem_transition *tx)
810+{
811+ void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
812+ enum kvm_pgtable_prot prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_OWNED);
813+
814+ return pkvm_create_mappings_locked(start, end, prot);
815+}
816+
583817 static int check_share(struct pkvm_mem_share *share)
584818 {
585819 const struct pkvm_mem_transition *tx = &share->tx;
@@ -732,6 +966,94 @@ static int do_unshare(struct pkvm_mem_share *share) · 146 unmodified lines
732966 return WARN_ON(__do_unshare(share));
733967 }
734968
969+static int check_donation(struct pkvm_mem_donation *donation)
970+{
971+ const struct pkvm_mem_transition *tx = &donation->tx;
972+ u64 completer_addr;
973+ int ret;
974+
975+ switch (tx->initiator.id) {
976+ case PKVM_ID_HOST:
977+ ret = host_request_owned_transition(&completer_addr, tx);
978+ break;
979+ case PKVM_ID_HYP:
980+ ret = hyp_request_donation(&completer_addr, tx);
981+ break;
982+ default:
983+ ret = -EINVAL;
984+ }
985+
986+ if (ret)
987+ return ret;
988+
989+ switch (tx->completer.id) {
990+ case PKVM_ID_HOST:
991+ ret = host_ack_donation(completer_addr, tx);
992+ break;
993+ case PKVM_ID_HYP:
994+ ret = hyp_ack_donation(completer_addr, tx);
995+ break;
996+ default:
997+ ret = -EINVAL;
998+ }
999+
1000+ return ret;
1001+}
1002+
1003+static int __do_donate(struct pkvm_mem_donation *donation)
1004+{
1005+ const struct pkvm_mem_transition *tx = &donation->tx;
1006+ u64 completer_addr;
1007+ int ret;
1008+
1009+ switch (tx->initiator.id) {
1010+ case PKVM_ID_HOST:
1011+ ret = host_initiate_donation(&completer_addr, tx);
1012+ break;
1013+ case PKVM_ID_HYP:
1014+ ret = hyp_initiate_donation(&completer_addr, tx);
1015+ break;
1016+ default:
1017+ ret = -EINVAL;
1018+ }
1019+
1020+ if (ret)
1021+ return ret;
1022+
1023+ switch (tx->completer.id) {
1024+ case PKVM_ID_HOST:
1025+ ret = host_complete_donation(completer_addr, tx);
1026+ break;
1027+ case PKVM_ID_HYP:
1028+ ret = hyp_complete_donation(completer_addr, tx);
1029+ break;
1030+ default:
1031+ ret = -EINVAL;
1032+ }
1033+
1034+ return ret;
1035+}
1036+
1037+/*
1038+ * do_donate():
1039+ *
1040+ * The page owner transfers ownership to another component, losing access
1041+ * as a consequence.
1042+ *
1043+ * Initiator: OWNED => NOPAGE
1044+ * Completer: NOPAGE => OWNED
1045+ */
1046+static int do_donate(struct pkvm_mem_donation *donation)
1047+{
1048+ int ret;
1049+
1050+ ret = check_donation(donation);
1051+ if (ret)
1052+ return ret;
1053+
1054+ return WARN_ON(__do_donate(donation));
1055+}
1056+
7351057 int __pkvm_host_share_hyp(u64 pfn)
7361058 {
7371059 int ret;
@@ -797,3 +1119,112 @@ int __pkvm_host_unshare_hyp(u64 pfn) · 59 unmodified lines
7971119
7981120 return ret;
7991121 }
1122+
1123+int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
1124+{
1125+ int ret;
1126+ u64 host_addr = hyp_pfn_to_phys(pfn);
1127+ u64 hyp_addr = (u64)__hyp_va(host_addr);
1128+ struct pkvm_mem_donation donation = {
1129+ .tx = {
1130+ .nr_pages = nr_pages,
1131+ .initiator = {
1132+ .id = PKVM_ID_HOST,
1133+ .addr = host_addr,
1134+ .host = {
1135+ .completer_addr = hyp_addr,
1136+ },
1137+ },
1138+ .completer = {
1139+ .id = PKVM_ID_HYP,
1140+ },
1141+ },
1142+ };
1143+
1144+ host_lock_component();
1145+ hyp_lock_component();
1146+
1147+ ret = do_donate(&donation);
1148+
1149+ hyp_unlock_component();
1150+ host_unlock_component();
1151+
1152+ return ret;
1153+}
1154+
1155+int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages)
1156+{
1157+ int ret;
1158+ u64 host_addr = hyp_pfn_to_phys(pfn);
1159+ u64 hyp_addr = (u64)__hyp_va(host_addr);
1160+ struct pkvm_mem_donation donation = {
1161+ .tx = {
1162+ .nr_pages = nr_pages,
1163+ .initiator = {
1164+ .id = PKVM_ID_HYP,
1165+ .addr = hyp_addr,
1166+ .hyp = {
1167+ .completer_addr = host_addr,
1168+ },
1169+ },
1170+ .completer = {
1171+ .id = PKVM_ID_HOST,
1172+ },
1173+ },
1174+ };
1175+
1176+ host_lock_component();
1177+ hyp_lock_component();
1178+
1179+ ret = do_donate(&donation);
1180+
1181+ hyp_unlock_component();
1182+ host_unlock_component();
1183+
1184+ return ret;
1185+}
1186+
1187+int hyp_pin_shared_mem(void *from, void *to)
1188+{
1189+ u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1190+ u64 end = PAGE_ALIGN((u64)to);
1191+ u64 size = end - start;
1192+ int ret;
1193+
1194+ host_lock_component();
1195+ hyp_lock_component();
1196+
1197+ ret = __host_check_page_state_range(__hyp_pa(start), size,
1198+ PKVM_PAGE_SHARED_OWNED);
1199+ if (ret)
1200+ goto unlock;
1201+
1202+ ret = __hyp_check_page_state_range(start, size,
1203+ PKVM_PAGE_SHARED_BORROWED);
1204+ if (ret)
1205+ goto unlock;
1206+
1207+ for (cur = start; cur < end; cur += PAGE_SIZE)
1208+ hyp_page_ref_inc(hyp_virt_to_page(cur));
1209+
1210+unlock:
1211+ hyp_unlock_component();
1212+ host_unlock_component();
1213+
1214+ return ret;
1215+}
1216+
1217+void hyp_unpin_shared_mem(void *from, void *to)
1218+{
1219+ u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1220+ u64 end = PAGE_ALIGN((u64)to);
1221+
1222+ host_lock_component();
1223+ hyp_lock_component();
1224+
1225+ for (cur = start; cur < end; cur += PAGE_SIZE)
1226+ hyp_page_ref_dec(hyp_virt_to_page(cur));
1227+
1228+ hyp_unlock_component();
1229+ host_unlock_component();
1230+}
arch/arm64/kvm/hyp/nvhe/mm.c+163−4
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <nvhe/early_alloc.h>
1515 #include <nvhe/gfp.h>
1616 #include <nvhe/memory.h>
17+#include <nvhe/mem_protect.h>
1718 #include <nvhe/mm.h>
1819 #include <nvhe/spinlock.h>
1920
@@ -25,6 +26,12 @@ unsigned int hyp_memblock_nr; · 5 unmodified lines
2526
2627 static u64 __io_map_base;
2728
29+struct hyp_fixmap_slot {
30+ u64 addr;
31+ kvm_pte_t *ptep;
32+};
33+static DEFINE_PER_CPU(struct hyp_fixmap_slot, fixmap_slots);
34+
2835 static int __pkvm_create_mappings(unsigned long start, unsigned long size,
2936 unsigned long phys, enum kvm_pgtable_prot prot)
3037 {
@@ -129,13 +136,36 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) · 98 unmodified lines
129136 return ret;
130137 }
131138
132−int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)
139+int hyp_back_vmemmap(phys_addr_t back)
133140 {
134− unsigned long start, end;
141+ unsigned long i, start, size, end = 0;
142+ int ret;
135143
136− hyp_vmemmap_range(phys, size, &start, &end);
144+ for (i = 0; i < hyp_memblock_nr; i++) {
145+ start = hyp_memory[i].base;
146+ start = ALIGN_DOWN((u64)hyp_phys_to_page(start), PAGE_SIZE);
147+ /*
148+ * The begining of the hyp_vmemmap region for the current
149+ * memblock may already be backed by the page backing the end
150+ * the previous region, so avoid mapping it twice.
151+ */
152+ start = max(start, end);
153+
154+ end = hyp_memory[i].base + hyp_memory[i].size;
155+ end = PAGE_ALIGN((u64)hyp_phys_to_page(end));
156+ if (start >= end)
157+ continue;
158+
159+ size = end - start;
160+ ret = __pkvm_create_mappings(start, size, back, PAGE_HYP);
161+ if (ret)
162+ return ret;
163+
164+ memset(hyp_phys_to_virt(back), 0, size);
165+ back += size;
166+ }
137167
138− return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);
168+ return 0;
139169 }
140170
141171 static void *__hyp_bp_vect_base;
@@ -189,6 +219,102 @@ int hyp_map_vectors(void) · 47 unmodified lines
189219 return 0;
190220 }
191221
222+void *hyp_fixmap_map(phys_addr_t phys)
223+{
224+ struct hyp_fixmap_slot *slot = this_cpu_ptr(&fixmap_slots);
225+ kvm_pte_t pte, *ptep = slot->ptep;
226+
227+ pte = *ptep;
228+ pte &= ~kvm_phys_to_pte(KVM_PHYS_INVALID);
229+ pte |= kvm_phys_to_pte(phys) | KVM_PTE_VALID;
230+ WRITE_ONCE(*ptep, pte);
231+ dsb(ishst);
232+
233+ return (void *)slot->addr;
234+}
235+
236+static void fixmap_clear_slot(struct hyp_fixmap_slot *slot)
237+{
238+ kvm_pte_t *ptep = slot->ptep;
239+ u64 addr = slot->addr;
240+
241+ WRITE_ONCE(*ptep, *ptep & ~KVM_PTE_VALID);
242+
243+ /*
244+ * Irritatingly, the architecture requires that we use inner-shareable
245+ * broadcast TLB invalidation here in case another CPU speculates
246+ * through our fixmap and decides to create an "amalagamation of the
247+ * values held in the TLB" due to the apparent lack of a
248+ * break-before-make sequence.
249+ *
250+ * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#mf10dfbaf1eaef9274c581b81c53758918c1d0f03
251+ */
252+ dsb(ishst);
253+ __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), (KVM_PGTABLE_MAX_LEVELS - 1));
254+ dsb(ish);
255+ isb();
256+}
257+
258+void hyp_fixmap_unmap(void)
259+{
260+ fixmap_clear_slot(this_cpu_ptr(&fixmap_slots));
261+}
262+
263+static int __create_fixmap_slot_cb(const struct kvm_pgtable_visit_ctx *ctx,
264+ enum kvm_pgtable_walk_flags visit)
265+{
266+ struct hyp_fixmap_slot *slot = per_cpu_ptr(&fixmap_slots, (u64)ctx->arg);
267+
268+ if (!kvm_pte_valid(ctx->old) || ctx->level != KVM_PGTABLE_MAX_LEVELS - 1)
269+ return -EINVAL;
270+
271+ slot->addr = ctx->addr;
272+ slot->ptep = ctx->ptep;
273+
274+ /*
275+ * Clear the PTE, but keep the page-table page refcount elevated to
276+ * prevent it from ever being freed. This lets us manipulate the PTEs
277+ * by hand safely without ever needing to allocate memory.
278+ */
279+ fixmap_clear_slot(slot);
280+
281+ return 0;
282+}
283+
284+static int create_fixmap_slot(u64 addr, u64 cpu)
285+{
286+ struct kvm_pgtable_walker walker = {
287+ .cb = __create_fixmap_slot_cb,
288+ .flags = KVM_PGTABLE_WALK_LEAF,
289+ .arg = (void *)cpu,
290+ };
291+
292+ return kvm_pgtable_walk(&pkvm_pgtable, addr, PAGE_SIZE, &walker);
293+}
294+
295+int hyp_create_pcpu_fixmap(void)
296+{
297+ unsigned long addr, i;
298+ int ret;
299+
300+ for (i = 0; i < hyp_nr_cpus; i++) {
301+ ret = pkvm_alloc_private_va_range(PAGE_SIZE, &addr);
302+ if (ret)
303+ return ret;
304+
305+ ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PAGE_SIZE,
306+ __hyp_pa(__hyp_bss_start), PAGE_HYP);
307+ if (ret)
308+ return ret;
309+
310+ ret = create_fixmap_slot(addr, i);
311+ if (ret)
312+ return ret;
313+ }
314+
315+ return 0;
316+}
317+
192318 int hyp_create_idmap(u32 hyp_va_bits)
193319 {
194320 unsigned long start, end;
@@ -213,3 +339,36 @@ int hyp_create_idmap(u32 hyp_va_bits) · 18 unmodified lines
213339
214340 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
215341 }
342+
343+static void *admit_host_page(void *arg)
344+{
345+ struct kvm_hyp_memcache *host_mc = arg;
346+
347+ if (!host_mc->nr_pages)
348+ return NULL;
349+
350+ /*
351+ * The host still owns the pages in its memcache, so we need to go
352+ * through a full host-to-hyp donation cycle to change it. Fortunately,
353+ * __pkvm_host_donate_hyp() takes care of races for us, so if it
354+ * succeeds we're good to go.
355+ */
356+ if (__pkvm_host_donate_hyp(hyp_phys_to_pfn(host_mc->head), 1))
357+ return NULL;
358+
359+ return pop_hyp_memcache(host_mc, hyp_phys_to_virt);
360+}
361+
362+/* Refill our local memcache by poping pages from the one provided by the host. */
363+int refill_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages,
364+ struct kvm_hyp_memcache *host_mc)
365+{
366+ struct kvm_hyp_memcache tmp = *host_mc;
367+ int ret;
368+
369+ ret = __topup_hyp_memcache(mc, min_pages, admit_host_page,
370+ hyp_virt_to_phys, &tmp);
371+ *host_mc = tmp;
372+
373+ return ret;
374+}
arch/arm64/kvm/hyp/nvhe/page_alloc.c+7−22
@@ -93,11 +93,16 @@ static inline struct hyp_page *node_to_page(struct list_head *node) · 92 unmodified lines
9393 static void __hyp_attach_page(struct hyp_pool *pool,
9494 struct hyp_page *p)
9595 {
96+ phys_addr_t phys = hyp_page_to_phys(p);
9697 unsigned short order = p->order;
9798 struct hyp_page *buddy;
9899
99100 memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
100101
102+ /* Skip coalescing for 'external' pages being freed into the pool. */
103+ if (phys < pool->range_start || phys >= pool->range_end)
104+ goto insert;
105+
101106 /*
102107 * Only the first struct hyp_page of a high-order page (otherwise known
103108 * as the 'head') should have p->order set. The non-head pages should
@@ -116,6 +121,7 @@ static void __hyp_attach_page(struct hyp_pool *pool, · 12 unmodified lines
116121 p = min(p, buddy);
117122 }
118123
124+insert:
119125 /* Mark the new head, and insert it */
120126 p->order = order;
121127 page_add_to_list(p, &pool->free_area[order]);
@@ -144,25 +150,6 @@ static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, · 22 unmodified lines
144150 return p;
145151 }
146152
147−static inline void hyp_page_ref_inc(struct hyp_page *p)
148−{
149− BUG_ON(p->refcount == USHRT_MAX);
150− p->refcount++;
151−}
152−
153−static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
154−{
155− BUG_ON(!p->refcount);
156− p->refcount--;
157− return (p->refcount == 0);
158−}
159−
160−static inline void hyp_set_page_refcounted(struct hyp_page *p)
161−{
162− BUG_ON(p->refcount);
163− p->refcount = 1;
164−}
165−
166153 static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p)
167154 {
168155 if (hyp_page_ref_dec_and_test(p))
@@ -249,10 +236,8 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, · 80 unmodified lines
249236
250237 /* Init the vmemmap portion */
251238 p = hyp_phys_to_page(phys);
252− for (i = 0; i < nr_pages; i++) {
253− p[i].order = 0;
239+ for (i = 0; i < nr_pages; i++)
254240 hyp_set_page_refcounted(&p[i]);
255− }
256241
257242 /* Attach the unused pages to the buddy tree */
258243 for (i = reserved_pages; i < nr_pages; i++)
arch/arm64/kvm/hyp/nvhe/pkvm.c+436−0
@@ -7,8 +7,17 @@ · 6 unmodified lines
77 #include <linux/kvm_host.h>
88 #include <linux/mm.h>
99 #include <nvhe/fixed_config.h>
10+#include <nvhe/mem_protect.h>
11+#include <nvhe/memory.h>
12+#include <nvhe/pkvm.h>
1013 #include <nvhe/trap_handler.h>
1114
15+/* Used by icache_is_vpipt(). */
16+unsigned long __icache_flags;
17+
18+/* Used by kvm_get_vttbr(). */
19+unsigned int kvm_arm_vmid_bits;
20+
1221 /*
1322 * Set trap register values based on features in ID_AA64PFR0.
1423 */
@@ -183,3 +192,430 @@ void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu) · 168 unmodified lines
183192 pvm_init_traps_aa64mmfr0(vcpu);
184193 pvm_init_traps_aa64mmfr1(vcpu);
185194 }
195+
196+/*
197+ * Start the VM table handle at the offset defined instead of at 0.
198+ * Mainly for sanity checking and debugging.
199+ */
200+#define HANDLE_OFFSET 0x1000
201+
202+static unsigned int vm_handle_to_idx(pkvm_handle_t handle)
203+{
204+ return handle - HANDLE_OFFSET;
205+}
206+
207+static pkvm_handle_t idx_to_vm_handle(unsigned int idx)
208+{
209+ return idx + HANDLE_OFFSET;
210+}
211+
212+/*
213+ * Spinlock for protecting state related to the VM table. Protects writes
214+ * to 'vm_table' and 'nr_table_entries' as well as reads and writes to
215+ * 'last_hyp_vcpu_lookup'.
216+ */
217+static DEFINE_HYP_SPINLOCK(vm_table_lock);
218+
219+/*
220+ * The table of VM entries for protected VMs in hyp.
221+ * Allocated at hyp initialization and setup.
222+ */
223+static struct pkvm_hyp_vm **vm_table;
224+
225+void pkvm_hyp_vm_table_init(void *tbl)
226+{
227+ WARN_ON(vm_table);
228+ vm_table = tbl;
229+}
230+
231+/*
232+ * Return the hyp vm structure corresponding to the handle.
233+ */
234+static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle)
235+{
236+ unsigned int idx = vm_handle_to_idx(handle);
237+
238+ if (unlikely(idx >= KVM_MAX_PVMS))
239+ return NULL;
240+
241+ return vm_table[idx];
242+}
243+
244+struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
245+ unsigned int vcpu_idx)
246+{
247+ struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
248+ struct pkvm_hyp_vm *hyp_vm;
249+
250+ hyp_spin_lock(&vm_table_lock);
251+ hyp_vm = get_vm_by_handle(handle);
252+ if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx)
253+ goto unlock;
254+
255+ hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
256+ hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
257+unlock:
258+ hyp_spin_unlock(&vm_table_lock);
259+ return hyp_vcpu;
260+}
261+
262+void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
263+{
264+ struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
265+
266+ hyp_spin_lock(&vm_table_lock);
267+ hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
268+ hyp_spin_unlock(&vm_table_lock);
269+}
270+
271+static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
272+{
273+ if (host_vcpu)
274+ hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1);
275+}
276+
277+static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[],
278+ unsigned int nr_vcpus)
279+{
280+ int i;
281+
282+ for (i = 0; i < nr_vcpus; i++)
283+ unpin_host_vcpu(hyp_vcpus[i]->host_vcpu);
284+}
285+
286+static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
287+ unsigned int nr_vcpus)
288+{
289+ hyp_vm->host_kvm = host_kvm;
290+ hyp_vm->kvm.created_vcpus = nr_vcpus;
291+ hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr;
292+}
293+
294+static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
295+ struct pkvm_hyp_vm *hyp_vm,
296+ struct kvm_vcpu *host_vcpu,
297+ unsigned int vcpu_idx)
298+{
299+ int ret = 0;
300+
301+ if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
302+ return -EBUSY;
303+
304+ if (host_vcpu->vcpu_idx != vcpu_idx) {
305+ ret = -EINVAL;
306+ goto done;
307+ }
308+
309+ hyp_vcpu->host_vcpu = host_vcpu;
310+
311+ hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
312+ hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id);
313+ hyp_vcpu->vcpu.vcpu_idx = vcpu_idx;
314+
315+ hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
316+ hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
317+done:
318+ if (ret)
319+ unpin_host_vcpu(host_vcpu);
320+ return ret;
321+}
322+
323+static int find_free_vm_table_entry(struct kvm *host_kvm)
324+{
325+ int i;
326+
327+ for (i = 0; i < KVM_MAX_PVMS; ++i) {
328+ if (!vm_table[i])
329+ return i;
330+ }
331+
332+ return -ENOMEM;
333+}
334+
335+/*
336+ * Allocate a VM table entry and insert a pointer to the new vm.
337+ *
338+ * Return a unique handle to the protected VM on success,
339+ * negative error code on failure.
340+ */
341+static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm,
342+ struct pkvm_hyp_vm *hyp_vm)
343+{
344+ struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu;
345+ int idx;
346+
347+ hyp_assert_lock_held(&vm_table_lock);
348+
349+ /*
350+ * Initializing protected state might have failed, yet a malicious
351+ * host could trigger this function. Thus, ensure that 'vm_table'
352+ * exists.
353+ */
354+ if (unlikely(!vm_table))
355+ return -EINVAL;
356+
357+ idx = find_free_vm_table_entry(host_kvm);
358+ if (idx < 0)
359+ return idx;
360+
361+ hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx);
362+
363+ /* VMID 0 is reserved for the host */
364+ atomic64_set(&mmu->vmid.id, idx + 1);
365+
366+ mmu->arch = &hyp_vm->kvm.arch;
367+ mmu->pgt = &hyp_vm->pgt;
368+
369+ vm_table[idx] = hyp_vm;
370+ return hyp_vm->kvm.arch.pkvm.handle;
371+}
372+
373+/*
374+ * Deallocate and remove the VM table entry corresponding to the handle.
375+ */
376+static void remove_vm_table_entry(pkvm_handle_t handle)
377+{
378+ hyp_assert_lock_held(&vm_table_lock);
379+ vm_table[vm_handle_to_idx(handle)] = NULL;
380+}
381+
382+static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
383+{
384+ return size_add(sizeof(struct pkvm_hyp_vm),
385+ size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
386+}
387+
388+static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
389+{
390+ void *va = (void *)kern_hyp_va(host_va);
391+
392+ if (!PAGE_ALIGNED(va))
393+ return NULL;
394+
395+ if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va),
396+ PAGE_ALIGN(size) >> PAGE_SHIFT))
397+ return NULL;
398+
399+ return va;
400+}
401+
402+static void *map_donated_memory(unsigned long host_va, size_t size)
403+{
404+ void *va = map_donated_memory_noclear(host_va, size);
405+
406+ if (va)
407+ memset(va, 0, size);
408+
409+ return va;
410+}
411+
412+static void __unmap_donated_memory(void *va, size_t size)
413+{
414+ WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va),
415+ PAGE_ALIGN(size) >> PAGE_SHIFT));
416+}
417+
418+static void unmap_donated_memory(void *va, size_t size)
419+{
420+ if (!va)
421+ return;
422+
423+ memset(va, 0, size);
424+ __unmap_donated_memory(va, size);
425+}
426+
427+static void unmap_donated_memory_noclear(void *va, size_t size)
428+{
429+ if (!va)
430+ return;
431+
432+ __unmap_donated_memory(va, size);
433+}
434+
435+/*
436+ * Initialize the hypervisor copy of the protected VM state using the
437+ * memory donated by the host.
438+ *
439+ * Unmaps the donated memory from the host at stage 2.
440+ *
441+ * host_kvm: A pointer to the host's struct kvm.
442+ * vm_hva: The host va of the area being donated for the VM state.
443+ * Must be page aligned.
444+ * pgd_hva: The host va of the area being donated for the stage-2 PGD for
445+ * the VM. Must be page aligned. Its size is implied by the VM's
446+ * VTCR.
447+ *
448+ * Return a unique handle to the protected VM on success,
449+ * negative error code on failure.
450+ */
451+int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
452+ unsigned long pgd_hva)
453+{
454+ struct pkvm_hyp_vm *hyp_vm = NULL;
455+ size_t vm_size, pgd_size;
456+ unsigned int nr_vcpus;
457+ void *pgd = NULL;
458+ int ret;
459+
460+ ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1);
461+ if (ret)
462+ return ret;
463+
464+ nr_vcpus = READ_ONCE(host_kvm->created_vcpus);
465+ if (nr_vcpus < 1) {
466+ ret = -EINVAL;
467+ goto err_unpin_kvm;
468+ }
469+
470+ vm_size = pkvm_get_hyp_vm_size(nr_vcpus);
471+ pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr);
472+
473+ ret = -ENOMEM;
474+
475+ hyp_vm = map_donated_memory(vm_hva, vm_size);
476+ if (!hyp_vm)
477+ goto err_remove_mappings;
478+
479+ pgd = map_donated_memory_noclear(pgd_hva, pgd_size);
480+ if (!pgd)
481+ goto err_remove_mappings;
482+
483+ init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus);
484+
485+ hyp_spin_lock(&vm_table_lock);
486+ ret = insert_vm_table_entry(host_kvm, hyp_vm);
487+ if (ret < 0)
488+ goto err_unlock;
489+
490+ ret = kvm_guest_prepare_stage2(hyp_vm, pgd);
491+ if (ret)
492+ goto err_remove_vm_table_entry;
493+ hyp_spin_unlock(&vm_table_lock);
494+
495+ return hyp_vm->kvm.arch.pkvm.handle;
496+
497+err_remove_vm_table_entry:
498+ remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle);
499+err_unlock:
500+ hyp_spin_unlock(&vm_table_lock);
501+err_remove_mappings:
502+ unmap_donated_memory(hyp_vm, vm_size);
503+ unmap_donated_memory(pgd, pgd_size);
504+err_unpin_kvm:
505+ hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
506+ return ret;
507+}
508+
509+/*
510+ * Initialize the hypervisor copy of the protected vCPU state using the
511+ * memory donated by the host.
512+ *
513+ * handle: The handle for the protected vm.
514+ * host_vcpu: A pointer to the corresponding host vcpu.
515+ * vcpu_hva: The host va of the area being donated for the vcpu state.
516+ * Must be page aligned. The size of the area must be equal to
517+ * the page-aligned size of 'struct pkvm_hyp_vcpu'.
518+ * Return 0 on success, negative error code on failure.
519+ */
520+int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
521+ unsigned long vcpu_hva)
522+{
523+ struct pkvm_hyp_vcpu *hyp_vcpu;
524+ struct pkvm_hyp_vm *hyp_vm;
525+ unsigned int idx;
526+ int ret;
527+
528+ hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
529+ if (!hyp_vcpu)
530+ return -ENOMEM;
531+
532+ hyp_spin_lock(&vm_table_lock);
533+
534+ hyp_vm = get_vm_by_handle(handle);
535+ if (!hyp_vm) {
536+ ret = -ENOENT;
537+ goto unlock;
538+ }
539+
540+ idx = hyp_vm->nr_vcpus;
541+ if (idx >= hyp_vm->kvm.created_vcpus) {
542+ ret = -EINVAL;
543+ goto unlock;
544+ }
545+
546+ ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx);
547+ if (ret)
548+ goto unlock;
549+
550+ hyp_vm->vcpus[idx] = hyp_vcpu;
551+ hyp_vm->nr_vcpus++;
552+unlock:
553+ hyp_spin_unlock(&vm_table_lock);
554+
555+ if (ret)
556+ unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
557+
558+ return ret;
559+}
560+
561+static void
562+teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size)
563+{
564+ size = PAGE_ALIGN(size);
565+ memset(addr, 0, size);
566+
567+ for (void *start = addr; start < addr + size; start += PAGE_SIZE)
568+ push_hyp_memcache(mc, start, hyp_virt_to_phys);
569+
570+ unmap_donated_memory_noclear(addr, size);
571+}
572+
573+int __pkvm_teardown_vm(pkvm_handle_t handle)
574+{
575+ struct kvm_hyp_memcache *mc;
576+ struct pkvm_hyp_vm *hyp_vm;
577+ struct kvm *host_kvm;
578+ unsigned int idx;
579+ size_t vm_size;
580+ int err;
581+
582+ hyp_spin_lock(&vm_table_lock);
583+ hyp_vm = get_vm_by_handle(handle);
584+ if (!hyp_vm) {
585+ err = -ENOENT;
586+ goto err_unlock;
587+ }
588+
589+ if (WARN_ON(hyp_page_count(hyp_vm))) {
590+ err = -EBUSY;
591+ goto err_unlock;
592+ }
593+
594+ host_kvm = hyp_vm->host_kvm;
595+
596+ /* Ensure the VMID is clean before it can be reallocated */
597+ __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
598+ remove_vm_table_entry(handle);
599+ hyp_spin_unlock(&vm_table_lock);
600+
601+ /* Reclaim guest pages (including page-table pages) */
602+ mc = &host_kvm->arch.pkvm.teardown_mc;
603+ reclaim_guest_pages(hyp_vm, mc);
604+ unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus);
605+
606+ /* Push the metadata pages to the teardown memcache */
607+ for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) {
608+ struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
609+
610+ teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu));
611+ }
612+
613+ vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
614+ teardown_donated_memory(mc, hyp_vm, vm_size);
615+ hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
616+ return 0;
617+
618+err_unlock:
619+ hyp_spin_unlock(&vm_table_lock);
620+ return err;
621+}
arch/arm64/kvm/hyp/nvhe/setup.c+61−37
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <nvhe/memory.h>
1717 #include <nvhe/mem_protect.h>
1818 #include <nvhe/mm.h>
19+#include <nvhe/pkvm.h>
1920 #include <nvhe/trap_handler.h>
2021
2122 unsigned long hyp_nr_cpus;
@@ -24,6 +25,7 @@ unsigned long hyp_nr_cpus; · 2 unmodified lines
2425 (unsigned long)__per_cpu_start)
2526
2627 static void *vmemmap_base;
28+static void *vm_table_base;
2729 static void *hyp_pgt_base;
2830 static void *host_s2_pgt_base;
2931 static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
@@ -31,16 +33,20 @@ static struct hyp_pool hpool; · 1 unmodified line
3133
3234 static int divide_memory_pool(void *virt, unsigned long size)
3335 {
34− unsigned long vstart, vend, nr_pages;
36+ unsigned long nr_pages;
3537
3638 hyp_early_alloc_init(virt, size);
3739
38− hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend);
39− nr_pages = (vend - vstart) >> PAGE_SHIFT;
40+ nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page));
4041 vmemmap_base = hyp_early_alloc_contig(nr_pages);
4142 if (!vmemmap_base)
4243 return -ENOMEM;
4344
45+ nr_pages = hyp_vm_table_pages();
46+ vm_table_base = hyp_early_alloc_contig(nr_pages);
47+ if (!vm_table_base)
48+ return -ENOMEM;
49+
4450 nr_pages = hyp_s1_pgtable_pages();
4551 hyp_pgt_base = hyp_early_alloc_contig(nr_pages);
4652 if (!hyp_pgt_base)
@@ -78,7 +84,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, · 31 unmodified lines
7884 if (ret)
7985 return ret;
8086
81− ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base));
87+ ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base));
8288 if (ret)
8389 return ret;
8490
@@ -138,20 +144,17 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, · 53 unmodified lines
138144 }
139145
140146 /*
141− * Map the host's .bss and .rodata sections RO in the hypervisor, but
142− * transfer the ownership from the host to the hypervisor itself to
143− * make sure it can't be donated or shared with another entity.
147+ * Map the host sections RO in the hypervisor, but transfer the
148+ * ownership from the host to the hypervisor itself to make sure they
149+ * can't be donated or shared with another entity.
144150 *
145151 * The ownership transition requires matching changes in the host
146152 * stage-2. This will be done later (see finalize_host_mappings()) once
147153 * the hyp_vmemmap is addressable.
148154 */
149155 prot = pkvm_mkstate(PAGE_HYP_RO, PKVM_PAGE_SHARED_OWNED);
150− ret = pkvm_create_mappings(__start_rodata, __end_rodata, prot);
151− if (ret)
152− return ret;
153−
154− ret = pkvm_create_mappings(__hyp_bss_end, __bss_stop, prot);
156+ ret = pkvm_create_mappings(&kvm_vgic_global_state,
157+ &kvm_vgic_global_state + 1, prot);
155158 if (ret)
156159 return ret;
157160
@@ -186,33 +189,20 @@ static void hpool_put_page(void *addr) · 28 unmodified lines
186189 hyp_put_page(&hpool, addr);
187190 }
188191
189−static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level,
190− kvm_pte_t *ptep,
191− enum kvm_pgtable_walk_flags flag,
192− void * const arg)
192+static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
193+ enum kvm_pgtable_walk_flags visit)
193194 {
194− struct kvm_pgtable_mm_ops *mm_ops = arg;
195195 enum kvm_pgtable_prot prot;
196196 enum pkvm_page_state state;
197− kvm_pte_t pte = *ptep;
198197 phys_addr_t phys;
199198
200− if (!kvm_pte_valid(pte))
199+ if (!kvm_pte_valid(ctx->old))
201200 return 0;
202201
203− /*
204− * Fix-up the refcount for the page-table pages as the early allocator
205− * was unable to access the hyp_vmemmap and so the buddy allocator has
206− * initialised the refcount to '1'.
207− */
208− mm_ops->get_page(ptep);
209− if (flag != KVM_PGTABLE_WALK_LEAF)
210− return 0;
211−
212− if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
202+ if (ctx->level != (KVM_PGTABLE_MAX_LEVELS - 1))
213203 return -EINVAL;
214204
215− phys = kvm_pte_to_phys(pte);
205+ phys = kvm_pte_to_phys(ctx->old);
216206 if (!addr_is_memory(phys))
217207 return -EINVAL;
218208
@@ -220,10 +210,10 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 1 unmodified line
220210 * Adjust the host stage-2 mappings to match the ownership attributes
221211 * configured in the hypervisor stage-1.
222212 */
223− state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(pte));
213+ state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(ctx->old));
224214 switch (state) {
225215 case PKVM_PAGE_OWNED:
226− return host_stage2_set_owner_locked(phys, PAGE_SIZE, pkvm_hyp_id);
216+ return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP);
227217 case PKVM_PAGE_SHARED_OWNED:
228218 prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_BORROWED);
229219 break;
@@ -237,12 +227,25 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 7 unmodified lines
237227 return host_stage2_idmap_locked(phys, PAGE_SIZE, prot);
238228 }
239229
240−static int finalize_host_mappings(void)
230+static int fix_hyp_pgtable_refcnt_walker(const struct kvm_pgtable_visit_ctx *ctx,
231+ enum kvm_pgtable_walk_flags visit)
232+{
233+ /*
234+ * Fix-up the refcount for the page-table pages as the early allocator
235+ * was unable to access the hyp_vmemmap and so the buddy allocator has
236+ * initialised the refcount to '1'.
237+ */
238+ if (kvm_pte_valid(ctx->old))
239+ ctx->mm_ops->get_page(ctx->ptep);
240+
241+ return 0;
242+}
243+
244+static int fix_host_ownership(void)
241245 {
242246 struct kvm_pgtable_walker walker = {
243− .cb = finalize_host_mappings_walker,
244− .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
245− .arg = pkvm_pgtable.mm_ops,
247+ .cb = fix_host_ownership_walker,
248+ .flags = KVM_PGTABLE_WALK_LEAF,
246249 };
247250 int i, ret;
248251
@@ -258,6 +261,18 @@ static int finalize_host_mappings(void) · 9 unmodified lines
258261 return 0;
259262 }
260263
264+static int fix_hyp_pgtable_refcnt(void)
265+{
266+ struct kvm_pgtable_walker walker = {
267+ .cb = fix_hyp_pgtable_refcnt_walker,
268+ .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
269+ .arg = pkvm_pgtable.mm_ops,
270+ };
271+
272+ return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits),
273+ &walker);
274+}
275+
261276 void __noreturn __pkvm_init_finalise(void)
262277 {
263278 struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data);
@@ -287,10 +302,19 @@ void __noreturn __pkvm_init_finalise(void) · 23 unmodified lines
287302 };
288303 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
289304
290− ret = finalize_host_mappings();
305+ ret = fix_host_ownership();
306+ if (ret)
307+ goto out;
308+
309+ ret = fix_hyp_pgtable_refcnt();
310+ if (ret)
311+ goto out;
312+
313+ ret = hyp_create_pcpu_fixmap();
291314 if (ret)
292315 goto out;
293316
317+ pkvm_hyp_vm_table_init(vm_table_base);
294318 out:
295319 /*
296320 * We tail-called to here from handle___pkvm_init() and will not return,
arch/arm64/kvm/hyp/pgtable.c+343−309
@@ -49,35 +49,38 @@ · 48 unmodified lines
4949 #define KVM_INVALID_PTE_OWNER_MASK GENMASK(9, 2)
5050 #define KVM_MAX_OWNER_ID 1
5151
52+/*
53+ * Used to indicate a pte for which a 'break-before-make' sequence is in
54+ * progress.
55+ */
56+#define KVM_INVALID_PTE_LOCKED BIT(10)
57+
5258 struct kvm_pgtable_walk_data {
53− struct kvm_pgtable *pgt;
5459 struct kvm_pgtable_walker *walker;
5560
5661 u64 addr;
5762 u64 end;
5863 };
5964
60−#define KVM_PHYS_INVALID (-1ULL)
61−
6265 static bool kvm_phys_is_valid(u64 phys)
6366 {
6467 return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX));
6568 }
6669
67−static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)
70+static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys)
6871 {
69− u64 granule = kvm_granule_size(level);
72+ u64 granule = kvm_granule_size(ctx->level);
7073
71− if (!kvm_level_supports_block_mapping(level))
74+ if (!kvm_level_supports_block_mapping(ctx->level))
7275 return false;
7376
74− if (granule > (end - addr))
77+ if (granule > (ctx->end - ctx->addr))
7578 return false;
7679
7780 if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule))
7881 return false;
7982
80− return IS_ALIGNED(addr, granule);
83+ return IS_ALIGNED(ctx->addr, granule);
8184 }
8285
8386 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)
@@ -88,7 +91,7 @@ static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) · 4 unmodified lines
8891 return (data->addr >> shift) & mask;
8992 }
9093
91−static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)
94+static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)
9295 {
9396 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */
9497 u64 mask = BIT(pgt->ia_bits) - 1;
@@ -96,11 +99,6 @@ static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) · 1 unmodified line
9699 return (addr & mask) >> shift;
97100 }
98101
99−static u32 kvm_pgd_page_idx(struct kvm_pgtable_walk_data *data)
100−{
101− return __kvm_pgd_page_idx(data->pgt, data->addr);
102−}
103−
104102 static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level)
105103 {
106104 struct kvm_pgtable pgt = {
@@ -108,7 +106,7 @@ static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) · 1 unmodified line
108106 .start_level = start_level,
109107 };
110108
111− return __kvm_pgd_page_idx(&pgt, -1ULL) + 1;
109+ return kvm_pgd_page_idx(&pgt, -1ULL) + 1;
112110 }
113111
114112 static bool kvm_pte_table(kvm_pte_t pte, u32 level)
@@ -122,16 +120,6 @@ static bool kvm_pte_table(kvm_pte_t pte, u32 level) · 7 unmodified lines
122120 return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE;
123121 }
124122
125−static kvm_pte_t kvm_phys_to_pte(u64 pa)
126−{
127− kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK;
128−
129− if (PAGE_SHIFT == 16)
130− pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48);
131−
132− return pte;
133−}
134−
135123 static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
136124 {
137125 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
@@ -142,16 +130,13 @@ static void kvm_clear_pte(kvm_pte_t *ptep) · 4 unmodified lines
142130 WRITE_ONCE(*ptep, 0);
143131 }
144132
145−static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,
146− struct kvm_pgtable_mm_ops *mm_ops)
133+static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
147134 {
148− kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
135+ kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
149136
150137 pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);
151138 pte |= KVM_PTE_VALID;
152−
153− WARN_ON(kvm_pte_valid(old));
154− smp_store_release(ptep, pte);
139+ return pte;
155140 }
156141
157142 static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
@@ -172,36 +157,47 @@ static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id) · 14 unmodified lines
172157 return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
173158 }
174159
175−static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,
176− u32 level, kvm_pte_t *ptep,
177− enum kvm_pgtable_walk_flags flag)
160+static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data,
161+ const struct kvm_pgtable_visit_ctx *ctx,
162+ enum kvm_pgtable_walk_flags visit)
178163 {
179164 struct kvm_pgtable_walker *walker = data->walker;
180− return walker->cb(addr, data->end, level, ptep, flag, walker->arg);
165+
166+ /* Ensure the appropriate lock is held (e.g. RCU lock for stage-2 MMU) */
167+ WARN_ON_ONCE(kvm_pgtable_walk_shared(ctx) && !kvm_pgtable_walk_lock_held());
168+ return walker->cb(ctx, visit);
181169 }
182170
183171 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
184− kvm_pte_t *pgtable, u32 level);
172+ struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level);
185173
186174 static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
187− kvm_pte_t *ptep, u32 level)
175+ struct kvm_pgtable_mm_ops *mm_ops,
176+ kvm_pteref_t pteref, u32 level)
188177 {
189− int ret = 0;
190− u64 addr = data->addr;
191− kvm_pte_t *childp, pte = *ptep;
192− bool table = kvm_pte_table(pte, level);
193178 enum kvm_pgtable_walk_flags flags = data->walker->flags;
179+ kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);
180+ struct kvm_pgtable_visit_ctx ctx = {
181+ .ptep = ptep,
182+ .old = READ_ONCE(*ptep),
183+ .arg = data->walker->arg,
184+ .mm_ops = mm_ops,
185+ .addr = data->addr,
186+ .end = data->end,
187+ .level = level,
188+ .flags = flags,
189+ };
190+ int ret = 0;
191+ kvm_pteref_t childp;
192+ bool table = kvm_pte_table(ctx.old, level);
194193
195− if (table && (flags & KVM_PGTABLE_WALK_TABLE_PRE)) {
196− ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,
197− KVM_PGTABLE_WALK_TABLE_PRE);
198− }
194+ if (table && (ctx.flags & KVM_PGTABLE_WALK_TABLE_PRE))
195+ ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_PRE);
199196
200− if (!table && (flags & KVM_PGTABLE_WALK_LEAF)) {
201− ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,
202− KVM_PGTABLE_WALK_LEAF);
203− pte = *ptep;
204− table = kvm_pte_table(pte, level);
197+ if (!table && (ctx.flags & KVM_PGTABLE_WALK_LEAF)) {
198+ ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_LEAF);
199+ ctx.old = READ_ONCE(*ptep);
200+ table = kvm_pte_table(ctx.old, level);
205201 }
206202
207203 if (ret)
@@ -213,22 +209,20 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, · 5 unmodified lines
213209 goto out;
214210 }
215211
216− childp = kvm_pte_follow(pte, data->pgt->mm_ops);
217− ret = __kvm_pgtable_walk(data, childp, level + 1);
212+ childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
213+ ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
218214 if (ret)
219215 goto out;
220216
221− if (flags & KVM_PGTABLE_WALK_TABLE_POST) {
222− ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,
223− KVM_PGTABLE_WALK_TABLE_POST);
224− }
217+ if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
218+ ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST);
225219
226220 out:
227221 return ret;
228222 }
229223
230224 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
231− kvm_pte_t *pgtable, u32 level)
225+ struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level)
232226 {
233227 u32 idx;
234228 int ret = 0;
@@ -237,12 +231,12 @@ static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, · 2 unmodified lines
237231 return -EINVAL;
238232
239233 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {
240− kvm_pte_t *ptep = &pgtable[idx];
234+ kvm_pteref_t pteref = &pgtable[idx];
241235
242236 if (data->addr >= data->end)
243237 break;
244238
245− ret = __kvm_pgtable_visit(data, ptep, level);
239+ ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
246240 if (ret)
247241 break;
248242 }
@@ -250,11 +244,10 @@ static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, · 1 unmodified line
250244 return ret;
251245 }
252246
253−static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data)
247+static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)
254248 {
255249 u32 idx;
256250 int ret = 0;
257− struct kvm_pgtable *pgt = data->pgt;
258251 u64 limit = BIT(pgt->ia_bits);
259252
260253 if (data->addr > limit || data->end > limit)
@@ -263,10 +256,10 @@ static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data) · 2 unmodified lines
263256 if (!pgt->pgd)
264257 return -EINVAL;
265258
266− for (idx = kvm_pgd_page_idx(data); data->addr < data->end; ++idx) {
267− kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];
259+ for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {
260+ kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];
268261
269− ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);
262+ ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
270263 if (ret)
271264 break;
272265 }
@@ -278,13 +271,20 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, · 5 unmodified lines
278271 struct kvm_pgtable_walker *walker)
279272 {
280273 struct kvm_pgtable_walk_data walk_data = {
281− .pgt = pgt,
282274 .addr = ALIGN_DOWN(addr, PAGE_SIZE),
283275 .end = PAGE_ALIGN(walk_data.addr + size),
284276 .walker = walker,
285277 };
278+ int r;
286279
287− return _kvm_pgtable_walk(&walk_data);
280+ r = kvm_pgtable_walk_begin(walker);
281+ if (r)
282+ return r;
283+
284+ r = _kvm_pgtable_walk(pgt, &walk_data);
285+ kvm_pgtable_walk_end(walker);
286+
287+ return r;
288288 }
289289
290290 struct leaf_walk_data {
@@ -292,13 +292,13 @@ struct leaf_walk_data { · 1 unmodified line
292292 u32 level;
293293 };
294294
295−static int leaf_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
296− enum kvm_pgtable_walk_flags flag, void * const arg)
295+static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
296+ enum kvm_pgtable_walk_flags visit)
297297 {
298− struct leaf_walk_data *data = arg;
298+ struct leaf_walk_data *data = ctx->arg;
299299
300− data->pte = *ptep;
301− data->level = level;
300+ data->pte = ctx->old;
301+ data->level = ctx->level;
302302
303303 return 0;
304304 }
@@ -329,7 +329,6 @@ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, · 24 unmodified lines
329329 struct hyp_map_data {
330330 u64 phys;
331331 kvm_pte_t attr;
332− struct kvm_pgtable_mm_ops *mm_ops;
333332 };
334333
335334 static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
@@ -383,47 +382,49 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte) · 47 unmodified lines
383382 return prot;
384383 }
385384
386−static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
387− kvm_pte_t *ptep, struct hyp_map_data *data)
385+static bool hyp_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
386+ struct hyp_map_data *data)
388387 {
389− kvm_pte_t new, old = *ptep;
390− u64 granule = kvm_granule_size(level), phys = data->phys;
388+ kvm_pte_t new;
389+ u64 granule = kvm_granule_size(ctx->level), phys = data->phys;
391390
392− if (!kvm_block_mapping_supported(addr, end, phys, level))
391+ if (!kvm_block_mapping_supported(ctx, phys))
393392 return false;
394393
395394 data->phys += granule;
396− new = kvm_init_valid_leaf_pte(phys, data->attr, level);
397− if (old == new)
395+ new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
396+ if (ctx->old == new)
398397 return true;
399− if (!kvm_pte_valid(old))
400− data->mm_ops->get_page(ptep);
401− else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
398+ if (!kvm_pte_valid(ctx->old))
399+ ctx->mm_ops->get_page(ctx->ptep);
400+ else if (WARN_ON((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
402401 return false;
403402
404− smp_store_release(ptep, new);
403+ smp_store_release(ctx->ptep, new);
405404 return true;
406405 }
407406
408−static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
409− enum kvm_pgtable_walk_flags flag, void * const arg)
407+static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
408+ enum kvm_pgtable_walk_flags visit)
410409 {
411− kvm_pte_t *childp;
412− struct hyp_map_data *data = arg;
413− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
410+ kvm_pte_t *childp, new;
411+ struct hyp_map_data *data = ctx->arg;
412+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
414413
415− if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))
414+ if (hyp_map_walker_try_leaf(ctx, data))
416415 return 0;
417416
418− if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
417+ if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1))
419418 return -EINVAL;
420419
421420 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
422421 if (!childp)
423422 return -ENOMEM;
424423
425− kvm_set_table_pte(ptep, childp, mm_ops);
426− mm_ops->get_page(ptep);
424+ new = kvm_init_table_pte(childp, mm_ops);
425+ mm_ops->get_page(ctx->ptep);
426+ smp_store_release(ctx->ptep, new);
427+
427428 return 0;
428429 }
429430
@@ -433,7 +434,6 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, · 3 unmodified lines
433434 int ret;
434435 struct hyp_map_data map_data = {
435436 .phys = ALIGN_DOWN(phys, PAGE_SIZE),
436− .mm_ops = pgt->mm_ops,
437437 };
438438 struct kvm_pgtable_walker walker = {
439439 .cb = hyp_map_walker,
@@ -451,44 +451,39 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, · 11 unmodified lines
451451 return ret;
452452 }
453453
454−struct hyp_unmap_data {
455− u64 unmapped;
456− struct kvm_pgtable_mm_ops *mm_ops;
457−};
458−
459−static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
460− enum kvm_pgtable_walk_flags flag, void * const arg)
454+static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
455+ enum kvm_pgtable_walk_flags visit)
461456 {
462− kvm_pte_t pte = *ptep, *childp = NULL;
463− u64 granule = kvm_granule_size(level);
464− struct hyp_unmap_data *data = arg;
465− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
457+ kvm_pte_t *childp = NULL;
458+ u64 granule = kvm_granule_size(ctx->level);
459+ u64 *unmapped = ctx->arg;
460+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
466461
467− if (!kvm_pte_valid(pte))
462+ if (!kvm_pte_valid(ctx->old))
468463 return -EINVAL;
469464
470− if (kvm_pte_table(pte, level)) {
471− childp = kvm_pte_follow(pte, mm_ops);
465+ if (kvm_pte_table(ctx->old, ctx->level)) {
466+ childp = kvm_pte_follow(ctx->old, mm_ops);
472467
473468 if (mm_ops->page_count(childp) != 1)
474469 return 0;
475470
476− kvm_clear_pte(ptep);
471+ kvm_clear_pte(ctx->ptep);
477472 dsb(ishst);
478− __tlbi_level(vae2is, __TLBI_VADDR(addr, 0), level);
473+ __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
479474 } else {
480− if (end - addr < granule)
475+ if (ctx->end - ctx->addr < granule)
481476 return -EINVAL;
482477
483− kvm_clear_pte(ptep);
478+ kvm_clear_pte(ctx->ptep);
484479 dsb(ishst);
485− __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level);
486− data->unmapped += granule;
480+ __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
481+ *unmapped += granule;
487482 }
488483
489484 dsb(ish);
490485 isb();
491− mm_ops->put_page(ptep);
486+ mm_ops->put_page(ctx->ptep);
492487
493488 if (childp)
494489 mm_ops->put_page(childp);
@@ -498,12 +493,10 @@ static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 3 unmodified lines
498493
499494 u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
500495 {
501− struct hyp_unmap_data unmap_data = {
502− .mm_ops = pgt->mm_ops,
503− };
496+ u64 unmapped = 0;
504497 struct kvm_pgtable_walker walker = {
505498 .cb = hyp_unmap_walker,
506− .arg = &unmap_data,
499+ .arg = &unmapped,
507500 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
508501 };
509502
@@ -511,7 +504,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) · 1 unmodified line
511504 return 0;
512505
513506 kvm_pgtable_walk(pgt, addr, size, &walker);
514− return unmap_data.unmapped;
507+ return unmapped;
515508 }
516509
517510 int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
@@ -519,7 +512,7 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, · 1 unmodified line
519512 {
520513 u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits);
521514
522− pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
515+ pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
523516 if (!pgt->pgd)
524517 return -ENOMEM;
525518
@@ -532,19 +525,18 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, · 6 unmodified lines
532525 return 0;
533526 }
534527
535−static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
536− enum kvm_pgtable_walk_flags flag, void * const arg)
528+static int hyp_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
529+ enum kvm_pgtable_walk_flags visit)
537530 {
538− struct kvm_pgtable_mm_ops *mm_ops = arg;
539− kvm_pte_t pte = *ptep;
531+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
540532
541− if (!kvm_pte_valid(pte))
533+ if (!kvm_pte_valid(ctx->old))
542534 return 0;
543535
544− mm_ops->put_page(ptep);
536+ mm_ops->put_page(ctx->ptep);
545537
546− if (kvm_pte_table(pte, level))
547− mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
538+ if (kvm_pte_table(ctx->old, ctx->level))
539+ mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
548540
549541 return 0;
550542 }
@@ -554,11 +546,10 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) · 3 unmodified lines
554546 struct kvm_pgtable_walker walker = {
555547 .cb = hyp_free_walker,
556548 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
557− .arg = pgt->mm_ops,
558549 };
559550
560551 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
561− pgt->mm_ops->put_page(pgt->pgd);
552+ pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
562553 pgt->pgd = NULL;
563554 }
564555
@@ -573,8 +564,6 @@ struct stage2_map_data { · 8 unmodified lines
573564 struct kvm_s2_mmu *mmu;
574565 void *memcache;
575566
576− struct kvm_pgtable_mm_ops *mm_ops;
577−
578567 /* Force mappings to page granularity */
579568 bool force_pte;
580569 };
@@ -682,19 +671,92 @@ static bool stage2_pte_is_counted(kvm_pte_t pte) · 101 unmodified lines
682671 return !!pte;
683672 }
684673
685−static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,
686− u32 level, struct kvm_pgtable_mm_ops *mm_ops)
674+static bool stage2_pte_is_locked(kvm_pte_t pte)
675+{
676+ return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED);
677+}
678+
679+static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
680+{
681+ if (!kvm_pgtable_walk_shared(ctx)) {
682+ WRITE_ONCE(*ctx->ptep, new);
683+ return true;
684+ }
685+
686+ return cmpxchg(ctx->ptep, ctx->old, new) == ctx->old;
687+}
688+
689+/**
690+ * stage2_try_break_pte() - Invalidates a pte according to the
691+ * 'break-before-make' requirements of the
692+ * architecture.
693+ *
694+ * @ctx: context of the visited pte.
695+ * @mmu: stage-2 mmu
696+ *
697+ * Returns: true if the pte was successfully broken.
698+ *
699+ * If the removed pte was valid, performs the necessary serialization and TLB
700+ * invalidation for the old value. For counted ptes, drops the reference count
701+ * on the containing table page.
702+ */
703+static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
704+ struct kvm_s2_mmu *mmu)
705+{
706+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
707+
708+ if (stage2_pte_is_locked(ctx->old)) {
709+ /*
710+ * Should never occur if this walker has exclusive access to the
711+ * page tables.
712+ */
713+ WARN_ON(!kvm_pgtable_walk_shared(ctx));
714+ return false;
715+ }
716+
717+ if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED))
718+ return false;
719+
720+ /*
721+ * Perform the appropriate TLB invalidation based on the evicted pte
722+ * value (if any).
723+ */
724+ if (kvm_pte_table(ctx->old, ctx->level))
725+ kvm_call_hyp(__kvm_tlb_flush_vmid, mmu);
726+ else if (kvm_pte_valid(ctx->old))
727+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level);
728+
729+ if (stage2_pte_is_counted(ctx->old))
730+ mm_ops->put_page(ctx->ptep);
731+
732+ return true;
733+}
734+
735+static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
736+{
737+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
738+
739+ WARN_ON(!stage2_pte_is_locked(*ctx->ptep));
740+
741+ if (stage2_pte_is_counted(new))
742+ mm_ops->get_page(ctx->ptep);
743+
744+ smp_store_release(ctx->ptep, new);
745+}
746+
747+static void stage2_put_pte(const struct kvm_pgtable_visit_ctx *ctx, struct kvm_s2_mmu *mmu,
748+ struct kvm_pgtable_mm_ops *mm_ops)
687749 {
688750 /*
689751 * Clear the existing PTE, and perform break-before-make with
690752 * TLB maintenance if it was valid.
691753 */
692− if (kvm_pte_valid(*ptep)) {
693− kvm_clear_pte(ptep);
694− kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);
754+ if (kvm_pte_valid(ctx->old)) {
755+ kvm_clear_pte(ctx->ptep);
756+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level);
695757 }
696758
697− mm_ops->put_page(ptep);
759+ mm_ops->put_page(ctx->ptep);
698760 }
699761
700762 static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)
@@ -708,44 +770,42 @@ static bool stage2_pte_executable(kvm_pte_t pte) · 7 unmodified lines
708770 return !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN);
709771 }
710772
711−static bool stage2_leaf_mapping_allowed(u64 addr, u64 end, u32 level,
773+static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx,
712774 struct stage2_map_data *data)
713775 {
714− if (data->force_pte && (level < (KVM_PGTABLE_MAX_LEVELS - 1)))
776+ if (data->force_pte && (ctx->level < (KVM_PGTABLE_MAX_LEVELS - 1)))
715777 return false;
716778
717− return kvm_block_mapping_supported(addr, end, data->phys, level);
779+ return kvm_block_mapping_supported(ctx, data->phys);
718780 }
719781
720−static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
721− kvm_pte_t *ptep,
782+static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
722783 struct stage2_map_data *data)
723784 {
724− kvm_pte_t new, old = *ptep;
725− u64 granule = kvm_granule_size(level), phys = data->phys;
785+ kvm_pte_t new;
786+ u64 granule = kvm_granule_size(ctx->level), phys = data->phys;
726787 struct kvm_pgtable *pgt = data->mmu->pgt;
727− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
788+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
728789
729− if (!stage2_leaf_mapping_allowed(addr, end, level, data))
790+ if (!stage2_leaf_mapping_allowed(ctx, data))
730791 return -E2BIG;
731792
732793 if (kvm_phys_is_valid(phys))
733− new = kvm_init_valid_leaf_pte(phys, data->attr, level);
794+ new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
734795 else
735796 new = kvm_init_invalid_leaf_owner(data->owner_id);
736797
737− if (stage2_pte_is_counted(old)) {
738− /*
739− * Skip updating the PTE if we are trying to recreate the exact
740− * same mapping or only change the access permissions. Instead,
741− * the vCPU will exit one more time from guest if still needed
742− * and then go through the path of relaxing permissions.
743− */
744− if (!stage2_pte_needs_update(old, new))
745− return -EAGAIN;
798+ /*
799+ * Skip updating the PTE if we are trying to recreate the exact
800+ * same mapping or only change the access permissions. Instead,
801+ * the vCPU will exit one more time from guest if still needed
802+ * and then go through the path of relaxing permissions.
803+ */
804+ if (!stage2_pte_needs_update(ctx->old, new))
805+ return -EAGAIN;
746806
747− stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);
748− }
807+ if (!stage2_try_break_pte(ctx, data->mmu))
808+ return -EAGAIN;
749809
750810 /* Perform CMOs before installation of the guest stage-2 PTE */
751811 if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new))
@@ -755,56 +815,43 @@ static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, · 3 unmodified lines
755815 if (mm_ops->icache_inval_pou && stage2_pte_executable(new))
756816 mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);
757817
758− smp_store_release(ptep, new);
759− if (stage2_pte_is_counted(new))
760− mm_ops->get_page(ptep);
818+ stage2_make_pte(ctx, new);
819+
761820 if (kvm_phys_is_valid(phys))
762821 data->phys += granule;
763822 return 0;
764823 }
765824
766−static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,
767− kvm_pte_t *ptep,
825+static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx,
768826 struct stage2_map_data *data)
769827 {
770− if (data->anchor)
771− return 0;
828+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
829+ kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
830+ int ret;
772831
773− if (!stage2_leaf_mapping_allowed(addr, end, level, data))
832+ if (!stage2_leaf_mapping_allowed(ctx, data))
774833 return 0;
775834
776− data->childp = kvm_pte_follow(*ptep, data->mm_ops);
777− kvm_clear_pte(ptep);
835+ ret = stage2_map_walker_try_leaf(ctx, data);
836+ if (ret)
837+ return ret;
778838
779− /*
780− * Invalidate the whole stage-2, as we may have numerous leaf
781− * entries below us which would otherwise need invalidating
782− * individually.
783− */
784− kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu);
785− data->anchor = ptep;
839+ mm_ops->free_removed_table(childp, ctx->level);
786840 return 0;
787841 }
788842
789−static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
843+static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx,
790844 struct stage2_map_data *data)
791845 {
792− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
793− kvm_pte_t *childp, pte = *ptep;
846+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
847+ kvm_pte_t *childp, new;
794848 int ret;
795849
796− if (data->anchor) {
797− if (stage2_pte_is_counted(pte))
798− mm_ops->put_page(ptep);
799−
800− return 0;
801− }
802−
803− ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);
850+ ret = stage2_map_walker_try_leaf(ctx, data);
804851 if (ret != -E2BIG)
805852 return ret;
806853
807− if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
854+ if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1))
808855 return -EINVAL;
809856
810857 if (!data->memcache)
@@ -814,99 +861,62 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 3 unmodified lines
814861 if (!childp)
815862 return -ENOMEM;
816863
864+ if (!stage2_try_break_pte(ctx, data->mmu)) {
865+ mm_ops->put_page(childp);
866+ return -EAGAIN;
867+ }
868+
817869 /*
818870 * If we've run into an existing block mapping then replace it with
819871 * a table. Accesses beyond 'end' that fall within the new table
820872 * will be mapped lazily.
821873 */
822− if (stage2_pte_is_counted(pte))
823− stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);
824−
825− kvm_set_table_pte(ptep, childp, mm_ops);
826− mm_ops->get_page(ptep);
874+ new = kvm_init_table_pte(childp, mm_ops);
875+ stage2_make_pte(ctx, new);
827876
828877 return 0;
829878 }
830879
831−static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level,
832− kvm_pte_t *ptep,
833− struct stage2_map_data *data)
834−{
835− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
836− kvm_pte_t *childp;
837− int ret = 0;
838−
839− if (!data->anchor)
840− return 0;
841−
842− if (data->anchor == ptep) {
843− childp = data->childp;
844− data->anchor = NULL;
845− data->childp = NULL;
846− ret = stage2_map_walk_leaf(addr, end, level, ptep, data);
847− } else {
848− childp = kvm_pte_follow(*ptep, mm_ops);
849− }
850−
851− mm_ops->put_page(childp);
852− mm_ops->put_page(ptep);
853−
854− return ret;
855−}
856−
857880 /*
858− * This is a little fiddly, as we use all three of the walk flags. The idea
859− * is that the TABLE_PRE callback runs for table entries on the way down,
860− * looking for table entries which we could conceivably replace with a
861− * block entry for this mapping. If it finds one, then it sets the 'anchor'
862− * field in 'struct stage2_map_data' to point at the table entry, before
863− * clearing the entry to zero and descending into the now detached table.
881+ * The TABLE_PRE callback runs for table entries on the way down, looking
882+ * for table entries which we could conceivably replace with a block entry
883+ * for this mapping. If it finds one it replaces the entry and calls
884+ * kvm_pgtable_mm_ops::free_removed_table() to tear down the detached table.
864885 *
865− * The behaviour of the LEAF callback then depends on whether or not the
866− * anchor has been set. If not, then we're not using a block mapping higher
867− * up the table and we perform the mapping at the existing leaves instead.
868− * If, on the other hand, the anchor _is_ set, then we drop references to
869− * all valid leaves so that the pages beneath the anchor can be freed.
870− *
871− * Finally, the TABLE_POST callback does nothing if the anchor has not
872− * been set, but otherwise frees the page-table pages while walking back up
873− * the page-table, installing the block entry when it revisits the anchor
874− * pointer and clearing the anchor to NULL.
886+ * Otherwise, the LEAF callback performs the mapping at the existing leaves
887+ * instead.
875888 */
876−static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
877− enum kvm_pgtable_walk_flags flag, void * const arg)
889+static int stage2_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
890+ enum kvm_pgtable_walk_flags visit)
878891 {
879− struct stage2_map_data *data = arg;
892+ struct stage2_map_data *data = ctx->arg;
880893
881− switch (flag) {
894+ switch (visit) {
882895 case KVM_PGTABLE_WALK_TABLE_PRE:
883− return stage2_map_walk_table_pre(addr, end, level, ptep, data);
896+ return stage2_map_walk_table_pre(ctx, data);
884897 case KVM_PGTABLE_WALK_LEAF:
885− return stage2_map_walk_leaf(addr, end, level, ptep, data);
886− case KVM_PGTABLE_WALK_TABLE_POST:
887− return stage2_map_walk_table_post(addr, end, level, ptep, data);
898+ return stage2_map_walk_leaf(ctx, data);
899+ default:
900+ return -EINVAL;
888901 }
889−
890− return -EINVAL;
891902 }
892903
893904 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
894905 u64 phys, enum kvm_pgtable_prot prot,
895− void *mc)
906+ void *mc, enum kvm_pgtable_walk_flags flags)
896907 {
897908 int ret;
898909 struct stage2_map_data map_data = {
899910 .phys = ALIGN_DOWN(phys, PAGE_SIZE),
900911 .mmu = pgt->mmu,
901912 .memcache = mc,
902− .mm_ops = pgt->mm_ops,
903913 .force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),
904914 };
905915 struct kvm_pgtable_walker walker = {
906916 .cb = stage2_map_walker,
907− .flags = KVM_PGTABLE_WALK_TABLE_PRE |
908− KVM_PGTABLE_WALK_LEAF |
909− KVM_PGTABLE_WALK_TABLE_POST,
917+ .flags = flags |
918+ KVM_PGTABLE_WALK_TABLE_PRE |
919+ KVM_PGTABLE_WALK_LEAF,
910920 .arg = &map_data,
911921 };
912922
@@ -930,15 +940,13 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, · 17 unmodified lines
930940 .phys = KVM_PHYS_INVALID,
931941 .mmu = pgt->mmu,
932942 .memcache = mc,
933− .mm_ops = pgt->mm_ops,
934943 .owner_id = owner_id,
935944 .force_pte = true,
936945 };
937946 struct kvm_pgtable_walker walker = {
938947 .cb = stage2_map_walker,
939948 .flags = KVM_PGTABLE_WALK_TABLE_PRE |
940− KVM_PGTABLE_WALK_LEAF |
941− KVM_PGTABLE_WALK_TABLE_POST,
949+ KVM_PGTABLE_WALK_LEAF,
942950 .arg = &map_data,
943951 };
944952
@@ -949,30 +957,29 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, · 4 unmodified lines
949957 return ret;
950958 }
951959
952−static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
953− enum kvm_pgtable_walk_flags flag,
954− void * const arg)
960+static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
961+ enum kvm_pgtable_walk_flags visit)
955962 {
956− struct kvm_pgtable *pgt = arg;
963+ struct kvm_pgtable *pgt = ctx->arg;
957964 struct kvm_s2_mmu *mmu = pgt->mmu;
958− struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
959− kvm_pte_t pte = *ptep, *childp = NULL;
965+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
966+ kvm_pte_t *childp = NULL;
960967 bool need_flush = false;
961968
962− if (!kvm_pte_valid(pte)) {
963− if (stage2_pte_is_counted(pte)) {
964− kvm_clear_pte(ptep);
965− mm_ops->put_page(ptep);
969+ if (!kvm_pte_valid(ctx->old)) {
970+ if (stage2_pte_is_counted(ctx->old)) {
971+ kvm_clear_pte(ctx->ptep);
972+ mm_ops->put_page(ctx->ptep);
966973 }
967974 return 0;
968975 }
969976
970− if (kvm_pte_table(pte, level)) {
971− childp = kvm_pte_follow(pte, mm_ops);
977+ if (kvm_pte_table(ctx->old, ctx->level)) {
978+ childp = kvm_pte_follow(ctx->old, mm_ops);
972979
973980 if (mm_ops->page_count(childp) != 1)
974981 return 0;
975− } else if (stage2_pte_cacheable(pgt, pte)) {
982+ } else if (stage2_pte_cacheable(pgt, ctx->old)) {
976983 need_flush = !stage2_has_fwb(pgt);
977984 }
978985
@@ -981,11 +988,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 2 unmodified lines
981988 * block entry and rely on the remaining portions being faulted
982989 * back lazily.
983990 */
984− stage2_put_pte(ptep, mmu, addr, level, mm_ops);
991+ stage2_put_pte(ctx, mmu, mm_ops);
985992
986993 if (need_flush && mm_ops->dcache_clean_inval_poc)
987− mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
988− kvm_granule_size(level));
994+ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
995+ kvm_granule_size(ctx->level));
989996
990997 if (childp)
991998 mm_ops->put_page(childp);
@@ -1009,21 +1016,19 @@ struct stage2_attr_data { · 17 unmodified lines
10091016 kvm_pte_t attr_clr;
10101017 kvm_pte_t pte;
10111018 u32 level;
1012− struct kvm_pgtable_mm_ops *mm_ops;
10131019 };
10141020
1015−static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
1016− enum kvm_pgtable_walk_flags flag,
1017− void * const arg)
1021+static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx,
1022+ enum kvm_pgtable_walk_flags visit)
10181023 {
1019− kvm_pte_t pte = *ptep;
1020− struct stage2_attr_data *data = arg;
1021− struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
1024+ kvm_pte_t pte = ctx->old;
1025+ struct stage2_attr_data *data = ctx->arg;
1026+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
10221027
1023− if (!kvm_pte_valid(pte))
1028+ if (!kvm_pte_valid(ctx->old))
10241029 return 0;
10251030
1026− data->level = level;
1031+ data->level = ctx->level;
10271032 data->pte = pte;
10281033 pte &= ~data->attr_clr;
10291034 pte |= data->attr_set;
@@ -1039,10 +1044,12 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 9 unmodified lines
10391044 * stage-2 PTE if we are going to add executable permission.
10401045 */
10411046 if (mm_ops->icache_inval_pou &&
1042− stage2_pte_executable(pte) && !stage2_pte_executable(*ptep))
1047+ stage2_pte_executable(pte) && !stage2_pte_executable(ctx->old))
10431048 mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
1044− kvm_granule_size(level));
1045− WRITE_ONCE(*ptep, pte);
1049+ kvm_granule_size(ctx->level));
1050+
1051+ if (!stage2_try_set_pte(ctx, pte))
1052+ return -EAGAIN;
10461053 }
10471054
10481055 return 0;
@@ -1051,19 +1058,18 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 2 unmodified lines
10511058 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,
10521059 u64 size, kvm_pte_t attr_set,
10531060 kvm_pte_t attr_clr, kvm_pte_t *orig_pte,
1054− u32 *level)
1061+ u32 *level, enum kvm_pgtable_walk_flags flags)
10551062 {
10561063 int ret;
10571064 kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;
10581065 struct stage2_attr_data data = {
10591066 .attr_set = attr_set & attr_mask,
10601067 .attr_clr = attr_clr & attr_mask,
1061− .mm_ops = pgt->mm_ops,
10621068 };
10631069 struct kvm_pgtable_walker walker = {
10641070 .cb = stage2_attr_walker,
10651071 .arg = &data,
1066− .flags = KVM_PGTABLE_WALK_LEAF,
1072+ .flags = flags | KVM_PGTABLE_WALK_LEAF,
10671073 };
10681074
10691075 ret = kvm_pgtable_walk(pgt, addr, size, &walker);
@@ -1082,14 +1088,14 @@ int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) · 12 unmodified lines
10821088 {
10831089 return stage2_update_leaf_attrs(pgt, addr, size, 0,
10841090 KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
1085− NULL, NULL);
1091+ NULL, NULL, 0);
10861092 }
10871093
10881094 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)
10891095 {
10901096 kvm_pte_t pte = 0;
10911097 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,
1092− &pte, NULL);
1098+ &pte, NULL, 0);
10931099 dsb(ishst);
10941100 return pte;
10951101 }
@@ -1098,7 +1104,7 @@ kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) · 2 unmodified lines
10981104 {
10991105 kvm_pte_t pte = 0;
11001106 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF,
1101− &pte, NULL);
1107+ &pte, NULL, 0);
11021108 /*
11031109 * "But where's the TLBI?!", you scream.
11041110 * "Over in the core code", I sigh.
@@ -1111,7 +1117,7 @@ kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) · 6 unmodified lines
11111117 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr)
11121118 {
11131119 kvm_pte_t pte = 0;
1114− stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL);
1120+ stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL, 0);
11151121 return pte & KVM_PTE_LEAF_ATTR_LO_S2_AF;
11161122 }
11171123
@@ -1134,26 +1140,25 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, · 16 unmodified lines
11341140 if (prot & KVM_PGTABLE_PROT_X)
11351141 clr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;
11361142
1137− ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);
1143+ ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level,
1144+ KVM_PGTABLE_WALK_SHARED);
11381145 if (!ret)
11391146 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);
11401147 return ret;
11411148 }
11421149
1143−static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
1144− enum kvm_pgtable_walk_flags flag,
1145− void * const arg)
1150+static int stage2_flush_walker(const struct kvm_pgtable_visit_ctx *ctx,
1151+ enum kvm_pgtable_walk_flags visit)
11461152 {
1147− struct kvm_pgtable *pgt = arg;
1153+ struct kvm_pgtable *pgt = ctx->arg;
11481154 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1149− kvm_pte_t pte = *ptep;
11501155
1151− if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))
1156+ if (!kvm_pte_valid(ctx->old) || !stage2_pte_cacheable(pgt, ctx->old))
11521157 return 0;
11531158
11541159 if (mm_ops->dcache_clean_inval_poc)
1155− mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),
1156− kvm_granule_size(level));
1160+ mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1161+ kvm_granule_size(ctx->level));
11571162 return 0;
11581163 }
11591164
@@ -1184,7 +1189,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 24 unmodified lines
11841189 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
11851190
11861191 pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1187− pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz);
1192+ pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
11881193 if (!pgt->pgd)
11891194 return -ENOMEM;
11901195
@@ -1200,20 +1205,27 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 9 unmodified lines
12001205 return 0;
12011206 }
12021207
1203−static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
1204− enum kvm_pgtable_walk_flags flag,
1205− void * const arg)
1208+size_t kvm_pgtable_stage2_pgd_size(u64 vtcr)
1209+{
1210+ u32 ia_bits = VTCR_EL2_IPA(vtcr);
1211+ u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
1212+ u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
1213+
1214+ return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1215+}
1216+
1217+static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
1218+ enum kvm_pgtable_walk_flags visit)
12061219 {
1207− struct kvm_pgtable_mm_ops *mm_ops = arg;
1208− kvm_pte_t pte = *ptep;
1220+ struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
12091221
1210− if (!stage2_pte_is_counted(pte))
1222+ if (!stage2_pte_is_counted(ctx->old))
12111223 return 0;
12121224
1213− mm_ops->put_page(ptep);
1225+ mm_ops->put_page(ctx->ptep);
12141226
1215− if (kvm_pte_table(pte, level))
1216− mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
1227+ if (kvm_pte_table(ctx->old, ctx->level))
1228+ mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
12171229
12181230 return 0;
12191231 }
@@ -1225,11 +1237,33 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) · 5 unmodified lines
12251237 .cb = stage2_free_walker,
12261238 .flags = KVM_PGTABLE_WALK_LEAF |
12271239 KVM_PGTABLE_WALK_TABLE_POST,
1228− .arg = pgt->mm_ops,
12291240 };
12301241
12311242 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
12321243 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
1233− pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz);
1244+ pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);
12341245 pgt->pgd = NULL;
12351246 }
1247+
1248+void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level)
1249+{
1250+ kvm_pteref_t ptep = (kvm_pteref_t)pgtable;
1251+ struct kvm_pgtable_walker walker = {
1252+ .cb = stage2_free_walker,
1253+ .flags = KVM_PGTABLE_WALK_LEAF |
1254+ KVM_PGTABLE_WALK_TABLE_POST,
1255+ };
1256+ struct kvm_pgtable_walk_data data = {
1257+ .walker = &walker,
1258+
1259+ /*
1260+ * At this point the IPA really doesn't matter, as the page
1261+ * table being traversed has already been removed from the stage
1262+ * 2. Set an appropriate range to cover the entire page table.
1263+ */
1264+ .addr = 0,
1265+ .end = kvm_granule_size(level),
1266+ };
1267+
1268+ WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));
1269+}
arch/arm64/kvm/hyp/vhe/Makefile+1−1
@@ -1,6 +1,6 @@
11 # SPDX-License-Identifier: GPL-2.0
22 #
3−# Makefile for Kernel-based Virtual Machine module, HYP/nVHE part
3+# Makefile for Kernel-based Virtual Machine module, HYP/VHE part
44 #
55
66 asflags-y := -D__KVM_VHE_HYPERVISOR__
arch/arm64/kvm/mmu.c+132−61
@@ -128,6 +128,25 @@ static void kvm_s2_free_pages_exact(void *virt, size_t size) · 127 unmodified lines
128128 free_pages_exact(virt, size);
129129 }
130130
131+static struct kvm_pgtable_mm_ops kvm_s2_mm_ops;
132+
133+static void stage2_free_removed_table_rcu_cb(struct rcu_head *head)
134+{
135+ struct page *page = container_of(head, struct page, rcu_head);
136+ void *pgtable = page_to_virt(page);
137+ u32 level = page_private(page);
138+
139+ kvm_pgtable_stage2_free_removed(&kvm_s2_mm_ops, pgtable, level);
140+}
141+
142+static void stage2_free_removed_table(void *addr, u32 level)
143+{
144+ struct page *page = virt_to_page(addr);
145+
146+ set_page_private(page, (unsigned long)level);
147+ call_rcu(&page->rcu_head, stage2_free_removed_table_rcu_cb);
148+}
149+
131150 static void kvm_host_get_page(void *addr)
132151 {
133152 get_page(virt_to_page(addr));
@@ -640,8 +659,8 @@ static struct kvm_pgtable_mm_ops kvm_user_mm_ops = { · 506 unmodified lines
640659 static int get_user_mapping_size(struct kvm *kvm, u64 addr)
641660 {
642661 struct kvm_pgtable pgt = {
643− .pgd = (kvm_pte_t *)kvm->mm->pgd,
644− .ia_bits = VA_BITS,
662+ .pgd = (kvm_pteref_t)kvm->mm->pgd,
663+ .ia_bits = vabits_actual,
645664 .start_level = (KVM_PGTABLE_MAX_LEVELS -
646665 CONFIG_PGTABLE_LEVELS),
647666 .mm_ops = &kvm_user_mm_ops,
@@ -662,6 +681,7 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { · 14 unmodified lines
662681 .zalloc_page = stage2_memcache_zalloc_page,
663682 .zalloc_pages_exact = kvm_s2_zalloc_pages_exact,
664683 .free_pages_exact = kvm_s2_free_pages_exact,
684+ .free_removed_table = stage2_free_removed_table,
665685 .get_page = kvm_host_get_page,
666686 .put_page = kvm_s2_put_page,
667687 .page_count = kvm_host_page_count,
@@ -675,15 +695,42 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { · 7 unmodified lines
675695 * kvm_init_stage2_mmu - Initialise a S2 MMU structure
676696 * @kvm: The pointer to the KVM structure
677697 * @mmu: The pointer to the s2 MMU structure
698+ * @type: The machine type of the virtual machine
678699 *
679700 * Allocates only the stage-2 HW PGD level table(s).
680701 * Note we don't need locking here as this is only called when the VM is
681702 * created, which can only be done once.
682703 */
683−int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
704+int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type)
684705 {
706+ u32 kvm_ipa_limit = get_kvm_ipa_limit();
685707 int cpu, err;
686708 struct kvm_pgtable *pgt;
709+ u64 mmfr0, mmfr1;
710+ u32 phys_shift;
711+
712+ if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
713+ return -EINVAL;
714+
715+ phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
716+ if (is_protected_kvm_enabled()) {
717+ phys_shift = kvm_ipa_limit;
718+ } else if (phys_shift) {
719+ if (phys_shift > kvm_ipa_limit ||
720+ phys_shift < ARM64_MIN_PARANGE_BITS)
721+ return -EINVAL;
722+ } else {
723+ phys_shift = KVM_PHYS_SHIFT;
724+ if (phys_shift > kvm_ipa_limit) {
725+ pr_warn_once("%s using unsupported default IPA limit, upgrade your VMM\n",
726+ current->comm);
727+ return -EINVAL;
728+ }
729+ }
730+
731+ mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
732+ mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
733+ kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
687734
688735 if (mmu->pgt != NULL) {
689736 kvm_err("kvm_arch already initialized?\n");
@@ -807,6 +854,32 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) · 117 unmodified lines
807854 }
808855 }
809856
857+static void hyp_mc_free_fn(void *addr, void *unused)
858+{
859+ free_page((unsigned long)addr);
860+}
861+
862+static void *hyp_mc_alloc_fn(void *unused)
863+{
864+ return (void *)__get_free_page(GFP_KERNEL_ACCOUNT);
865+}
866+
867+void free_hyp_memcache(struct kvm_hyp_memcache *mc)
868+{
869+ if (is_protected_kvm_enabled())
870+ __free_hyp_memcache(mc, hyp_mc_free_fn,
871+ kvm_host_va, NULL);
872+}
873+
874+int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages)
875+{
876+ if (!is_protected_kvm_enabled())
877+ return 0;
878+
879+ return __topup_hyp_memcache(mc, min_pages, hyp_mc_alloc_fn,
880+ kvm_host_pa, NULL);
881+}
882+
810883 /**
811884 * kvm_phys_addr_ioremap - map a device range to guest IPA
812885 *
@@ -841,7 +914,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, · 28 unmodified lines
841914
842915 write_lock(&kvm->mmu_lock);
843916 ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,
844− &cache);
917+ &cache, 0);
845918 write_unlock(&kvm->mmu_lock);
846919 if (ret)
847920 break;
@@ -1091,32 +1164,26 @@ static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva) · 243 unmodified lines
10911164 * - mmap_lock protects between a VM faulting a page in and the VMM performing
10921165 * an mprotect() to add VM_MTE
10931166 */
1094−static int sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
1095− unsigned long size)
1167+static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
1168+ unsigned long size)
10961169 {
10971170 unsigned long i, nr_pages = size >> PAGE_SHIFT;
1098− struct page *page;
1171+ struct page *page = pfn_to_page(pfn);
10991172
11001173 if (!kvm_has_mte(kvm))
1101− return 0;
1102−
1103− /*
1104− * pfn_to_online_page() is used to reject ZONE_DEVICE pages
1105− * that may not support tags.
1106− */
1107− page = pfn_to_online_page(pfn);
1108−
1109− if (!page)
1110− return -EFAULT;
1174+ return;
11111175
11121176 for (i = 0; i < nr_pages; i++, page++) {
1113− if (!test_bit(PG_mte_tagged, &page->flags)) {
1177+ if (try_page_mte_tagging(page)) {
11141178 mte_clear_page_tags(page_address(page));
1115− set_bit(PG_mte_tagged, &page->flags);
1179+ set_page_mte_tagged(page);
11161180 }
11171181 }
1182+}
11181183
1119− return 0;
1184+static bool kvm_vma_mte_allowed(struct vm_area_struct *vma)
1185+{
1186+ return vma->vm_flags & VM_MTE_ALLOWED;
11201187 }
11211188
11221189 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
@@ -1127,7 +1194,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 4 unmodified lines
11271194 bool write_fault, writable, force_pte = false;
11281195 bool exec_fault;
11291196 bool device = false;
1130− bool shared;
11311197 unsigned long mmu_seq;
11321198 struct kvm *kvm = vcpu->kvm;
11331199 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
@@ -1136,7 +1202,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 2 unmodified lines
11361202 gfn_t gfn;
11371203 kvm_pfn_t pfn;
11381204 bool logging_active = memslot_is_logging(memslot);
1139− bool use_read_lock = false;
11401205 unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
11411206 unsigned long vma_pagesize, fault_granule;
11421207 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
@@ -1171,14 +1236,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 28 unmodified lines
11711236 if (logging_active) {
11721237 force_pte = true;
11731238 vma_shift = PAGE_SHIFT;
1174− use_read_lock = (fault_status == FSC_PERM && write_fault &&
1175− fault_granule == PAGE_SIZE);
11761239 } else {
11771240 vma_shift = get_vma_page_shift(vma, hva);
11781241 }
11791242
1180− shared = (vma->vm_flags & VM_SHARED);
1181−
11821243 switch (vma_shift) {
11831244 #ifndef __PAGETABLE_PMD_FOLDED
11841245 case PUD_SHIFT:
@@ -1271,15 +1332,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 86 unmodified lines
12711332 if (exec_fault && device)
12721333 return -ENOEXEC;
12731334
1274− /*
1275− * To reduce MMU contentions and enhance concurrency during dirty
1276− * logging dirty logging, only acquire read lock for permission
1277− * relaxation.
1278− */
1279− if (use_read_lock)
1280− read_lock(&kvm->mmu_lock);
1281− else
1282− write_lock(&kvm->mmu_lock);
1335+ read_lock(&kvm->mmu_lock);
12831336 pgt = vcpu->arch.hw_mmu->pgt;
12841337 if (mmu_invalidate_retry(kvm, mmu_seq))
12851338 goto out_unlock;
@@ -1298,13 +1351,13 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 12 unmodified lines
12981351 }
12991352
13001353 if (fault_status != FSC_PERM && !device && kvm_has_mte(kvm)) {
1301− /* Check the VMM hasn't introduced a new VM_SHARED VMA */
1302− if (!shared)
1303− ret = sanitise_mte_tags(kvm, pfn, vma_pagesize);
1304− else
1354+ /* Check the VMM hasn't introduced a new disallowed VMA */
1355+ if (kvm_vma_mte_allowed(vma)) {
1356+ sanitise_mte_tags(kvm, pfn, vma_pagesize);
1357+ } else {
13051358 ret = -EFAULT;
1306− if (ret)
13071359 goto out_unlock;
1360+ }
13081361 }
13091362
13101363 if (writable)
@@ -1323,15 +1376,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 12 unmodified lines
13231376 * permissions only if vma_pagesize equals fault_granule. Otherwise,
13241377 * kvm_pgtable_stage2_map() should be called to change block size.
13251378 */
1326− if (fault_status == FSC_PERM && vma_pagesize == fault_granule) {
1379+ if (fault_status == FSC_PERM && vma_pagesize == fault_granule)
13271380 ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
1328− } else {
1329− WARN_ONCE(use_read_lock, "Attempted stage-2 map outside of write lock\n");
1330−
1381+ else
13311382 ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
13321383 __pfn_to_phys(pfn), prot,
1333− memcache);
1334− }
1384+ memcache, KVM_PGTABLE_WALK_SHARED);
13351385
13361386 /* Mark the page dirty only if the fault is handled successfully */
13371387 if (writable && !ret) {
@@ -1340,10 +1390,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 2 unmodified lines
13401390 }
13411391
13421392 out_unlock:
1343− if (use_read_lock)
1344− read_unlock(&kvm->mmu_lock);
1345− else
1346− write_unlock(&kvm->mmu_lock);
1393+ read_unlock(&kvm->mmu_lock);
13471394 kvm_set_pfn_accessed(pfn);
13481395 kvm_release_pfn_clean(pfn);
13491396 return ret != -EAGAIN ? ret : 0;
@@ -1526,15 +1573,18 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) · 176 unmodified lines
15261573 bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
15271574 {
15281575 kvm_pfn_t pfn = pte_pfn(range->pte);
1529− int ret;
15301576
15311577 if (!kvm->arch.mmu.pgt)
15321578 return false;
15331579
15341580 WARN_ON(range->end - range->start != 1);
15351581
1536− ret = sanitise_mte_tags(kvm, pfn, PAGE_SIZE);
1537− if (ret)
1582+ /*
1583+ * If the page isn't tagged, defer to user_mem_abort() for sanitising
1584+ * the MTE tags. The S2 pte should have been unmapped by
1585+ * mmu_notifier_invalidate_range_end().
1586+ */
1587+ if (kvm_has_mte(kvm) && !page_mte_tagged(pfn_to_page(pfn)))
15381588 return false;
15391589
15401590 /*
@@ -1549,7 +1599,7 @@ bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) · 8 unmodified lines
15491599 */
15501600 kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT,
15511601 PAGE_SIZE, __pfn_to_phys(pfn),
1552− KVM_PGTABLE_PROT_R, NULL);
1602+ KVM_PGTABLE_PROT_R, NULL, 0);
15531603
15541604 return false;
15551605 }
@@ -1618,6 +1668,8 @@ static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = { · 62 unmodified lines
16181668 int kvm_mmu_init(u32 *hyp_va_bits)
16191669 {
16201670 int err;
1671+ u32 idmap_bits;
1672+ u32 kernel_bits;
16211673
16221674 hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
16231675 hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
@@ -1631,7 +1683,31 @@ int kvm_mmu_init(u32 *hyp_va_bits) · 7 unmodified lines
16311683 */
16321684 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
16331685
1634− *hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
1686+ /*
1687+ * The ID map may be configured to use an extended virtual address
1688+ * range. This is only the case if system RAM is out of range for the
1689+ * currently configured page size and VA_BITS_MIN, in which case we will
1690+ * also need the extended virtual range for the HYP ID map, or we won't
1691+ * be able to enable the EL2 MMU.
1692+ *
1693+ * However, in some cases the ID map may be configured for fewer than
1694+ * the number of VA bits used by the regular kernel stage 1. This
1695+ * happens when VA_BITS=52 and the kernel image is placed in PA space
1696+ * below 48 bits.
1697+ *
1698+ * At EL2, there is only one TTBR register, and we can't switch between
1699+ * translation tables *and* update TCR_EL2.T0SZ at the same time. Bottom
1700+ * line: we need to use the extended range with *both* our translation
1701+ * tables.
1702+ *
1703+ * So use the maximum of the idmap VA bits and the regular kernel stage
1704+ * 1 VA bits to assure that the hypervisor can both ID map its code page
1705+ * and map any kernel memory.
1706+ */
1707+ idmap_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
1708+ kernel_bits = vabits_actual;
1709+ *hyp_va_bits = max(idmap_bits, kernel_bits);
1710+
16351711 kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);
16361712 kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
16371713 kvm_debug("HYP VA range: %lx:%lx\n",
@@ -1740,12 +1816,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, · 102 unmodified lines
17401816 if (!vma)
17411817 break;
17421818
1743− /*
1744− * VM_SHARED mappings are not allowed with MTE to avoid races
1745− * when updating the PG_mte_tagged page flag, see
1746− * sanitise_mte_tags for more details.
1747− */
1748− if (kvm_has_mte(kvm) && vma->vm_flags & VM_SHARED) {
1819+ if (kvm_has_mte(kvm) && !kvm_vma_mte_allowed(vma)) {
17491820 ret = -EINVAL;
17501821 break;
17511822 }
arch/arm64/kvm/pkvm.c+122−16
@@ -6,6 +6,7 @@ · 5 unmodified lines
66
77 #include <linux/kvm_host.h>
88 #include <linux/memblock.h>
9+#include <linux/mutex.h>
910 #include <linux/sort.h>
1011
1112 #include <asm/kvm_pkvm.h>
@@ -53,7 +54,7 @@ static int __init register_memblock_regions(void) · 41 unmodified lines
5354
5455 void __init kvm_hyp_reserve(void)
5556 {
56− u64 nr_pages, prev, hyp_mem_pages = 0;
57+ u64 hyp_mem_pages = 0;
5758 int ret;
5859
5960 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
@@ -71,21 +72,8 @@ void __init kvm_hyp_reserve(void) · 11 unmodified lines
7172
7273 hyp_mem_pages += hyp_s1_pgtable_pages();
7374 hyp_mem_pages += host_s2_pgtable_pages();
74−
75− /*
76− * The hyp_vmemmap needs to be backed by pages, but these pages
77− * themselves need to be present in the vmemmap, so compute the number
78− * of pages needed by looking for a fixed point.
79− */
80− nr_pages = 0;
81− do {
82− prev = nr_pages;
83− nr_pages = hyp_mem_pages + prev;
84− nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE,
85− PAGE_SIZE);
86− nr_pages += __hyp_pgtable_max_pages(nr_pages);
87− } while (nr_pages != prev);
88− hyp_mem_pages += nr_pages;
75+ hyp_mem_pages += hyp_vm_table_pages();
76+ hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
8977
9078 /*
9179 * Try to allocate a PMD-aligned region to reduce TLB pressure once
@@ -107,3 +95,121 @@ void __init kvm_hyp_reserve(void) · 15 unmodified lines
10795 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
10896 hyp_mem_base);
10997 }
98+
99+/*
100+ * Allocates and donates memory for hypervisor VM structs at EL2.
101+ *
102+ * Allocates space for the VM state, which includes the hyp vm as well as
103+ * the hyp vcpus.
104+ *
105+ * Stores an opaque handler in the kvm struct for future reference.
106+ *
107+ * Return 0 on success, negative error code on failure.
108+ */
109+static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
110+{
111+ size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz;
112+ struct kvm_vcpu *host_vcpu;
113+ pkvm_handle_t handle;
114+ void *pgd, *hyp_vm;
115+ unsigned long idx;
116+ int ret;
117+
118+ if (host_kvm->created_vcpus < 1)
119+ return -EINVAL;
120+
121+ pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.vtcr);
122+
123+ /*
124+ * The PGD pages will be reclaimed using a hyp_memcache which implies
125+ * page granularity. So, use alloc_pages_exact() to get individual
126+ * refcounts.
127+ */
128+ pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
129+ if (!pgd)
130+ return -ENOMEM;
131+
132+ /* Allocate memory to donate to hyp for vm and vcpu pointers. */
133+ hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
134+ size_mul(sizeof(void *),
135+ host_kvm->created_vcpus)));
136+ hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
137+ if (!hyp_vm) {
138+ ret = -ENOMEM;
139+ goto free_pgd;
140+ }
141+
142+ /* Donate the VM memory to hyp and let hyp initialize it. */
143+ ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd);
144+ if (ret < 0)
145+ goto free_vm;
146+
147+ handle = ret;
148+
149+ host_kvm->arch.pkvm.handle = handle;
150+
151+ /* Donate memory for the vcpus at hyp and initialize it. */
152+ hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
153+ kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
154+ void *hyp_vcpu;
155+
156+ /* Indexing of the vcpus to be sequential starting at 0. */
157+ if (WARN_ON(host_vcpu->vcpu_idx != idx)) {
158+ ret = -EINVAL;
159+ goto destroy_vm;
160+ }
161+
162+ hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
163+ if (!hyp_vcpu) {
164+ ret = -ENOMEM;
165+ goto destroy_vm;
166+ }
167+
168+ ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu,
169+ hyp_vcpu);
170+ if (ret) {
171+ free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
172+ goto destroy_vm;
173+ }
174+ }
175+
176+ return 0;
177+
178+destroy_vm:
179+ pkvm_destroy_hyp_vm(host_kvm);
180+ return ret;
181+free_vm:
182+ free_pages_exact(hyp_vm, hyp_vm_sz);
183+free_pgd:
184+ free_pages_exact(pgd, pgd_sz);
185+ return ret;
186+}
187+
188+int pkvm_create_hyp_vm(struct kvm *host_kvm)
189+{
190+ int ret = 0;
191+
192+ mutex_lock(&host_kvm->lock);
193+ if (!host_kvm->arch.pkvm.handle)
194+ ret = __pkvm_create_hyp_vm(host_kvm);
195+ mutex_unlock(&host_kvm->lock);
196+
197+ return ret;
198+}
199+
200+void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
201+{
202+ if (host_kvm->arch.pkvm.handle) {
203+ WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm,
204+ host_kvm->arch.pkvm.handle));
205+ }
206+
207+ host_kvm->arch.pkvm.handle = 0;
208+ free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
209+}
210+
211+int pkvm_init_host_vm(struct kvm *host_kvm)
212+{
213+ mutex_init(&host_kvm->lock);
214+ return 0;
215+}
arch/arm64/kvm/pmu-emul.c+196−286
@@ -15,16 +15,25 @@ · 14 unmodified lines
1515 #include <kvm/arm_pmu.h>
1616 #include <kvm/arm_vgic.h>
1717
18+#define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0)
19+
1820 DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
1921
2022 static LIST_HEAD(arm_pmus);
2123 static DEFINE_MUTEX(arm_pmus_lock);
2224
23−static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);
24−static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx);
25−static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc);
25+static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc);
26+static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc);
27+
28+static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc)
29+{
30+ return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]);
31+}
2632
27−#define PERF_ATTR_CFG1_KVM_PMU_CHAINED 0x1
33+static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx)
34+{
35+ return &vcpu->arch.pmu.pmc[cnt_idx];
36+}
2837
2938 static u32 kvm_pmu_event_mask(struct kvm *kvm)
3039 {
@@ -47,113 +56,46 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) · 16 unmodified lines
4756 }
4857
4958 /**
50− * kvm_pmu_idx_is_64bit - determine if select_idx is a 64bit counter
51− * @vcpu: The vcpu pointer
52− * @select_idx: The counter index
59+ * kvm_pmc_is_64bit - determine if counter is 64bit
60+ * @pmc: counter context
5361 */
54−static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)
62+static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc)
5563 {
56− return (select_idx == ARMV8_PMU_CYCLE_IDX &&
57− __vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);
64+ return (pmc->idx == ARMV8_PMU_CYCLE_IDX ||
65+ kvm_pmu_is_3p5(kvm_pmc_to_vcpu(pmc)));
5866 }
5967
60−static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
68+static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc)
6169 {
62− struct kvm_pmu *pmu;
63− struct kvm_vcpu_arch *vcpu_arch;
70+ u64 val = __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), PMCR_EL0);
6471
65− pmc -= pmc->idx;
66− pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
67− vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
68− return container_of(vcpu_arch, struct kvm_vcpu, arch);
72+ return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) ||
73+ (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC));
6974 }
7075
71−/**
72− * kvm_pmu_pmc_is_chained - determine if the pmc is chained
73− * @pmc: The PMU counter pointer
74− */
75−static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)
76+static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc)
7677 {
77− struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
78−
79− return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
78+ return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX &&
79+ !kvm_pmc_has_64bit_overflow(pmc));
8080 }
8181
82−/**
83− * kvm_pmu_idx_is_high_counter - determine if select_idx is a high/low counter
84− * @select_idx: The counter index
85− */
86−static bool kvm_pmu_idx_is_high_counter(u64 select_idx)
87−{
88− return select_idx & 0x1;
89−}
90−
91−/**
92− * kvm_pmu_get_canonical_pmc - obtain the canonical pmc
93− * @pmc: The PMU counter pointer
94− *
95− * When a pair of PMCs are chained together we use the low counter (canonical)
96− * to hold the underlying perf event.
97− */
98−static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)
99−{
100− if (kvm_pmu_pmc_is_chained(pmc) &&
101− kvm_pmu_idx_is_high_counter(pmc->idx))
102− return pmc - 1;
103−
104− return pmc;
105−}
106−static struct kvm_pmc *kvm_pmu_get_alternate_pmc(struct kvm_pmc *pmc)
82+static u32 counter_index_to_reg(u64 idx)
10783 {
108− if (kvm_pmu_idx_is_high_counter(pmc->idx))
109− return pmc - 1;
110− else
111− return pmc + 1;
84+ return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx;
11285 }
11386
114−/**
115− * kvm_pmu_idx_has_chain_evtype - determine if the event type is chain
116− * @vcpu: The vcpu pointer
117− * @select_idx: The counter index
118− */
119−static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)
87+static u32 counter_index_to_evtreg(u64 idx)
12088 {
121− u64 eventsel, reg;
122−
123− select_idx |= 0x1;
124−
125− if (select_idx == ARMV8_PMU_CYCLE_IDX)
126− return false;
127−
128− reg = PMEVTYPER0_EL0 + select_idx;
129− eventsel = __vcpu_sys_reg(vcpu, reg) & kvm_pmu_event_mask(vcpu->kvm);
130−
131− return eventsel == ARMV8_PMUV3_PERFCTR_CHAIN;
89+ return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx;
13290 }
13391
134−/**
135− * kvm_pmu_get_pair_counter_value - get PMU counter value
136− * @vcpu: The vcpu pointer
137− * @pmc: The PMU counter pointer
138− */
139−static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,
140− struct kvm_pmc *pmc)
92+static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc)
14193 {
142− u64 counter, counter_high, reg, enabled, running;
143−
144− if (kvm_pmu_pmc_is_chained(pmc)) {
145− pmc = kvm_pmu_get_canonical_pmc(pmc);
146− reg = PMEVCNTR0_EL0 + pmc->idx;
94+ struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
95+ u64 counter, reg, enabled, running;
14796
148− counter = __vcpu_sys_reg(vcpu, reg);
149− counter_high = __vcpu_sys_reg(vcpu, reg + 1);
150−
151− counter = lower_32_bits(counter) | (counter_high << 32);
152− } else {
153− reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
154− ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
155− counter = __vcpu_sys_reg(vcpu, reg);
156− }
97+ reg = counter_index_to_reg(pmc->idx);
98+ counter = __vcpu_sys_reg(vcpu, reg);
15799
158100 /*
159101 * The real counter value is equal to the value of counter register plus
@@ -163,6 +105,9 @@ static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu, · 3 unmodified lines
163105 counter += perf_event_read_value(pmc->perf_event, &enabled,
164106 &running);
165107
108+ if (!kvm_pmc_is_64bit(pmc))
109+ counter = lower_32_bits(counter);
110+
166111 return counter;
167112 }
168113
@@ -173,22 +118,37 @@ static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu, · 4 unmodified lines
173118 */
174119 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
175120 {
176− u64 counter;
177− struct kvm_pmu *pmu = &vcpu->arch.pmu;
178− struct kvm_pmc *pmc = &pmu->pmc[select_idx];
179−
180121 if (!kvm_vcpu_has_pmu(vcpu))
181122 return 0;
182123
183− counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
124+ return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx));
125+}
184126
185− if (kvm_pmu_pmc_is_chained(pmc) &&
186− kvm_pmu_idx_is_high_counter(select_idx))
187− counter = upper_32_bits(counter);
188− else if (select_idx != ARMV8_PMU_CYCLE_IDX)
189− counter = lower_32_bits(counter);
127+static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force)
128+{
129+ struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
130+ u64 reg;
190131
191− return counter;
132+ kvm_pmu_release_perf_event(pmc);
133+
134+ reg = counter_index_to_reg(pmc->idx);
135+
136+ if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX &&
137+ !force) {
138+ /*
139+ * Even with PMUv3p5, AArch32 cannot write to the top
140+ * 32bit of the counters. The only possible course of
141+ * action is to use PMCR.P, which will reset them to
142+ * 0 (the only use of the 'force' parameter).
143+ */
144+ val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32);
145+ val |= lower_32_bits(val);
146+ }
147+
148+ __vcpu_sys_reg(vcpu, reg) = val;
149+
150+ /* Recreate the perf event to reflect the updated sample_period */
151+ kvm_pmu_create_perf_event(pmc);
192152 }
193153
194154 /**
@@ -199,17 +159,10 @@ u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) · 4 unmodified lines
199159 */
200160 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
201161 {
202− u64 reg;
203−
204162 if (!kvm_vcpu_has_pmu(vcpu))
205163 return;
206164
207− reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
208− ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
209− __vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
210−
211− /* Recreate the perf event to reflect the updated sample_period */
212− kvm_pmu_create_perf_event(vcpu, select_idx);
165+ kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false);
213166 }
214167
215168 /**
@@ -218,7 +171,6 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) · 2 unmodified lines
218171 */
219172 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
220173 {
221− pmc = kvm_pmu_get_canonical_pmc(pmc);
222174 if (pmc->perf_event) {
223175 perf_event_disable(pmc->perf_event);
224176 perf_event_release_kernel(pmc->perf_event);
@@ -232,29 +184,20 @@ static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) · 7 unmodified lines
232184 *
233185 * If this counter has been configured to monitor some event, release it here.
234186 */
235−static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
187+static void kvm_pmu_stop_counter(struct kvm_pmc *pmc)
236188 {
237− u64 counter, reg, val;
189+ struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
190+ u64 reg, val;
238191
239− pmc = kvm_pmu_get_canonical_pmc(pmc);
240192 if (!pmc->perf_event)
241193 return;
242194
243− counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
195+ val = kvm_pmu_get_pmc_value(pmc);
244196
245− if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {
246− reg = PMCCNTR_EL0;
247− val = counter;
248− } else {
249− reg = PMEVCNTR0_EL0 + pmc->idx;
250− val = lower_32_bits(counter);
251− }
197+ reg = counter_index_to_reg(pmc->idx);
252198
253199 __vcpu_sys_reg(vcpu, reg) = val;
254200
255− if (kvm_pmu_pmc_is_chained(pmc))
256− __vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);
257−
258201 kvm_pmu_release_perf_event(pmc);
259202 }
260203
@@ -280,13 +223,10 @@ void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) · 19 unmodified lines
280223 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
281224 {
282225 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
283− struct kvm_pmu *pmu = &vcpu->arch.pmu;
284226 int i;
285227
286228 for_each_set_bit(i, &mask, 32)
287− kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
288−
289− bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
229+ kvm_pmu_stop_counter(kvm_vcpu_idx_to_pmc(vcpu, i));
290230 }
291231
292232 /**
@@ -297,10 +237,9 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) · 4 unmodified lines
297237 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
298238 {
299239 int i;
300− struct kvm_pmu *pmu = &vcpu->arch.pmu;
301240
302241 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
303− kvm_pmu_release_perf_event(&pmu->pmc[i]);
242+ kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i));
304243 irq_work_sync(&vcpu->arch.pmu.overflow_work);
305244 }
306245
@@ -325,9 +264,6 @@ u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) · 18 unmodified lines
325264 void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
326265 {
327266 int i;
328− struct kvm_pmu *pmu = &vcpu->arch.pmu;
329− struct kvm_pmc *pmc;
330−
331267 if (!kvm_vcpu_has_pmu(vcpu))
332268 return;
333269
@@ -335,17 +271,16 @@ void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) · 1 unmodified line
335271 return;
336272
337273 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
274+ struct kvm_pmc *pmc;
275+
338276 if (!(val & BIT(i)))
339277 continue;
340278
341− pmc = &pmu->pmc[i];
342−
343− /* A change in the enable state may affect the chain state */
344− kvm_pmu_update_pmc_chained(vcpu, i);
345− kvm_pmu_create_perf_event(vcpu, i);
279+ pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
346280
347− /* At this point, pmc must be the canonical */
348− if (pmc->perf_event) {
281+ if (!pmc->perf_event) {
282+ kvm_pmu_create_perf_event(pmc);
283+ } else {
349284 perf_event_enable(pmc->perf_event);
350285 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
351286 kvm_debug("fail to enable perf event\n");
@@ -363,23 +298,18 @@ void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) · 11 unmodified lines
363298 void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
364299 {
365300 int i;
366− struct kvm_pmu *pmu = &vcpu->arch.pmu;
367− struct kvm_pmc *pmc;
368301
369302 if (!kvm_vcpu_has_pmu(vcpu) || !val)
370303 return;
371304
372305 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
306+ struct kvm_pmc *pmc;
307+
373308 if (!(val & BIT(i)))
374309 continue;
375310
376− pmc = &pmu->pmc[i];
377−
378− /* A change in the enable state may affect the chain state */
379− kvm_pmu_update_pmc_chained(vcpu, i);
380− kvm_pmu_create_perf_event(vcpu, i);
311+ pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
381312
382− /* At this point, pmc must be the canonical */
383313 if (pmc->perf_event)
384314 perf_event_disable(pmc->perf_event);
385315 }
@@ -476,14 +406,69 @@ void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) · 90 unmodified lines
476406 static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work)
477407 {
478408 struct kvm_vcpu *vcpu;
479− struct kvm_pmu *pmu;
480−
481− pmu = container_of(work, struct kvm_pmu, overflow_work);
482− vcpu = kvm_pmc_to_vcpu(pmu->pmc);
483409
410+ vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work);
484411 kvm_vcpu_kick(vcpu);
485412 }
486413
414+/*
415+ * Perform an increment on any of the counters described in @mask,
416+ * generating the overflow if required, and propagate it as a chained
417+ * event if possible.
418+ */
419+static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu,
420+ unsigned long mask, u32 event)
421+{
422+ int i;
423+
424+ if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
425+ return;
426+
427+ /* Weed out disabled counters */
428+ mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
429+
430+ for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) {
431+ struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
432+ u64 type, reg;
433+
434+ /* Filter on event type */
435+ type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i));
436+ type &= kvm_pmu_event_mask(vcpu->kvm);
437+ if (type != event)
438+ continue;
439+
440+ /* Increment this counter */
441+ reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1;
442+ if (!kvm_pmc_is_64bit(pmc))
443+ reg = lower_32_bits(reg);
444+ __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) = reg;
445+
446+ /* No overflow? move on */
447+ if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg))
448+ continue;
449+
450+ /* Mark overflow */
451+ __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
452+
453+ if (kvm_pmu_counter_can_chain(pmc))
454+ kvm_pmu_counter_increment(vcpu, BIT(i + 1),
455+ ARMV8_PMUV3_PERFCTR_CHAIN);
456+ }
457+}
458+
459+/* Compute the sample period for a given counter value */
460+static u64 compute_period(struct kvm_pmc *pmc, u64 counter)
461+{
462+ u64 val;
463+
464+ if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc))
465+ val = (-counter) & GENMASK(63, 0);
466+ else
467+ val = (-counter) & GENMASK(31, 0);
468+
469+ return val;
470+}
471+
487472 /**
488473 * When the perf event overflows, set the overflow status and inform the vcpu.
489474 */
@@ -503,10 +488,7 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 13 unmodified lines
503488 * Reset the sample period to the architectural limit,
504489 * i.e. the point where the counter overflows.
505490 */
506− period = -(local64_read(&perf_event->count));
507−
508− if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
509− period &= GENMASK(31, 0);
491+ period = compute_period(pmc, local64_read(&perf_event->count));
510492
511493 local64_set(&perf_event->hw.period_left, 0);
512494 perf_event->attr.sample_period = period;
@@ -514,6 +496,10 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 1 unmodified line
514496
515497 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
516498
499+ if (kvm_pmu_counter_can_chain(pmc))
500+ kvm_pmu_counter_increment(vcpu, BIT(idx + 1),
501+ ARMV8_PMUV3_PERFCTR_CHAIN);
502+
517503 if (kvm_pmu_overflow_status(vcpu)) {
518504 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
519505
@@ -533,50 +519,7 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 13 unmodified lines
533519 */
534520 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
535521 {
536− struct kvm_pmu *pmu = &vcpu->arch.pmu;
537− int i;
538−
539− if (!kvm_vcpu_has_pmu(vcpu))
540− return;
541−
542− if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
543− return;
544−
545− /* Weed out disabled counters */
546− val &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
547−
548− for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
549− u64 type, reg;
550−
551− if (!(val & BIT(i)))
552− continue;
553−
554− /* PMSWINC only applies to ... SW_INC! */
555− type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i);
556− type &= kvm_pmu_event_mask(vcpu->kvm);
557− if (type != ARMV8_PMUV3_PERFCTR_SW_INCR)
558− continue;
559−
560− /* increment this even SW_INC counter */
561− reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
562− reg = lower_32_bits(reg);
563− __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
564−
565− if (reg) /* no overflow on the low part */
566− continue;
567−
568− if (kvm_pmu_pmc_is_chained(&pmu->pmc[i])) {
569− /* increment the high counter */
570− reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) + 1;
571− reg = lower_32_bits(reg);
572− __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) = reg;
573− if (!reg) /* mark overflow on the high counter */
574− __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i + 1);
575− } else {
576− /* mark overflow on low counter */
577− __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
578− }
579− }
522+ kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR);
580523 }
581524
582525 /**
@@ -591,6 +534,12 @@ void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) · 8 unmodified lines
591534 if (!kvm_vcpu_has_pmu(vcpu))
592535 return;
593536
537+ /* Fixup PMCR_EL0 to reconcile the PMU version and the LP bit */
538+ if (!kvm_pmu_is_3p5(vcpu))
539+ val &= ~ARMV8_PMU_PMCR_LP;
540+
541+ __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
542+
594543 if (val & ARMV8_PMU_PMCR_E) {
595544 kvm_pmu_enable_counter_mask(vcpu,
596545 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0));
@@ -606,49 +555,44 @@ void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) · 9 unmodified lines
606555 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
607556 mask &= ~BIT(ARMV8_PMU_CYCLE_IDX);
608557 for_each_set_bit(i, &mask, 32)
609− kvm_pmu_set_counter_value(vcpu, i, 0);
558+ kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true);
610559 }
611560 }
612561
613−static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
562+static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc)
614563 {
564+ struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
615565 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
616− (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
566+ (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx));
617567 }
618568
619569 /**
620570 * kvm_pmu_create_perf_event - create a perf event for a counter
621− * @vcpu: The vcpu pointer
622− * @select_idx: The number of selected counter
571+ * @pmc: Counter context
623572 */
624−static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)
573+static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc)
625574 {
575+ struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
626576 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu;
627− struct kvm_pmu *pmu = &vcpu->arch.pmu;
628− struct kvm_pmc *pmc;
629577 struct perf_event *event;
630578 struct perf_event_attr attr;
631− u64 eventsel, counter, reg, data;
579+ u64 eventsel, reg, data;
632580
633− /*
634− * For chained counters the event type and filtering attributes are
635− * obtained from the low/even counter. We also use this counter to
636− * determine if the event is enabled/disabled.
637− */
638− pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);
639−
640− reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
641− ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;
581+ reg = counter_index_to_evtreg(pmc->idx);
642582 data = __vcpu_sys_reg(vcpu, reg);
643583
644− kvm_pmu_stop_counter(vcpu, pmc);
584+ kvm_pmu_stop_counter(pmc);
645585 if (pmc->idx == ARMV8_PMU_CYCLE_IDX)
646586 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES;
647587 else
648588 eventsel = data & kvm_pmu_event_mask(vcpu->kvm);
649589
650− /* Software increment event doesn't need to be backed by a perf event */
651− if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR)
590+ /*
591+ * Neither SW increment nor chained events need to be backed
592+ * by a perf event.
593+ */
594+ if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR ||
595+ eventsel == ARMV8_PMUV3_PERFCTR_CHAIN)
652596 return;
653597
654598 /*
@@ -663,37 +607,25 @@ static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx) · 8 unmodified lines
663607 attr.type = arm_pmu->pmu.type;
664608 attr.size = sizeof(attr);
665609 attr.pinned = 1;
666− attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);
610+ attr.disabled = !kvm_pmu_counter_is_enabled(pmc);
667611 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
668612 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
669613 attr.exclude_hv = 1; /* Don't count EL2 events */
670614 attr.exclude_host = 1; /* Don't count host events */
671615 attr.config = eventsel;
672616
673− counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);
674−
675− if (kvm_pmu_pmc_is_chained(pmc)) {
676− /**
677− * The initial sample period (overflow count) of an event. For
678− * chained counters we only support overflow interrupts on the
679− * high counter.
680− */
681− attr.sample_period = (-counter) & GENMASK(63, 0);
682− attr.config1 |= PERF_ATTR_CFG1_KVM_PMU_CHAINED;
617+ /*
618+ * If counting with a 64bit counter, advertise it to the perf
619+ * code, carefully dealing with the initial sample period
620+ * which also depends on the overflow.
621+ */
622+ if (kvm_pmc_is_64bit(pmc))
623+ attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT;
683624
684− event = perf_event_create_kernel_counter(&attr, -1, current,
685− kvm_pmu_perf_overflow,
686− pmc + 1);
687− } else {
688− /* The initial sample period (overflow count) of an event. */
689− if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
690− attr.sample_period = (-counter) & GENMASK(63, 0);
691− else
692− attr.sample_period = (-counter) & GENMASK(31, 0);
625+ attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc));
693626
694− event = perf_event_create_kernel_counter(&attr, -1, current,
627+ event = perf_event_create_kernel_counter(&attr, -1, current,
695628 kvm_pmu_perf_overflow, pmc);
696− }
697629
698630 if (IS_ERR(event)) {
699631 pr_err_once("kvm: pmu event creation failed %ld\n",
@@ -704,41 +636,6 @@ static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx) · 4 unmodified lines
704636 pmc->perf_event = event;
705637 }
706638
707−/**
708− * kvm_pmu_update_pmc_chained - update chained bitmap
709− * @vcpu: The vcpu pointer
710− * @select_idx: The number of selected counter
711− *
712− * Update the chained bitmap based on the event type written in the
713− * typer register and the enable state of the odd register.
714− */
715−static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)
716−{
717− struct kvm_pmu *pmu = &vcpu->arch.pmu;
718− struct kvm_pmc *pmc = &pmu->pmc[select_idx], *canonical_pmc;
719− bool new_state, old_state;
720−
721− old_state = kvm_pmu_pmc_is_chained(pmc);
722− new_state = kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx) &&
723− kvm_pmu_counter_is_enabled(vcpu, pmc->idx | 0x1);
724−
725− if (old_state == new_state)
726− return;
727−
728− canonical_pmc = kvm_pmu_get_canonical_pmc(pmc);
729− kvm_pmu_stop_counter(vcpu, canonical_pmc);
730− if (new_state) {
731− /*
732− * During promotion from !chained to chained we must ensure
733− * the adjacent counter is stopped and its event destroyed
734− */
735− kvm_pmu_stop_counter(vcpu, kvm_pmu_get_alternate_pmc(pmc));
736− set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
737− return;
738− }
739− clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
740−}
741−
742639 /**
743640 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
744641 * @vcpu: The vcpu pointer
@@ -752,6 +649,7 @@ static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx) · 7 unmodified lines
752649 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
753650 u64 select_idx)
754651 {
652+ struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx);
755653 u64 reg, mask;
756654
757655 if (!kvm_vcpu_has_pmu(vcpu))
@@ -761,20 +659,19 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, · 3 unmodified lines
761659 mask &= ~ARMV8_PMU_EVTYPE_EVENT;
762660 mask |= kvm_pmu_event_mask(vcpu->kvm);
763661
764− reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
765− ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + select_idx;
662+ reg = counter_index_to_evtreg(pmc->idx);
766663
767664 __vcpu_sys_reg(vcpu, reg) = data & mask;
768665
769− kvm_pmu_update_pmc_chained(vcpu, select_idx);
770− kvm_pmu_create_perf_event(vcpu, select_idx);
666+ kvm_pmu_create_perf_event(pmc);
771667 }
772668
773669 void kvm_host_pmu_init(struct arm_pmu *pmu)
774670 {
775671 struct arm_pmu_entry *entry;
776672
777− if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
673+ if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
674+ pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
778675 return;
779676
780677 mutex_lock(&arm_pmus_lock);
@@ -827,7 +724,7 @@ static struct arm_pmu *kvm_pmu_probe_armpmu(void) · 46 unmodified lines
827724
828725 if (event->pmu) {
829726 pmu = to_arm_pmu(event->pmu);
830− if (pmu->pmuver == 0 ||
727+ if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
831728 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
832729 pmu = NULL;
833730 }
@@ -849,6 +746,8 @@ u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) · 15 unmodified lines
849746
850747 if (!pmceid1) {
851748 val = read_sysreg(pmceid0_el0);
749+ /* always support CHAIN */
750+ val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN);
852751 base = 0;
853752 } else {
854753 val = read_sysreg(pmceid1_el0);
@@ -1150,3 +1049,14 @@ int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) · 295 unmodified lines
11501049
11511050 return -ENXIO;
11521051 }
1052+
1053+u8 kvm_arm_pmu_get_pmuver_limit(void)
1054+{
1055+ u64 tmp;
1056+
1057+ tmp = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1058+ tmp = cpuid_feature_cap_perfmon_field(tmp,
1059+ ID_AA64DFR0_EL1_PMUVer_SHIFT,
1060+ ID_AA64DFR0_EL1_PMUVer_V3P5);
1061+ return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), tmp);
1062+}
arch/arm64/kvm/reset.c+0−29
@@ -395,32 +395,3 @@ int kvm_set_ipa_limit(void) · 394 unmodified lines
395395
396396 return 0;
397397 }
398−
399−int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type)
400−{
401− u64 mmfr0, mmfr1;
402− u32 phys_shift;
403−
404− if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
405− return -EINVAL;
406−
407− phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
408− if (phys_shift) {
409− if (phys_shift > kvm_ipa_limit ||
410− phys_shift < ARM64_MIN_PARANGE_BITS)
411− return -EINVAL;
412− } else {
413− phys_shift = KVM_PHYS_SHIFT;
414− if (phys_shift > kvm_ipa_limit) {
415− pr_warn_once("%s using unsupported default IPA limit, upgrade your VMM\n",
416− current->comm);
417− return -EINVAL;
418− }
419− }
420−
421− mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
422− mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
423− kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
424−
425− return 0;
426−}
arch/arm64/kvm/sys_regs.c+135−22
@@ -639,22 +639,18 @@ static void reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) · 638 unmodified lines
639639
640640 static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
641641 {
642− u64 pmcr, val;
642+ u64 pmcr;
643643
644644 /* No PMU available, PMCR_EL0 may UNDEF... */
645645 if (!kvm_arm_support_pmu_v3())
646646 return;
647647
648− pmcr = read_sysreg(pmcr_el0);
649− /*
650− * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
651− * except PMCR.E resetting to zero.
652− */
653− val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
654− | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
648+ /* Only preserve PMCR_EL0.N, and reset the rest to 0 */
649+ pmcr = read_sysreg(pmcr_el0) & ARMV8_PMU_PMCR_N_MASK;
655650 if (!kvm_supports_32bit_el0())
656− val |= ARMV8_PMU_PMCR_LC;
657− __vcpu_sys_reg(vcpu, r->reg) = val;
651+ pmcr |= ARMV8_PMU_PMCR_LC;
652+
653+ __vcpu_sys_reg(vcpu, r->reg) = pmcr;
658654 }
659655
660656 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
@@ -697,13 +693,15 @@ static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 36 unmodified lines
697693 return false;
698694
699695 if (p->is_write) {
700− /* Only update writeable bits of PMCR */
696+ /*
697+ * Only update writeable bits of PMCR (continuing into
698+ * kvm_pmu_handle_pmcr() as well)
699+ */
701700 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
702701 val &= ~ARMV8_PMU_PMCR_MASK;
703702 val |= p->regval & ARMV8_PMU_PMCR_MASK;
704703 if (!kvm_supports_32bit_el0())
705704 val |= ARMV8_PMU_PMCR_LC;
706− __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
707705 kvm_pmu_handle_pmcr(vcpu, val);
708706 kvm_vcpu_pmu_restore_guest(vcpu);
709707 } else {
@@ -1062,6 +1060,40 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 352 unmodified lines
10621060 return true;
10631061 }
10641062
1063+static u8 vcpu_pmuver(const struct kvm_vcpu *vcpu)
1064+{
1065+ if (kvm_vcpu_has_pmu(vcpu))
1066+ return vcpu->kvm->arch.dfr0_pmuver.imp;
1067+
1068+ return vcpu->kvm->arch.dfr0_pmuver.unimp;
1069+}
1070+
1071+static u8 perfmon_to_pmuver(u8 perfmon)
1072+{
1073+ switch (perfmon) {
1074+ case ID_DFR0_EL1_PerfMon_PMUv3:
1075+ return ID_AA64DFR0_EL1_PMUVer_IMP;
1076+ case ID_DFR0_EL1_PerfMon_IMPDEF:
1077+ return ID_AA64DFR0_EL1_PMUVer_IMP_DEF;
1078+ default:
1079+ /* Anything ARMv8.1+ and NI have the same value. For now. */
1080+ return perfmon;
1081+ }
1082+}
1083+
1084+static u8 pmuver_to_perfmon(u8 pmuver)
1085+{
1086+ switch (pmuver) {
1087+ case ID_AA64DFR0_EL1_PMUVer_IMP:
1088+ return ID_DFR0_EL1_PerfMon_PMUv3;
1089+ case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
1090+ return ID_DFR0_EL1_PerfMon_IMPDEF;
1091+ default:
1092+ /* Anything ARMv8.1+ and NI have the same value. For now. */
1093+ return pmuver;
1094+ }
1095+}
1096+
10651097 /* Read a sanitised cpufeature ID register by sys_reg_desc */
10661098 static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r)
10671099 {
@@ -1111,18 +1143,17 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r · 43 unmodified lines
11111143 /* Limit debug to ARMv8.0 */
11121144 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer);
11131145 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6);
1114− /* Limit guests to PMUv3 for ARMv8.4 */
1115− val = cpuid_feature_cap_perfmon_field(val,
1116− ID_AA64DFR0_EL1_PMUVer_SHIFT,
1117− kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_EL1_PMUVer_V3P4 : 0);
1146+ /* Set PMUver to the required version */
1147+ val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer);
1148+ val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer),
1149+ vcpu_pmuver(vcpu));
11181150 /* Hide SPE from guests */
11191151 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer);
11201152 break;
11211153 case SYS_ID_DFR0_EL1:
1122− /* Limit guests to PMUv3 for ARMv8.4 */
1123− val = cpuid_feature_cap_perfmon_field(val,
1124− ID_DFR0_PERFMON_SHIFT,
1125− kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
1154+ val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon);
1155+ val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon),
1156+ pmuver_to_perfmon(vcpu_pmuver(vcpu)));
11261157 break;
11271158 }
11281159
@@ -1222,6 +1253,85 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, · 93 unmodified lines
12221253 return 0;
12231254 }
12241255
1256+static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1257+ const struct sys_reg_desc *rd,
1258+ u64 val)
1259+{
1260+ u8 pmuver, host_pmuver;
1261+ bool valid_pmu;
1262+
1263+ host_pmuver = kvm_arm_pmu_get_pmuver_limit();
1264+
1265+ /*
1266+ * Allow AA64DFR0_EL1.PMUver to be set from userspace as long
1267+ * as it doesn't promise more than what the HW gives us. We
1268+ * allow an IMPDEF PMU though, only if no PMU is supported
1269+ * (KVM backward compatibility handling).
1270+ */
1271+ pmuver = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), val);
1272+ if ((pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF && pmuver > host_pmuver))
1273+ return -EINVAL;
1274+
1275+ valid_pmu = (pmuver != 0 && pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF);
1276+
1277+ /* Make sure view register and PMU support do match */
1278+ if (kvm_vcpu_has_pmu(vcpu) != valid_pmu)
1279+ return -EINVAL;
1280+
1281+ /* We can only differ with PMUver, and anything else is an error */
1282+ val ^= read_id_reg(vcpu, rd);
1283+ val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer);
1284+ if (val)
1285+ return -EINVAL;
1286+
1287+ if (valid_pmu)
1288+ vcpu->kvm->arch.dfr0_pmuver.imp = pmuver;
1289+ else
1290+ vcpu->kvm->arch.dfr0_pmuver.unimp = pmuver;
1291+
1292+ return 0;
1293+}
1294+
1295+static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1296+ const struct sys_reg_desc *rd,
1297+ u64 val)
1298+{
1299+ u8 perfmon, host_perfmon;
1300+ bool valid_pmu;
1301+
1302+ host_perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1303+
1304+ /*
1305+ * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1306+ * it doesn't promise more than what the HW gives us on the
1307+ * AArch64 side (as everything is emulated with that), and
1308+ * that this is a PMUv3.
1309+ */
1310+ perfmon = FIELD_GET(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon), val);
1311+ if ((perfmon != ID_DFR0_EL1_PerfMon_IMPDEF && perfmon > host_perfmon) ||
1312+ (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3))
1313+ return -EINVAL;
1314+
1315+ valid_pmu = (perfmon != 0 && perfmon != ID_DFR0_EL1_PerfMon_IMPDEF);
1316+
1317+ /* Make sure view register and PMU support do match */
1318+ if (kvm_vcpu_has_pmu(vcpu) != valid_pmu)
1319+ return -EINVAL;
1320+
1321+ /* We can only differ with PerfMon, and anything else is an error */
1322+ val ^= read_id_reg(vcpu, rd);
1323+ val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon);
1324+ if (val)
1325+ return -EINVAL;
1326+
1327+ if (valid_pmu)
1328+ vcpu->kvm->arch.dfr0_pmuver.imp = perfmon_to_pmuver(perfmon);
1329+ else
1330+ vcpu->kvm->arch.dfr0_pmuver.unimp = perfmon_to_pmuver(perfmon);
1331+
1332+ return 0;
1333+}
1334+
12251335 /*
12261336 * cpufeature ID register user accessors
12271337 *
@@ -1443,7 +1553,9 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 215 unmodified lines
14431553 /* CRm=1 */
14441554 AA32_ID_SANITISED(ID_PFR0_EL1),
14451555 AA32_ID_SANITISED(ID_PFR1_EL1),
1446− AA32_ID_SANITISED(ID_DFR0_EL1),
1556+ { SYS_DESC(SYS_ID_DFR0_EL1), .access = access_id_reg,
1557+ .get_user = get_id_reg, .set_user = set_id_dfr0_el1,
1558+ .visibility = aa32_id_visibility, },
14471559 ID_HIDDEN(ID_AFR0_EL1),
14481560 AA32_ID_SANITISED(ID_MMFR0_EL1),
14491561 AA32_ID_SANITISED(ID_MMFR1_EL1),
@@ -1483,7 +1595,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 33 unmodified lines
14831595 ID_UNALLOCATED(4,7),
14841596
14851597 /* CRm=5 */
1486− ID_SANITISED(ID_AA64DFR0_EL1),
1598+ { SYS_DESC(SYS_ID_AA64DFR0_EL1), .access = access_id_reg,
1599+ .get_user = get_id_reg, .set_user = set_id_aa64dfr0_el1, },
14871600 ID_SANITISED(ID_AA64DFR1_EL1),
14881601 ID_UNALLOCATED(5,2),
14891602 ID_UNALLOCATED(5,3),
arch/arm64/kvm/vgic/vgic-its.c+20−0
@@ -2743,6 +2743,7 @@ static int vgic_its_has_attr(struct kvm_device *dev, · 2742 unmodified lines
27432743 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
27442744 {
27452745 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2746+ struct vgic_dist *dist = &kvm->arch.vgic;
27462747 int ret = 0;
27472748
27482749 if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
@@ -2762,7 +2763,9 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr) · 13 unmodified lines
27622763 vgic_its_reset(kvm, its);
27632764 break;
27642765 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2766+ dist->save_its_tables_in_progress = true;
27652767 ret = abi->save_tables(its);
2768+ dist->save_its_tables_in_progress = false;
27662769 break;
27672770 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
27682771 ret = abi->restore_tables(its);
@@ -2775,6 +2778,23 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr) · 6 unmodified lines
27752778 return ret;
27762779 }
27772780
2781+/*
2782+ * kvm_arch_allow_write_without_running_vcpu - allow writing guest memory
2783+ * without the running VCPU when dirty ring is enabled.
2784+ *
2785+ * The running VCPU is required to track dirty guest pages when dirty ring
2786+ * is enabled. Otherwise, the backup bitmap should be used to track the
2787+ * dirty guest pages. When vgic/its tables are being saved, the backup
2788+ * bitmap is used to track the dirty guest pages due to the missed running
2789+ * VCPU in the period.
2790+ */
2791+bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
2792+{
2793+ struct vgic_dist *dist = &kvm->arch.vgic;
2794+
2795+ return dist->save_its_tables_in_progress;
2796+}
2797+
27782798 static int vgic_its_set_attr(struct kvm_device *dev,
27792799 struct kvm_device_attr *attr)
27802800 {
arch/arm64/lib/mte.S+1−1
@@ -18,7 +18,7 @@ · 17 unmodified lines
1818 */
1919 .macro multitag_transfer_size, reg, tmp
2020 mrs_s \reg, SYS_GMID_EL1
21− ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_SIZE
21+ ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
2222 mov \tmp, #4
2323 lsl \reg, \tmp, \reg
2424 .endm
arch/arm64/mm/copypage.c+5−2
@@ -21,9 +21,12 @@ void copy_highpage(struct page *to, struct page *from) · 20 unmodified lines
2121
2222 copy_page(kto, kfrom);
2323
24− if (system_supports_mte() && test_bit(PG_mte_tagged, &from->flags)) {
25− set_bit(PG_mte_tagged, &to->flags);
24+ if (system_supports_mte() && page_mte_tagged(from)) {
25+ page_kasan_tag_reset(to);
26+ /* It's a new page, shouldn't have been tagged yet */
27+ WARN_ON_ONCE(!try_page_mte_tagging(to));
2628 mte_copy_page_tags(kto, kfrom);
29+ set_page_mte_tagged(to);
2730 }
2831 }
2932 EXPORT_SYMBOL(copy_highpage);
arch/arm64/mm/fault.c+3−1
@@ -937,6 +937,8 @@ struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, · 936 unmodified lines
937937
938938 void tag_clear_highpage(struct page *page)
939939 {
940+ /* Newly allocated page, shouldn't have been tagged yet */
941+ WARN_ON_ONCE(!try_page_mte_tagging(page));
940942 mte_zero_clear_page_tags(page_address(page));
941− set_bit(PG_mte_tagged, &page->flags);
943+ set_page_mte_tagged(page);
942944 }
arch/arm64/mm/mteswap.c+6−10
@@ -24,7 +24,7 @@ int mte_save_tags(struct page *page) · 23 unmodified lines
2424 {
2525 void *tag_storage, *ret;
2626
27− if (!test_bit(PG_mte_tagged, &page->flags))
27+ if (!page_mte_tagged(page))
2828 return 0;
2929
3030 tag_storage = mte_allocate_tag_storage();
@@ -46,21 +46,17 @@ int mte_save_tags(struct page *page) · 15 unmodified lines
4646 return 0;
4747 }
4848
49−bool mte_restore_tags(swp_entry_t entry, struct page *page)
49+void mte_restore_tags(swp_entry_t entry, struct page *page)
5050 {
5151 void *tags = xa_load(&mte_pages, entry.val);
5252
5353 if (!tags)
54− return false;
54+ return;
5555
56− /*
57− * Test PG_mte_tagged again in case it was racing with another
58− * set_pte_at().
59− */
60− if (!test_and_set_bit(PG_mte_tagged, &page->flags))
56+ if (try_page_mte_tagging(page)) {
6157 mte_restore_page_tags(page_address(page), tags);
62−
63− return true;
58+ set_page_mte_tagged(page);
59+ }
6460 }
6561
6662 void mte_invalidate_tags(int type, pgoff_t offset)
arch/arm64/tools/gen-sysreg.awk+1−1
@@ -33,7 +33,7 @@ function expect_fields(nf) { · 32 unmodified lines
3333 # Print a CPP macro definition, padded with spaces so that the macro bodies
3434 # line up in a column
3535 function define(name, val) {
36− printf "%-48s%s\n", "#define " name, val
36+ printf "%-56s%s\n", "#define " name, val
3737 }
3838
3939 # Print standard BITMASK/SHIFT/WIDTH CPP definitions for a field
arch/arm64/tools/sysreg+754−0
@@ -46,6 +46,760 @@ · 45 unmodified lines
4646 # feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
4747 # item ACCDATA) though it may be more taseful to do something else.
4848
49+Sysreg ID_PFR0_EL1 3 0 0 1 0
50+Res0 63:32
51+Enum 31:28 RAS
52+ 0b0000 NI
53+ 0b0001 RAS
54+ 0b0010 RASv1p1
55+EndEnum
56+Enum 27:24 DIT
57+ 0b0000 NI
58+ 0b0001 IMP
59+EndEnum
60+Enum 23:20 AMU
61+ 0b0000 NI
62+ 0b0001 AMUv1
63+ 0b0010 AMUv1p1
64+EndEnum
65+Enum 19:16 CSV2
66+ 0b0000 UNDISCLOSED
67+ 0b0001 IMP
68+ 0b0010 CSV2p1
69+EndEnum
70+Enum 15:12 State3
71+ 0b0000 NI
72+ 0b0001 IMP
73+EndEnum
74+Enum 11:8 State2
75+ 0b0000 NI
76+ 0b0001 NO_CV
77+ 0b0010 CV
78+EndEnum
79+Enum 7:4 State1
80+ 0b0000 NI
81+ 0b0001 THUMB
82+ 0b0010 THUMB2
83+EndEnum
84+Enum 3:0 State0
85+ 0b0000 NI
86+ 0b0001 IMP
87+EndEnum
88+EndSysreg
89+
90+Sysreg ID_PFR1_EL1 3 0 0 1 1
91+Res0 63:32
92+Enum 31:28 GIC
93+ 0b0000 NI
94+ 0b0001 GICv3
95+ 0b0010 GICv4p1
96+EndEnum
97+Enum 27:24 Virt_frac
98+ 0b0000 NI
99+ 0b0001 IMP
100+EndEnum
101+Enum 23:20 Sec_frac
102+ 0b0000 NI
103+ 0b0001 WALK_DISABLE
104+ 0b0010 SECURE_MEMORY
105+EndEnum
106+Enum 19:16 GenTimer
107+ 0b0000 NI
108+ 0b0001 IMP
109+ 0b0010 ECV
110+EndEnum
111+Enum 15:12 Virtualization
112+ 0b0000 NI
113+ 0b0001 IMP
114+EndEnum
115+Enum 11:8 MProgMod
116+ 0b0000 NI
117+ 0b0001 IMP
118+EndEnum
119+Enum 7:4 Security
120+ 0b0000 NI
121+ 0b0001 EL3
122+ 0b0001 NSACR_RFR
123+EndEnum
124+Enum 3:0 ProgMod
125+ 0b0000 NI
126+ 0b0001 IMP
127+EndEnum
128+EndSysreg
129+
130+Sysreg ID_DFR0_EL1 3 0 0 1 2
131+Res0 63:32
132+Enum 31:28 TraceFilt
133+ 0b0000 NI
134+ 0b0001 IMP
135+EndEnum
136+Enum 27:24 PerfMon
137+ 0b0000 NI
138+ 0b0001 PMUv1
139+ 0b0010 PMUv2
140+ 0b0011 PMUv3
141+ 0b0100 PMUv3p1
142+ 0b0101 PMUv3p4
143+ 0b0110 PMUv3p5
144+ 0b0111 PMUv3p7
145+ 0b1000 PMUv3p8
146+ 0b1111 IMPDEF
147+EndEnum
148+Enum 23:20 MProfDbg
149+ 0b0000 NI
150+ 0b0001 IMP
151+EndEnum
152+Enum 19:16 MMapTrc
153+ 0b0000 NI
154+ 0b0001 IMP
155+EndEnum
156+Enum 15:12 CopTrc
157+ 0b0000 NI
158+ 0b0001 IMP
159+EndEnum
160+Enum 11:8 MMapDbg
161+ 0b0000 NI
162+ 0b0100 Armv7
163+ 0b0101 Armv7p1
164+EndEnum
165+Field 7:4 CopSDbg
166+Enum 3:0 CopDbg
167+ 0b0000 NI
168+ 0b0010 Armv6
169+ 0b0011 Armv6p1
170+ 0b0100 Armv7
171+ 0b0101 Armv7p1
172+ 0b0110 Armv8
173+ 0b0111 VHE
174+ 0b1000 Debugv8p2
175+ 0b1001 Debugv8p4
176+ 0b1010 Debugv8p8
177+EndEnum
178+EndSysreg
179+
180+Sysreg ID_AFR0_EL1 3 0 0 1 3
181+Res0 63:16
182+Field 15:12 IMPDEF3
183+Field 11:8 IMPDEF2
184+Field 7:4 IMPDEF1
185+Field 3:0 IMPDEF0
186+EndSysreg
187+
188+Sysreg ID_MMFR0_EL1 3 0 0 1 4
189+Res0 63:32
190+Enum 31:28 InnerShr
191+ 0b0000 NC
192+ 0b0001 HW
193+ 0b1111 IGNORED
194+EndEnum
195+Enum 27:24 FCSE
196+ 0b0000 NI
197+ 0b0001 IMP
198+EndEnum
199+Enum 23:20 AuxReg
200+ 0b0000 NI
201+ 0b0001 ACTLR
202+ 0b0010 AIFSR
203+EndEnum
204+Enum 19:16 TCM
205+ 0b0000 NI
206+ 0b0001 IMPDEF
207+ 0b0010 TCM
208+ 0b0011 TCM_DMA
209+EndEnum
210+Enum 15:12 ShareLvl
211+ 0b0000 ONE
212+ 0b0001 TWO
213+EndEnum
214+Enum 11:8 OuterShr
215+ 0b0000 NC
216+ 0b0001 HW
217+ 0b1111 IGNORED
218+EndEnum
219+Enum 7:4 PMSA
220+ 0b0000 NI
221+ 0b0001 IMPDEF
222+ 0b0010 PMSAv6
223+ 0b0011 PMSAv7
224+EndEnum
225+Enum 3:0 VMSA
226+ 0b0000 NI
227+ 0b0001 IMPDEF
228+ 0b0010 VMSAv6
229+ 0b0011 VMSAv7
230+ 0b0100 VMSAv7_PXN
231+ 0b0101 VMSAv7_LONG
232+EndEnum
233+EndSysreg
234+
235+Sysreg ID_MMFR1_EL1 3 0 0 1 5
236+Res0 63:32
237+Enum 31:28 BPred
238+ 0b0000 NI
239+ 0b0001 BP_SW_MANGED
240+ 0b0010 BP_ASID_AWARE
241+ 0b0011 BP_NOSNOOP
242+ 0b0100 BP_INVISIBLE
243+EndEnum
244+Enum 27:24 L1TstCln
245+ 0b0000 NI
246+ 0b0001 NOINVALIDATE
247+ 0b0010 INVALIDATE
248+EndEnum
249+Enum 23:20 L1Uni
250+ 0b0000 NI
251+ 0b0001 INVALIDATE
252+ 0b0010 CLEAN_AND_INVALIDATE
253+EndEnum
254+Enum 19:16 L1Hvd
255+ 0b0000 NI
256+ 0b0001 INVALIDATE_ISIDE_ONLY
257+ 0b0010 INVALIDATE
258+ 0b0011 CLEAN_AND_INVALIDATE
259+EndEnum
260+Enum 15:12 L1UniSW
261+ 0b0000 NI
262+ 0b0001 CLEAN
263+ 0b0010 CLEAN_AND_INVALIDATE
264+ 0b0011 INVALIDATE
265+EndEnum
266+Enum 11:8 L1HvdSW
267+ 0b0000 NI
268+ 0b0001 CLEAN_AND_INVALIDATE
269+ 0b0010 INVALIDATE_DSIDE_ONLY
270+ 0b0011 INVALIDATE
271+EndEnum
272+Enum 7:4 L1UniVA
273+ 0b0000 NI
274+ 0b0001 CLEAN_AND_INVALIDATE
275+ 0b0010 INVALIDATE_BP
276+EndEnum
277+Enum 3:0 L1HvdVA
278+ 0b0000 NI
279+ 0b0001 CLEAN_AND_INVALIDATE
280+ 0b0010 INVALIDATE_BP
281+EndEnum
282+EndSysreg
283+
284+Sysreg ID_MMFR2_EL1 3 0 0 1 6
285+Res0 63:32
286+Enum 31:28 HWAccFlg
287+ 0b0000 NI
288+ 0b0001 IMP
289+EndEnum
290+Enum 27:24 WFIStall
291+ 0b0000 NI
292+ 0b0001 IMP
293+EndEnum
294+Enum 23:20 MemBarr
295+ 0b0000 NI
296+ 0b0001 DSB_ONLY
297+ 0b0010 IMP
298+EndEnum
299+Enum 19:16 UniTLB
300+ 0b0000 NI
301+ 0b0001 BY_VA
302+ 0b0010 BY_MATCH_ASID
303+ 0b0011 BY_ALL_ASID
304+ 0b0100 OTHER_TLBS
305+ 0b0101 BROADCAST
306+ 0b0110 BY_IPA
307+EndEnum
308+Enum 15:12 HvdTLB
309+ 0b0000 NI
310+EndEnum
311+Enum 11:8 L1HvdRng
312+ 0b0000 NI
313+ 0b0001 IMP
314+EndEnum
315+Enum 7:4 L1HvdBG
316+ 0b0000 NI
317+ 0b0001 IMP
318+EndEnum
319+Enum 3:0 L1HvdFG
320+ 0b0000 NI
321+ 0b0001 IMP
322+EndEnum
323+EndSysreg
324+
325+Sysreg ID_MMFR3_EL1 3 0 0 1 7
326+Res0 63:32
327+Enum 31:28 Supersec
328+ 0b0000 IMP
329+ 0b1111 NI
330+EndEnum
331+Enum 27:24 CMemSz
332+ 0b0000 4GB
333+ 0b0001 64GB
334+ 0b0010 1TB
335+EndEnum
336+Enum 23:20 CohWalk
337+ 0b0000 NI
338+ 0b0001 IMP
339+EndEnum
340+Enum 19:16 PAN
341+ 0b0000 NI
342+ 0b0001 PAN
343+ 0b0010 PAN2
344+EndEnum
345+Enum 15:12 MaintBcst
346+ 0b0000 NI
347+ 0b0001 NO_TLB
348+ 0b0010 ALL
349+EndEnum
350+Enum 11:8 BPMaint
351+ 0b0000 NI
352+ 0b0001 ALL
353+ 0b0010 BY_VA
354+EndEnum
355+Enum 7:4 CMaintSW
356+ 0b0000 NI
357+ 0b0001 IMP
358+EndEnum
359+Enum 3:0 CMaintVA
360+ 0b0000 NI
361+ 0b0001 IMP
362+EndEnum
363+EndSysreg
364+
365+Sysreg ID_ISAR0_EL1 3 0 0 2 0
366+Res0 63:28
367+Enum 27:24 Divide
368+ 0b0000 NI
369+ 0b0001 xDIV_T32
370+ 0b0010 xDIV_A32
371+EndEnum
372+Enum 23:20 Debug
373+ 0b0000 NI
374+ 0b0001 IMP
375+EndEnum
376+Enum 19:16 Coproc
377+ 0b0000 NI
378+ 0b0001 MRC
379+ 0b0010 MRC2
380+ 0b0011 MRRC
381+ 0b0100 MRRC2
382+EndEnum
383+Enum 15:12 CmpBranch
384+ 0b0000 NI
385+ 0b0001 IMP
386+EndEnum
387+Enum 11:8 BitField
388+ 0b0000 NI
389+ 0b0001 IMP
390+EndEnum
391+Enum 7:4 BitCount
392+ 0b0000 NI
393+ 0b0001 IMP
394+EndEnum
395+Enum 3:0 Swap
396+ 0b0000 NI
397+ 0b0001 IMP
398+EndEnum
399+EndSysreg
400+
401+Sysreg ID_ISAR1_EL1 3 0 0 2 1
402+Res0 63:32
403+Enum 31:28 Jazelle
404+ 0b0000 NI
405+ 0b0001 IMP
406+EndEnum
407+Enum 27:24 Interwork
408+ 0b0000 NI
409+ 0b0001 BX
410+ 0b0010 BLX
411+ 0b0011 A32_BX
412+EndEnum
413+Enum 23:20 Immediate
414+ 0b0000 NI
415+ 0b0001 IMP
416+EndEnum
417+Enum 19:16 IfThen
418+ 0b0000 NI
419+ 0b0001 IMP
420+EndEnum
421+Enum 15:12 Extend
422+ 0b0000 NI
423+ 0b0001 SXTB
424+ 0b0010 SXTB16
425+EndEnum
426+Enum 11:8 Except_AR
427+ 0b0000 NI
428+ 0b0001 IMP
429+EndEnum
430+Enum 7:4 Except
431+ 0b0000 NI
432+ 0b0001 IMP
433+EndEnum
434+Enum 3:0 Endian
435+ 0b0000 NI
436+ 0b0001 IMP
437+EndEnum
438+EndSysreg
439+
440+Sysreg ID_ISAR2_EL1 3 0 0 2 2
441+Res0 63:32
442+Enum 31:28 Reversal
443+ 0b0000 NI
444+ 0b0001 REV
445+ 0b0010 RBIT
446+EndEnum
447+Enum 27:24 PSR_AR
448+ 0b0000 NI
449+ 0b0001 IMP
450+EndEnum
451+Enum 23:20 MultU
452+ 0b0000 NI
453+ 0b0001 UMULL
454+ 0b0010 UMAAL
455+EndEnum
456+Enum 19:16 MultS
457+ 0b0000 NI
458+ 0b0001 SMULL
459+ 0b0010 SMLABB
460+ 0b0011 SMLAD
461+EndEnum
462+Enum 15:12 Mult
463+ 0b0000 NI
464+ 0b0001 MLA
465+ 0b0010 MLS
466+EndEnum
467+Enum 11:8 MultiAccessInt
468+ 0b0000 NI
469+ 0b0001 RESTARTABLE
470+ 0b0010 CONTINUABLE
471+EndEnum
472+Enum 7:4 MemHint
473+ 0b0000 NI
474+ 0b0001 PLD
475+ 0b0010 PLD2
476+ 0b0011 PLI
477+ 0b0100 PLDW
478+EndEnum
479+Enum 3:0 LoadStore
480+ 0b0000 NI
481+ 0b0001 DOUBLE
482+ 0b0010 ACQUIRE
483+EndEnum
484+EndSysreg
485+
486+Sysreg ID_ISAR3_EL1 3 0 0 2 3
487+Res0 63:32
488+Enum 31:28 T32EE
489+ 0b0000 NI
490+ 0b0001 IMP
491+EndEnum
492+Enum 27:24 TrueNOP
493+ 0b0000 NI
494+ 0b0001 IMP
495+EndEnum
496+Enum 23:20 T32Copy
497+ 0b0000 NI
498+ 0b0001 IMP
499+EndEnum
500+Enum 19:16 TabBranch
501+ 0b0000 NI
502+ 0b0001 IMP
503+EndEnum
504+Enum 15:12 SynchPrim
505+ 0b0000 NI
506+ 0b0001 EXCLUSIVE
507+ 0b0010 DOUBLE
508+EndEnum
509+Enum 11:8 SVC
510+ 0b0000 NI
511+ 0b0001 IMP
512+EndEnum
513+Enum 7:4 SIMD
514+ 0b0000 NI
515+ 0b0001 SSAT
516+ 0b0011 PKHBT
517+EndEnum
518+Enum 3:0 Saturate
519+ 0b0000 NI
520+ 0b0001 IMP
521+EndEnum
522+EndSysreg
523+
524+Sysreg ID_ISAR4_EL1 3 0 0 2 4
525+Res0 63:32
526+Enum 31:28 SWP_frac
527+ 0b0000 NI
528+ 0b0001 IMP
529+EndEnum
530+Enum 27:24 PSR_M
531+ 0b0000 NI
532+ 0b0001 IMP
533+EndEnum
534+Enum 23:20 SynchPrim_frac
535+ 0b0000 NI
536+ 0b0011 IMP
537+EndEnum
538+Enum 19:16 Barrier
539+ 0b0000 NI
540+ 0b0001 IMP
541+EndEnum
542+Enum 15:12 SMC
543+ 0b0000 NI
544+ 0b0001 IMP
545+EndEnum
546+Enum 11:8 Writeback
547+ 0b0000 NI
548+ 0b0001 IMP
549+EndEnum
550+Enum 7:4 WithShifts
551+ 0b0000 NI
552+ 0b0001 LSL3
553+ 0b0011 LS
554+ 0b0100 REG
555+EndEnum
556+Enum 3:0 Unpriv
557+ 0b0000 NI
558+ 0b0001 REG_BYTE
559+ 0b0010 SIGNED_HALFWORD
560+EndEnum
561+EndSysreg
562+
563+Sysreg ID_ISAR5_EL1 3 0 0 2 5
564+Res0 63:32
565+Enum 31:28 VCMA
566+ 0b0000 NI
567+ 0b0001 IMP
568+EndEnum
569+Enum 27:24 RDM
570+ 0b0000 NI
571+ 0b0001 IMP
572+EndEnum
573+Res0 23:20
574+Enum 19:16 CRC32
575+ 0b0000 NI
576+ 0b0001 IMP
577+EndEnum
578+Enum 15:12 SHA2
579+ 0b0000 NI
580+ 0b0001 IMP
581+EndEnum
582+Enum 11:8 SHA1
583+ 0b0000 NI
584+ 0b0001 IMP
585+EndEnum
586+Enum 7:4 AES
587+ 0b0000 NI
588+ 0b0001 IMP
589+ 0b0010 VMULL
590+EndEnum
591+Enum 3:0 SEVL
592+ 0b0000 NI
593+ 0b0001 IMP
594+EndEnum
595+EndSysreg
596+
597+Sysreg ID_ISAR6_EL1 3 0 0 2 7
598+Res0 63:28
599+Enum 27:24 I8MM
600+ 0b0000 NI
601+ 0b0001 IMP
602+EndEnum
603+Enum 23:20 BF16
604+ 0b0000 NI
605+ 0b0001 IMP
606+EndEnum
607+Enum 19:16 SPECRES
608+ 0b0000 NI
609+ 0b0001 IMP
610+EndEnum
611+Enum 15:12 SB
612+ 0b0000 NI
613+ 0b0001 IMP
614+EndEnum
615+Enum 11:8 FHM
616+ 0b0000 NI
617+ 0b0001 IMP
618+EndEnum
619+Enum 7:4 DP
620+ 0b0000 NI
621+ 0b0001 IMP
622+EndEnum
623+Enum 3:0 JSCVT
624+ 0b0000 NI
625+ 0b0001 IMP
626+EndEnum
627+EndSysreg
628+
629+Sysreg ID_MMFR4_EL1 3 0 0 2 6
630+Res0 63:32
631+Enum 31:28 EVT
632+ 0b0000 NI
633+ 0b0001 NO_TLBIS
634+ 0b0010 TLBIS
635+EndEnum
636+Enum 27:24 CCIDX
637+ 0b0000 NI
638+ 0b0001 IMP
639+EndEnum
640+Enum 23:20 LSM
641+ 0b0000 NI
642+ 0b0001 IMP
643+EndEnum
644+Enum 19:16 HPDS
645+ 0b0000 NI
646+ 0b0001 AA32HPD
647+ 0b0010 HPDS2
648+EndEnum
649+Enum 15:12 CnP
650+ 0b0000 NI
651+ 0b0001 IMP
652+EndEnum
653+Enum 11:8 XNX
654+ 0b0000 NI
655+ 0b0001 IMP
656+EndEnum
657+Enum 7:4 AC2
658+ 0b0000 NI
659+ 0b0001 IMP
660+EndEnum
661+Enum 3:0 SpecSEI
662+ 0b0000 NI
663+ 0b0001 IMP
664+EndEnum
665+EndSysreg
666+
667+Sysreg MVFR0_EL1 3 0 0 3 0
668+Res0 63:32
669+Enum 31:28 FPRound
670+ 0b0000 NI
671+ 0b0001 IMP
672+EndEnum
673+Enum 27:24 FPShVec
674+ 0b0000 NI
675+ 0b0001 IMP
676+EndEnum
677+Enum 23:20 FPSqrt
678+ 0b0000 NI
679+ 0b0001 IMP
680+EndEnum
681+Enum 19:16 FPDivide
682+ 0b0000 NI
683+ 0b0001 IMP
684+EndEnum
685+Enum 15:12 FPTrap
686+ 0b0000 NI
687+ 0b0001 IMP
688+EndEnum
689+Enum 11:8 FPDP
690+ 0b0000 NI
691+ 0b0001 VFPv2
692+ 0b0001 VFPv3
693+EndEnum
694+Enum 7:4 FPSP
695+ 0b0000 NI
696+ 0b0001 VFPv2
697+ 0b0001 VFPv3
698+EndEnum
699+Enum 3:0 SIMDReg
700+ 0b0000 NI
701+ 0b0001 IMP_16x64
702+ 0b0001 IMP_32x64
703+EndEnum
704+EndSysreg
705+
706+Sysreg MVFR1_EL1 3 0 0 3 1
707+Res0 63:32
708+Enum 31:28 SIMDFMAC
709+ 0b0000 NI
710+ 0b0001 IMP
711+EndEnum
712+Enum 27:24 FPHP
713+ 0b0000 NI
714+ 0b0001 FPHP
715+ 0b0010 FPHP_CONV
716+ 0b0011 FP16
717+EndEnum
718+Enum 23:20 SIMDHP
719+ 0b0000 NI
720+ 0b0001 SIMDHP
721+ 0b0001 SIMDHP_FLOAT
722+EndEnum
723+Enum 19:16 SIMDSP
724+ 0b0000 NI
725+ 0b0001 IMP
726+EndEnum
727+Enum 15:12 SIMDInt
728+ 0b0000 NI
729+ 0b0001 IMP
730+EndEnum
731+Enum 11:8 SIMDLS
732+ 0b0000 NI
733+ 0b0001 IMP
734+EndEnum
735+Enum 7:4 FPDNaN
736+ 0b0000 NI
737+ 0b0001 IMP
738+EndEnum
739+Enum 3:0 FPFtZ
740+ 0b0000 NI
741+ 0b0001 IMP
742+EndEnum
743+EndSysreg
744+
745+Sysreg MVFR2_EL1 3 0 0 3 2
746+Res0 63:8
747+Enum 7:4 FPMisc
748+ 0b0000 NI
749+ 0b0001 FP
750+ 0b0010 FP_DIRECTED_ROUNDING
751+ 0b0011 FP_ROUNDING
752+ 0b0100 FP_MAX_MIN
753+EndEnum
754+Enum 3:0 SIMDMisc
755+ 0b0000 NI
756+ 0b0001 SIMD_DIRECTED_ROUNDING
757+ 0b0010 SIMD_ROUNDING
758+ 0b0011 SIMD_MAX_MIN
759+EndEnum
760+EndSysreg
761+
762+Sysreg ID_PFR2_EL1 3 0 0 3 4
763+Res0 63:12
764+Enum 11:8 RAS_frac
765+ 0b0000 NI
766+ 0b0001 RASv1p1
767+EndEnum
768+Enum 7:4 SSBS
769+ 0b0000 NI
770+ 0b0001 IMP
771+EndEnum
772+Enum 3:0 CSV3
773+ 0b0000 NI
774+ 0b0001 IMP
775+EndEnum
776+EndSysreg
777+
778+Sysreg ID_DFR1_EL1 3 0 0 3 5
779+Res0 63:8
780+Enum 7:4 HPMN0
781+ 0b0000 NI
782+ 0b0001 IMP
783+EndEnum
784+Enum 3:0 MTPMU
785+ 0b0000 IMPDEF
786+ 0b0001 IMP
787+ 0b1111 NI
788+EndEnum
789+EndSysreg
790+
791+Sysreg ID_MMFR5_EL1 3 0 0 3 6
792+Res0 63:8
793+Enum 7:4 nTLBPA
794+ 0b0000 NI
795+ 0b0001 IMP
796+EndEnum
797+Enum 3:0 ETS
798+ 0b0000 NI
799+ 0b0001 IMP
800+EndEnum
801+EndSysreg
802+
49803 Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
50804 Enum 63:60 CSV3
51805 0b0000 NI
arch/x86/include/asm/kvm_host.h+0−2
@@ -2154,8 +2154,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu) · 2153 unmodified lines
21542154 #endif
21552155 }
21562156
2157−int kvm_cpu_dirty_log_size(void);
2158−
21592157 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
21602158
21612159 #define KVM_CLOCK_VALID_FLAGS \
arch/x86/kvm/x86.c+6−9
@@ -10208,20 +10208,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10207 unmodified lines
1020810208
1020910209 bool req_immediate_exit = false;
1021010210
10211− /* Forbid vmenter if vcpu dirty ring is soft-full */
10212− if (unlikely(vcpu->kvm->dirty_ring_size &&
10213− kvm_dirty_ring_soft_full(&vcpu->dirty_ring))) {
10214− vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
10215− trace_kvm_dirty_ring_exit(vcpu);
10216− r = 0;
10217− goto out;
10218− }
10219−
1022010211 if (kvm_request_pending(vcpu)) {
1022110212 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
1022210213 r = -EIO;
1022310214 goto out;
1022410215 }
10216+
10217+ if (kvm_dirty_ring_check_request(vcpu)) {
10218+ r = 0;
10219+ goto out;
10220+ }
10221+
1022510222 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
1022610223 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
1022710224 r = 0;
fs/proc/page.c+2−1
@@ -219,8 +219,9 @@ u64 stable_page_flags(struct page *page) · 218 unmodified lines
219219 u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
220220 u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
221221 u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
222−#ifdef CONFIG_64BIT
222+#ifdef CONFIG_ARCH_USES_PG_ARCH_X
223223 u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2);
224+ u |= kpf_copy_bit(k, KPF_ARCH_3, PG_arch_3);
224225 #endif
225226
226227 return u;
include/kvm/arm_pmu.h+13−2
@@ -11,7 +11,6 @@ · 10 unmodified lines
1111 #include <asm/perf_event.h>
1212
1313 #define ARMV8_PMU_CYCLE_IDX (ARMV8_PMU_MAX_COUNTERS - 1)
14−#define ARMV8_PMU_MAX_COUNTER_PAIRS ((ARMV8_PMU_MAX_COUNTERS + 1) >> 1)
1514
1615 #ifdef CONFIG_HW_PERF_EVENTS
1716
@@ -29,7 +28,6 @@ struct kvm_pmu { · 11 unmodified lines
2928 struct irq_work overflow_work;
3029 struct kvm_pmu_events events;
3130 struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS];
32− DECLARE_BITMAP(chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
3331 int irq_num;
3432 bool created;
3533 bool irq_level;
@@ -91,6 +89,14 @@ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); · 55 unmodified lines
9189 vcpu->arch.pmu.events = *kvm_get_pmu_events(); \
9290 } while (0)
9391
92+/*
93+ * Evaluates as true when emulating PMUv3p5, and false otherwise.
94+ */
95+#define kvm_pmu_is_3p5(vcpu) \
96+ (vcpu->kvm->arch.dfr0_pmuver.imp >= ID_AA64DFR0_EL1_PMUVer_V3P5)
97+
98+u8 kvm_arm_pmu_get_pmuver_limit(void);
99+
94100 #else
95101 struct kvm_pmu {
96102 };
@@ -153,9 +159,14 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) · 56 unmodified lines
153159 }
154160
155161 #define kvm_vcpu_has_pmu(vcpu) ({ false; })
162+#define kvm_pmu_is_3p5(vcpu) ({ false; })
156163 static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {}
157164 static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}
158165 static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}
166+static inline u8 kvm_arm_pmu_get_pmuver_limit(void)
167+{
168+ return 0;
169+}
159170
160171 #endif
161172
include/kvm/arm_vgic.h+1−0
@@ -263,6 +263,7 @@ struct vgic_dist { · 262 unmodified lines
263263 struct vgic_io_device dist_iodev;
264264
265265 bool has_its;
266+ bool save_its_tables_in_progress;
266267
267268 /*
268269 * Contains the attributes and gpa of the LPI configuration table.
include/linux/kernel-page-flags.h+1−0
@@ -18,5 +18,6 @@ · 17 unmodified lines
1818 #define KPF_UNCACHED 39
1919 #define KPF_SOFTDIRTY 40
2020 #define KPF_ARCH_2 41
21+#define KPF_ARCH_3 42
2122
2223 #endif /* LINUX_KERNEL_PAGE_FLAGS_H */
include/linux/kvm_dirty_ring.h+12−8
@@ -37,6 +37,11 @@ static inline u32 kvm_dirty_ring_get_rsvd_entries(void) · 36 unmodified lines
3737 return 0;
3838 }
3939
40+static inline bool kvm_use_dirty_bitmap(struct kvm *kvm)
41+{
42+ return true;
43+}
44+
4045 static inline int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring,
4146 int index, u32 size)
4247 {
@@ -49,7 +54,7 @@ static inline int kvm_dirty_ring_reset(struct kvm *kvm, · 6 unmodified lines
4954 return 0;
5055 }
5156
52−static inline void kvm_dirty_ring_push(struct kvm_dirty_ring *ring,
57+static inline void kvm_dirty_ring_push(struct kvm_vcpu *vcpu,
5358 u32 slot, u64 offset)
5459 {
5560 }
@@ -64,13 +69,11 @@ static inline void kvm_dirty_ring_free(struct kvm_dirty_ring *ring) · 8 unmodified lines
6469 {
6570 }
6671
67−static inline bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
68−{
69− return true;
70−}
71−
7272 #else /* CONFIG_HAVE_KVM_DIRTY_RING */
7373
74+int kvm_cpu_dirty_log_size(void);
75+bool kvm_use_dirty_bitmap(struct kvm *kvm);
76+bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm);
7477 u32 kvm_dirty_ring_get_rsvd_entries(void);
7578 int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size);
7679
@@ -84,13 +87,14 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring); · 7 unmodified lines
8487 * returns =0: successfully pushed
8588 * <0: unable to push, need to wait
8689 */
87−void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset);
90+void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset);
91+
92+bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu);
8893
8994 /* for use in vm_operations_struct */
9095 struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset);
9196
9297 void kvm_dirty_ring_free(struct kvm_dirty_ring *ring);
93−bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring);
9498
9599 #endif /* CONFIG_HAVE_KVM_DIRTY_RING */
96100
include/linux/kvm_host.h+6−4
@@ -163,10 +163,11 @@ static inline bool is_error_page(struct page *page) · 162 unmodified lines
163163 * Architecture-independent vcpu->requests bit members
164164 * Bits 3-7 are reserved for more arch-independent bits.
165165 */
166−#define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
167−#define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
168−#define KVM_REQ_UNBLOCK 2
169−#define KVM_REQUEST_ARCH_BASE 8
166+#define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
167+#define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
168+#define KVM_REQ_UNBLOCK 2
169+#define KVM_REQ_DIRTY_RING_SOFT_FULL 3
170+#define KVM_REQUEST_ARCH_BASE 8
170171
171172 /*
172173 * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to
@@ -795,6 +796,7 @@ struct kvm { · 622 unmodified lines
795796 pid_t userspace_pid;
796797 unsigned int max_halt_poll_ns;
797798 u32 dirty_ring_size;
799+ bool dirty_ring_with_bitmap;
798800 bool vm_bugged;
799801 bool vm_dead;
800802
include/linux/page-flags.h+2−1
@@ -132,8 +132,9 @@ enum pageflags { · 131 unmodified lines
132132 PG_young,
133133 PG_idle,
134134 #endif
135−#ifdef CONFIG_64BIT
135+#ifdef CONFIG_ARCH_USES_PG_ARCH_X
136136 PG_arch_2,
137+ PG_arch_3,
137138 #endif
138139 #ifdef CONFIG_KASAN_HW_TAGS
139140 PG_skip_kasan_poison,
include/trace/events/mmflags.h+5−4
@@ -91,10 +91,10 @@ · 90 unmodified lines
9191 #define IF_HAVE_PG_IDLE(flag,string)
9292 #endif
9393
94−#ifdef CONFIG_64BIT
95−#define IF_HAVE_PG_ARCH_2(flag,string) ,{1UL << flag, string}
94+#ifdef CONFIG_ARCH_USES_PG_ARCH_X
95+#define IF_HAVE_PG_ARCH_X(flag,string) ,{1UL << flag, string}
9696 #else
97−#define IF_HAVE_PG_ARCH_2(flag,string)
97+#define IF_HAVE_PG_ARCH_X(flag,string)
9898 #endif
9999
100100 #ifdef CONFIG_KASAN_HW_TAGS
@@ -130,7 +130,8 @@ IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \ · 29 unmodified lines
130130 IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \
131131 IF_HAVE_PG_IDLE(PG_young, "young" ) \
132132 IF_HAVE_PG_IDLE(PG_idle, "idle" ) \
133−IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) \
133+IF_HAVE_PG_ARCH_X(PG_arch_2, "arch_2" ) \
134+IF_HAVE_PG_ARCH_X(PG_arch_3, "arch_3" ) \
134135 IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison")
135136
136137 #define show_page_flags(flags) \
include/uapi/linux/kvm.h+1−0
@@ -1182,6 +1182,7 @@ struct kvm_ppc_resize_hpt { · 1181 unmodified lines
11821182 #define KVM_CAP_S390_CPU_TOPOLOGY 222
11831183 #define KVM_CAP_DIRTY_LOG_RING_ACQ_REL 223
11841184 #define KVM_CAP_S390_PROTECTED_ASYNC_DISABLE 224
1185+#define KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP 225
11851186
11861187 #ifdef KVM_CAP_IRQ_ROUTING
11871188
mm/Kconfig+8−0
@@ -1005,6 +1005,14 @@ config ARCH_USES_HIGH_VMA_FLAGS · 1004 unmodified lines
10051005 config ARCH_HAS_PKEYS
10061006 bool
10071007
1008+config ARCH_USES_PG_ARCH_X
1009+ bool
1010+ help
1011+ Enable the definition of PG_arch_x page flags with x > 1. Only
1012+ suitable for 64-bit architectures with CONFIG_FLATMEM or
1013+ CONFIG_SPARSEMEM_VMEMMAP enabled, otherwise there may not be
1014+ enough room for additional bits in page->flags.
1015+
10081016 config VM_EVENT_COUNTERS
10091017 default y
10101018 bool "Enable VM event counters for /proc/vmstat" if EXPERT
mm/huge_memory.c+2−1
@@ -2444,8 +2444,9 @@ static void __split_huge_page_tail(struct page *head, int tail, · 2443 unmodified lines
24442444 (1L << PG_workingset) |
24452445 (1L << PG_locked) |
24462446 (1L << PG_unevictable) |
2447−#ifdef CONFIG_64BIT
2447+#ifdef CONFIG_ARCH_USES_PG_ARCH_X
24482448 (1L << PG_arch_2) |
2449+ (1L << PG_arch_3) |
24492450 #endif
24502451 (1L << PG_dirty) |
24512452 LRU_GEN_MASK | LRU_REFS_MASK));
tools/include/linux/bitfield.h+176−0
@@ -0,0 +1,176 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2014 Felix Fietkau <nbd@nbd.name>
4+ * Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
5+ */
6+
7+#ifndef _LINUX_BITFIELD_H
8+#define _LINUX_BITFIELD_H
9+
10+#include <linux/build_bug.h>
11+#include <asm/byteorder.h>
12+
13+/*
14+ * Bitfield access macros
15+ *
16+ * FIELD_{GET,PREP} macros take as first parameter shifted mask
17+ * from which they extract the base mask and shift amount.
18+ * Mask must be a compilation time constant.
19+ *
20+ * Example:
21+ *
22+ * #define REG_FIELD_A GENMASK(6, 0)
23+ * #define REG_FIELD_B BIT(7)
24+ * #define REG_FIELD_C GENMASK(15, 8)
25+ * #define REG_FIELD_D GENMASK(31, 16)
26+ *
27+ * Get:
28+ * a = FIELD_GET(REG_FIELD_A, reg);
29+ * b = FIELD_GET(REG_FIELD_B, reg);
30+ *
31+ * Set:
32+ * reg = FIELD_PREP(REG_FIELD_A, 1) |
33+ * FIELD_PREP(REG_FIELD_B, 0) |
34+ * FIELD_PREP(REG_FIELD_C, c) |
35+ * FIELD_PREP(REG_FIELD_D, 0x40);
36+ *
37+ * Modify:
38+ * reg &= ~REG_FIELD_C;
39+ * reg |= FIELD_PREP(REG_FIELD_C, c);
40+ */
41+
42+#define __bf_shf(x) (__builtin_ffsll(x) - 1)
43+
44+#define __scalar_type_to_unsigned_cases(type) \
45+ unsigned type: (unsigned type)0, \
46+ signed type: (unsigned type)0
47+
48+#define __unsigned_scalar_typeof(x) typeof( \
49+ _Generic((x), \
50+ char: (unsigned char)0, \
51+ __scalar_type_to_unsigned_cases(char), \
52+ __scalar_type_to_unsigned_cases(short), \
53+ __scalar_type_to_unsigned_cases(int), \
54+ __scalar_type_to_unsigned_cases(long), \
55+ __scalar_type_to_unsigned_cases(long long), \
56+ default: (x)))
57+
58+#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x))
59+
60+#define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \
61+ ({ \
62+ BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \
63+ _pfx "mask is not constant"); \
64+ BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero"); \
65+ BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \
66+ ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \
67+ _pfx "value too large for the field"); \
68+ BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \
69+ __bf_cast_unsigned(_reg, ~0ull), \
70+ _pfx "type of reg too small for mask"); \
71+ __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \
72+ (1ULL << __bf_shf(_mask))); \
73+ })
74+
75+/**
76+ * FIELD_MAX() - produce the maximum value representable by a field
77+ * @_mask: shifted mask defining the field's length and position
78+ *
79+ * FIELD_MAX() returns the maximum value that can be held in the field
80+ * specified by @_mask.
81+ */
82+#define FIELD_MAX(_mask) \
83+ ({ \
84+ __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_MAX: "); \
85+ (typeof(_mask))((_mask) >> __bf_shf(_mask)); \
86+ })
87+
88+/**
89+ * FIELD_FIT() - check if value fits in the field
90+ * @_mask: shifted mask defining the field's length and position
91+ * @_val: value to test against the field
92+ *
93+ * Return: true if @_val can fit inside @_mask, false if @_val is too big.
94+ */
95+#define FIELD_FIT(_mask, _val) \
96+ ({ \
97+ __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_FIT: "); \
98+ !((((typeof(_mask))_val) << __bf_shf(_mask)) & ~(_mask)); \
99+ })
100+
101+/**
102+ * FIELD_PREP() - prepare a bitfield element
103+ * @_mask: shifted mask defining the field's length and position
104+ * @_val: value to put in the field
105+ *
106+ * FIELD_PREP() masks and shifts up the value. The result should
107+ * be combined with other fields of the bitfield using logical OR.
108+ */
109+#define FIELD_PREP(_mask, _val) \
110+ ({ \
111+ __BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: "); \
112+ ((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask); \
113+ })
114+
115+/**
116+ * FIELD_GET() - extract a bitfield element
117+ * @_mask: shifted mask defining the field's length and position
118+ * @_reg: value of entire bitfield
119+ *
120+ * FIELD_GET() extracts the field specified by @_mask from the
121+ * bitfield passed in as @_reg by masking and shifting it down.
122+ */
123+#define FIELD_GET(_mask, _reg) \
124+ ({ \
125+ __BF_FIELD_CHECK(_mask, _reg, 0U, "FIELD_GET: "); \
126+ (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
127+ })
128+
129+extern void __compiletime_error("value doesn't fit into mask")
130+__field_overflow(void);
131+extern void __compiletime_error("bad bitfield mask")
132+__bad_mask(void);
133+static __always_inline u64 field_multiplier(u64 field)
134+{
135+ if ((field | (field - 1)) & ((field | (field - 1)) + 1))
136+ __bad_mask();
137+ return field & -field;
138+}
139+static __always_inline u64 field_mask(u64 field)
140+{
141+ return field / field_multiplier(field);
142+}
143+#define field_max(field) ((typeof(field))field_mask(field))
144+#define ____MAKE_OP(type,base,to,from) \
145+static __always_inline __##type type##_encode_bits(base v, base field) \
146+{ \
147+ if (__builtin_constant_p(v) && (v & ~field_mask(field))) \
148+ __field_overflow(); \
149+ return to((v & field_mask(field)) * field_multiplier(field)); \
150+} \
151+static __always_inline __##type type##_replace_bits(__##type old, \
152+ base val, base field) \
153+{ \
154+ return (old & ~to(field)) | type##_encode_bits(val, field); \
155+} \
156+static __always_inline void type##p_replace_bits(__##type *p, \
157+ base val, base field) \
158+{ \
159+ *p = (*p & ~to(field)) | type##_encode_bits(val, field); \
160+} \
161+static __always_inline base type##_get_bits(__##type v, base field) \
162+{ \
163+ return (from(v) & field)/field_multiplier(field); \
164+}
165+#define __MAKE_OP(size) \
166+ ____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu) \
167+ ____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu) \
168+ ____MAKE_OP(u##size,u##size,,)
169+____MAKE_OP(u8,u8,,)
170+__MAKE_OP(16)
171+__MAKE_OP(32)
172+__MAKE_OP(64)
173+#undef __MAKE_OP
174+#undef ____MAKE_OP
175+
176+#endif
tools/testing/selftests/kvm/.gitignore+1−0
@@ -4,6 +4,7 @@ · 3 unmodified lines
44 /aarch64/debug-exceptions
55 /aarch64/get-reg-list
66 /aarch64/hypercalls
7+/aarch64/page_fault_test
78 /aarch64/psci_test
89 /aarch64/vcpu_width_config
910 /aarch64/vgic_init
tools/testing/selftests/kvm/Makefile+3−0
@@ -48,6 +48,7 @@ LIBKVM += lib/rbtree.c · 47 unmodified lines
4848 LIBKVM += lib/sparsebit.c
4949 LIBKVM += lib/test_util.c
5050 LIBKVM += lib/ucall_common.c
51+LIBKVM += lib/userfaultfd_util.c
5152
5253 LIBKVM_STRING += lib/string_override.c
5354
@@ -158,10 +159,12 @@ TEST_GEN_PROGS_aarch64 += aarch64/arch_timer · 104 unmodified lines
158159 TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions
159160 TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list
160161 TEST_GEN_PROGS_aarch64 += aarch64/hypercalls
162+TEST_GEN_PROGS_aarch64 += aarch64/page_fault_test
161163 TEST_GEN_PROGS_aarch64 += aarch64/psci_test
162164 TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config
163165 TEST_GEN_PROGS_aarch64 += aarch64/vgic_init
164166 TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq
167+TEST_GEN_PROGS_aarch64 += access_tracking_perf_test
165168 TEST_GEN_PROGS_aarch64 += demand_paging_test
166169 TEST_GEN_PROGS_aarch64 += dirty_log_test
167170 TEST_GEN_PROGS_aarch64 += dirty_log_perf_test
tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c+2−1
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include "kvm_util.h"
1414 #include "processor.h"
1515 #include "test_util.h"
16+#include <linux/bitfield.h>
1617
1718 #define BAD_ID_REG_VAL 0x1badc0deul
1819
@@ -145,7 +146,7 @@ static bool vcpu_aarch64_only(struct kvm_vcpu *vcpu) · 126 unmodified lines
145146
146147 vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1), &val);
147148
148− el0 = (val & ARM64_FEATURE_MASK(ID_AA64PFR0_EL0)) >> ID_AA64PFR0_EL0_SHIFT;
149+ el0 = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0), val);
149150 return el0 == ID_AA64PFR0_ELx_64BIT_ONLY;
150151 }
151152
tools/testing/selftests/kvm/aarch64/debug-exceptions.c+239−72
@@ -2,6 +2,7 @@ · 1 unmodified line
22 #include <test_util.h>
33 #include <kvm_util.h>
44 #include <processor.h>
5+#include <linux/bitfield.h>
56
67 #define MDSCR_KDE (1 << 13)
78 #define MDSCR_MDE (1 << 15)
@@ -11,17 +12,24 @@ · 3 unmodified lines
1112 #define DBGBCR_EXEC (0x0 << 3)
1213 #define DBGBCR_EL1 (0x1 << 1)
1314 #define DBGBCR_E (0x1 << 0)
15+#define DBGBCR_LBN_SHIFT 16
16+#define DBGBCR_BT_SHIFT 20
17+#define DBGBCR_BT_ADDR_LINK_CTX (0x1 << DBGBCR_BT_SHIFT)
18+#define DBGBCR_BT_CTX_LINK (0x3 << DBGBCR_BT_SHIFT)
1419
1520 #define DBGWCR_LEN8 (0xff << 5)
1621 #define DBGWCR_RD (0x1 << 3)
1722 #define DBGWCR_WR (0x2 << 3)
1823 #define DBGWCR_EL1 (0x1 << 1)
1924 #define DBGWCR_E (0x1 << 0)
25+#define DBGWCR_LBN_SHIFT 16
26+#define DBGWCR_WT_SHIFT 20
27+#define DBGWCR_WT_LINK (0x1 << DBGWCR_WT_SHIFT)
2028
2129 #define SPSR_D (1 << 9)
2230 #define SPSR_SS (1 << 21)
2331
24−extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start;
32+extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start, hw_bp_ctx;
2533 extern unsigned char iter_ss_begin, iter_ss_end;
2634 static volatile uint64_t sw_bp_addr, hw_bp_addr;
2735 static volatile uint64_t wp_addr, wp_data_addr;
@@ -29,8 +37,74 @@ static volatile uint64_t svc_addr; · 1 unmodified line
2937 static volatile uint64_t ss_addr[4], ss_idx;
3038 #define PC(v) ((uint64_t)&(v))
3139
40+#define GEN_DEBUG_WRITE_REG(reg_name) \
41+static void write_##reg_name(int num, uint64_t val) \
42+{ \
43+ switch (num) { \
44+ case 0: \
45+ write_sysreg(val, reg_name##0_el1); \
46+ break; \
47+ case 1: \
48+ write_sysreg(val, reg_name##1_el1); \
49+ break; \
50+ case 2: \
51+ write_sysreg(val, reg_name##2_el1); \
52+ break; \
53+ case 3: \
54+ write_sysreg(val, reg_name##3_el1); \
55+ break; \
56+ case 4: \
57+ write_sysreg(val, reg_name##4_el1); \
58+ break; \
59+ case 5: \
60+ write_sysreg(val, reg_name##5_el1); \
61+ break; \
62+ case 6: \
63+ write_sysreg(val, reg_name##6_el1); \
64+ break; \
65+ case 7: \
66+ write_sysreg(val, reg_name##7_el1); \
67+ break; \
68+ case 8: \
69+ write_sysreg(val, reg_name##8_el1); \
70+ break; \
71+ case 9: \
72+ write_sysreg(val, reg_name##9_el1); \
73+ break; \
74+ case 10: \
75+ write_sysreg(val, reg_name##10_el1); \
76+ break; \
77+ case 11: \
78+ write_sysreg(val, reg_name##11_el1); \
79+ break; \
80+ case 12: \
81+ write_sysreg(val, reg_name##12_el1); \
82+ break; \
83+ case 13: \
84+ write_sysreg(val, reg_name##13_el1); \
85+ break; \
86+ case 14: \
87+ write_sysreg(val, reg_name##14_el1); \
88+ break; \
89+ case 15: \
90+ write_sysreg(val, reg_name##15_el1); \
91+ break; \
92+ default: \
93+ GUEST_ASSERT(0); \
94+ } \
95+}
96+
97+/* Define write_dbgbcr()/write_dbgbvr()/write_dbgwcr()/write_dbgwvr() */
98+GEN_DEBUG_WRITE_REG(dbgbcr)
99+GEN_DEBUG_WRITE_REG(dbgbvr)
100+GEN_DEBUG_WRITE_REG(dbgwcr)
101+GEN_DEBUG_WRITE_REG(dbgwvr)
102+
32103 static void reset_debug_state(void)
33104 {
105+ uint8_t brps, wrps, i;
106+ uint64_t dfr0;
107+
34108 asm volatile("msr daifset, #8");
35109
36110 write_sysreg(0, osdlr_el1);
@@ -38,11 +112,21 @@ static void reset_debug_state(void) · 1 unmodified line
38112 isb();
39113
40114 write_sysreg(0, mdscr_el1);
41− /* This test only uses the first bp and wp slot. */
42− write_sysreg(0, dbgbvr0_el1);
43− write_sysreg(0, dbgbcr0_el1);
44− write_sysreg(0, dbgwcr0_el1);
45− write_sysreg(0, dbgwvr0_el1);
115+ write_sysreg(0, contextidr_el1);
116+
117+ /* Reset all bcr/bvr/wcr/wvr registers */
118+ dfr0 = read_sysreg(id_aa64dfr0_el1);
119+ brps = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_BRPS), dfr0);
120+ for (i = 0; i <= brps; i++) {
121+ write_dbgbcr(i, 0);
122+ write_dbgbvr(i, 0);
123+ }
124+ wrps = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_WRPS), dfr0);
125+ for (i = 0; i <= wrps; i++) {
126+ write_dbgwcr(i, 0);
127+ write_dbgwvr(i, 0);
128+ }
129+
46130 isb();
47131 }
48132
@@ -54,16 +138,10 @@ static void enable_os_lock(void) · 5 unmodified lines
54138 GUEST_ASSERT(read_sysreg(oslsr_el1) & 2);
55139 }
56140
57−static void install_wp(uint64_t addr)
141+static void enable_monitor_debug_exceptions(void)
58142 {
59− uint32_t wcr;
60143 uint32_t mdscr;
61144
62− wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E;
63− write_sysreg(wcr, dbgwcr0_el1);
64− write_sysreg(addr, dbgwvr0_el1);
65− isb();
66−
67145 asm volatile("msr daifclr, #8");
68146
69147 mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE;
@@ -71,21 +149,76 @@ static void install_wp(uint64_t addr) · 1 unmodified line
71149 isb();
72150 }
73151
74−static void install_hw_bp(uint64_t addr)
152+static void install_wp(uint8_t wpn, uint64_t addr)
153+{
154+ uint32_t wcr;
155+
156+ wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E;
157+ write_dbgwcr(wpn, wcr);
158+ write_dbgwvr(wpn, addr);
159+
160+ isb();
161+
162+ enable_monitor_debug_exceptions();
163+}
164+
165+static void install_hw_bp(uint8_t bpn, uint64_t addr)
75166 {
76167 uint32_t bcr;
77− uint32_t mdscr;
78168
79169 bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E;
80− write_sysreg(bcr, dbgbcr0_el1);
81− write_sysreg(addr, dbgbvr0_el1);
170+ write_dbgbcr(bpn, bcr);
171+ write_dbgbvr(bpn, addr);
82172 isb();
83173
84− asm volatile("msr daifclr, #8");
174+ enable_monitor_debug_exceptions();
175+}
85176
86− mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE;
87− write_sysreg(mdscr, mdscr_el1);
177+static void install_wp_ctx(uint8_t addr_wp, uint8_t ctx_bp, uint64_t addr,
178+ uint64_t ctx)
179+{
180+ uint32_t wcr;
181+ uint64_t ctx_bcr;
182+
183+ /* Setup a context-aware breakpoint for Linked Context ID Match */
184+ ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
185+ DBGBCR_BT_CTX_LINK;
186+ write_dbgbcr(ctx_bp, ctx_bcr);
187+ write_dbgbvr(ctx_bp, ctx);
188+
189+ /* Setup a linked watchpoint (linked to the context-aware breakpoint) */
190+ wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E |
191+ DBGWCR_WT_LINK | ((uint32_t)ctx_bp << DBGWCR_LBN_SHIFT);
192+ write_dbgwcr(addr_wp, wcr);
193+ write_dbgwvr(addr_wp, addr);
88194 isb();
195+
196+ enable_monitor_debug_exceptions();
197+}
198+
199+void install_hw_bp_ctx(uint8_t addr_bp, uint8_t ctx_bp, uint64_t addr,
200+ uint64_t ctx)
201+{
202+ uint32_t addr_bcr, ctx_bcr;
203+
204+ /* Setup a context-aware breakpoint for Linked Context ID Match */
205+ ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
206+ DBGBCR_BT_CTX_LINK;
207+ write_dbgbcr(ctx_bp, ctx_bcr);
208+ write_dbgbvr(ctx_bp, ctx);
209+
210+ /*
211+ * Setup a normal breakpoint for Linked Address Match, and link it
212+ * to the context-aware breakpoint.
213+ */
214+ addr_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
215+ DBGBCR_BT_ADDR_LINK_CTX |
216+ ((uint32_t)ctx_bp << DBGBCR_LBN_SHIFT);
217+ write_dbgbcr(addr_bp, addr_bcr);
218+ write_dbgbvr(addr_bp, addr);
219+ isb();
220+
221+ enable_monitor_debug_exceptions();
89222 }
90223
91224 static void install_ss(void)
@@ -101,52 +234,42 @@ static void install_ss(void) · 9 unmodified lines
101234
102235 static volatile char write_data;
103236
104−static void guest_code(void)
237+static void guest_code(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
105238 {
106− GUEST_SYNC(0);
239+ uint64_t ctx = 0xabcdef; /* a random context number */
107240
108241 /* Software-breakpoint */
109242 reset_debug_state();
110243 asm volatile("sw_bp: brk #0");
111244 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp));
112245
113− GUEST_SYNC(1);
114−
115246 /* Hardware-breakpoint */
116247 reset_debug_state();
117− install_hw_bp(PC(hw_bp));
248+ install_hw_bp(bpn, PC(hw_bp));
118249 asm volatile("hw_bp: nop");
119250 GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp));
120251
121− GUEST_SYNC(2);
122−
123252 /* Hardware-breakpoint + svc */
124253 reset_debug_state();
125− install_hw_bp(PC(bp_svc));
254+ install_hw_bp(bpn, PC(bp_svc));
126255 asm volatile("bp_svc: svc #0");
127256 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_svc));
128257 GUEST_ASSERT_EQ(svc_addr, PC(bp_svc) + 4);
129258
130− GUEST_SYNC(3);
131−
132259 /* Hardware-breakpoint + software-breakpoint */
133260 reset_debug_state();
134− install_hw_bp(PC(bp_brk));
261+ install_hw_bp(bpn, PC(bp_brk));
135262 asm volatile("bp_brk: brk #0");
136263 GUEST_ASSERT_EQ(sw_bp_addr, PC(bp_brk));
137264 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_brk));
138265
139− GUEST_SYNC(4);
140−
141266 /* Watchpoint */
142267 reset_debug_state();
143− install_wp(PC(write_data));
268+ install_wp(wpn, PC(write_data));
144269 write_data = 'x';
145270 GUEST_ASSERT_EQ(write_data, 'x');
146271 GUEST_ASSERT_EQ(wp_data_addr, PC(write_data));
147272
148− GUEST_SYNC(5);
149−
150273 /* Single-step */
151274 reset_debug_state();
152275 install_ss();
@@ -160,8 +283,6 @@ static void guest_code(void) · 7 unmodified lines
160283 GUEST_ASSERT_EQ(ss_addr[1], PC(ss_start) + 4);
161284 GUEST_ASSERT_EQ(ss_addr[2], PC(ss_start) + 8);
162285
163− GUEST_SYNC(6);
164−
165286 /* OS Lock does not block software-breakpoint */
166287 reset_debug_state();
167288 enable_os_lock();
@@ -169,30 +290,24 @@ static void guest_code(void) · 1 unmodified line
169290 asm volatile("sw_bp2: brk #0");
170291 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp2));
171292
172− GUEST_SYNC(7);
173−
174293 /* OS Lock blocking hardware-breakpoint */
175294 reset_debug_state();
176295 enable_os_lock();
177− install_hw_bp(PC(hw_bp2));
296+ install_hw_bp(bpn, PC(hw_bp2));
178297 hw_bp_addr = 0;
179298 asm volatile("hw_bp2: nop");
180299 GUEST_ASSERT_EQ(hw_bp_addr, 0);
181300
182− GUEST_SYNC(8);
183−
184301 /* OS Lock blocking watchpoint */
185302 reset_debug_state();
186303 enable_os_lock();
187304 write_data = '\0';
188305 wp_data_addr = 0;
189− install_wp(PC(write_data));
306+ install_wp(wpn, PC(write_data));
190307 write_data = 'x';
191308 GUEST_ASSERT_EQ(write_data, 'x');
192309 GUEST_ASSERT_EQ(wp_data_addr, 0);
193310
194− GUEST_SYNC(9);
195−
196311 /* OS Lock blocking single-step */
197312 reset_debug_state();
198313 enable_os_lock();
@@ -205,6 +320,27 @@ static void guest_code(void) · 6 unmodified lines
205320 : : : "x0");
206321 GUEST_ASSERT_EQ(ss_addr[0], 0);
207322
323+ /* Linked hardware-breakpoint */
324+ hw_bp_addr = 0;
325+ reset_debug_state();
326+ install_hw_bp_ctx(bpn, ctx_bpn, PC(hw_bp_ctx), ctx);
327+ /* Set context id */
328+ write_sysreg(ctx, contextidr_el1);
329+ isb();
330+ asm volatile("hw_bp_ctx: nop");
331+ write_sysreg(0, contextidr_el1);
332+ GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp_ctx));
333+
334+ /* Linked watchpoint */
335+ reset_debug_state();
336+ install_wp_ctx(wpn, ctx_bpn, PC(write_data), ctx);
337+ /* Set context id */
338+ write_sysreg(ctx, contextidr_el1);
339+ isb();
340+ write_data = 'x';
341+ GUEST_ASSERT_EQ(write_data, 'x');
342+ GUEST_ASSERT_EQ(wp_data_addr, PC(write_data));
343+
208344 GUEST_DONE();
209345 }
210346
@@ -276,20 +412,16 @@ static void guest_code_ss(int test_cnt) · 65 unmodified lines
276412 GUEST_DONE();
277413 }
278414
279−static int debug_version(struct kvm_vcpu *vcpu)
415+static int debug_version(uint64_t id_aa64dfr0)
280416 {
281− uint64_t id_aa64dfr0;
282−
283− vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1), &id_aa64dfr0);
284− return id_aa64dfr0 & 0xf;
417+ return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), id_aa64dfr0);
285418 }
286419
287−static void test_guest_debug_exceptions(void)
420+static void test_guest_debug_exceptions(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
288421 {
289422 struct kvm_vcpu *vcpu;
290423 struct kvm_vm *vm;
291424 struct ucall uc;
292− int stage;
293425
294426 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
295427
@@ -307,23 +439,19 @@ static void test_guest_debug_exceptions(void) · 11 unmodified lines
307439 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
308440 ESR_EC_SVC64, guest_svc_handler);
309441
310− for (stage = 0; stage < 11; stage++) {
311− vcpu_run(vcpu);
312−
313− switch (get_ucall(vcpu, &uc)) {
314− case UCALL_SYNC:
315− TEST_ASSERT(uc.args[1] == stage,
316− "Stage %d: Unexpected sync ucall, got %lx",
317− stage, (ulong)uc.args[1]);
318− break;
319− case UCALL_ABORT:
320− REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");
321− break;
322− case UCALL_DONE:
323− goto done;
324− default:
325− TEST_FAIL("Unknown ucall %lu", uc.cmd);
326− }
442+ /* Specify bpn/wpn/ctx_bpn to be tested */
443+ vcpu_args_set(vcpu, 3, bpn, wpn, ctx_bpn);
444+ pr_debug("Use bpn#%d, wpn#%d and ctx_bpn#%d\n", bpn, wpn, ctx_bpn);
445+
446+ vcpu_run(vcpu);
447+ switch (get_ucall(vcpu, &uc)) {
448+ case UCALL_ABORT:
449+ REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");
450+ break;
451+ case UCALL_DONE:
452+ goto done;
453+ default:
454+ TEST_FAIL("Unknown ucall %lu", uc.cmd);
327455 }
328456
329457 done:
@@ -400,6 +528,43 @@ void test_single_step_from_userspace(int test_cnt) · 70 unmodified lines
400528 kvm_vm_free(vm);
401529 }
402530
531+/*
532+ * Run debug testing using the various breakpoint#, watchpoint# and
533+ * context-aware breakpoint# with the given ID_AA64DFR0_EL1 configuration.
534+ */
535+void test_guest_debug_exceptions_all(uint64_t aa64dfr0)
536+{
537+ uint8_t brp_num, wrp_num, ctx_brp_num, normal_brp_num, ctx_brp_base;
538+ int b, w, c;
539+
540+ /* Number of breakpoints */
541+ brp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_BRPS), aa64dfr0) + 1;
542+ __TEST_REQUIRE(brp_num >= 2, "At least two breakpoints are required");
543+
544+ /* Number of watchpoints */
545+ wrp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_WRPS), aa64dfr0) + 1;
546+
547+ /* Number of context aware breakpoints */
548+ ctx_brp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_CTX_CMPS), aa64dfr0) + 1;
549+
550+ pr_debug("%s brp_num:%d, wrp_num:%d, ctx_brp_num:%d\n", __func__,
551+ brp_num, wrp_num, ctx_brp_num);
552+
553+ /* Number of normal (non-context aware) breakpoints */
554+ normal_brp_num = brp_num - ctx_brp_num;
555+
556+ /* Lowest context aware breakpoint number */
557+ ctx_brp_base = normal_brp_num;
558+
559+ /* Run tests with all supported breakpoints/watchpoints */
560+ for (c = ctx_brp_base; c < ctx_brp_base + ctx_brp_num; c++) {
561+ for (b = 0; b < normal_brp_num; b++) {
562+ for (w = 0; w < wrp_num; w++)
563+ test_guest_debug_exceptions(b, w, c);
564+ }
565+ }
566+}
567+
403568 static void help(char *name)
404569 {
405570 puts("");
@@ -414,9 +579,11 @@ int main(int argc, char *argv[]) · 8 unmodified lines
414579 struct kvm_vm *vm;
415580 int opt;
416581 int ss_iteration = 10000;
582+ uint64_t aa64dfr0;
417583
418584 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
419− __TEST_REQUIRE(debug_version(vcpu) >= 6,
585+ vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1), &aa64dfr0);
586+ __TEST_REQUIRE(debug_version(aa64dfr0) >= 6,
420587 "Armv8 debug architecture not supported.");
421588 kvm_vm_free(vm);
422589
@@ -432,7 +599,7 @@ int main(int argc, char *argv[]) · 9 unmodified lines
432599 }
433600 }
434601
435− test_guest_debug_exceptions();
602+ test_guest_debug_exceptions_all(aa64dfr0);
436603 test_single_step_from_userspace(ss_iteration);
437604
438605 return 0;
tools/testing/selftests/kvm/aarch64/page_fault_test.c+1117−0
@@ -0,0 +1,1117 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * page_fault_test.c - Test stage 2 faults.
4+ *
5+ * This test tries different combinations of guest accesses (e.g., write,
6+ * S1PTW), backing source type (e.g., anon) and types of faults (e.g., read on
7+ * hugetlbfs with a hole). It checks that the expected handling method is
8+ * called (e.g., uffd faults with the right address and write/read flag).
9+ */
10+
11+#define _GNU_SOURCE
12+#include <linux/bitmap.h>
13+#include <fcntl.h>
14+#include <test_util.h>
15+#include <kvm_util.h>
16+#include <processor.h>
17+#include <asm/sysreg.h>
18+#include <linux/bitfield.h>
19+#include "guest_modes.h"
20+#include "userfaultfd_util.h"
21+
22+/* Guest virtual addresses that point to the test page and its PTE. */
23+#define TEST_GVA 0xc0000000
24+#define TEST_EXEC_GVA (TEST_GVA + 0x8)
25+#define TEST_PTE_GVA 0xb0000000
26+#define TEST_DATA 0x0123456789ABCDEF
27+
28+static uint64_t *guest_test_memory = (uint64_t *)TEST_GVA;
29+
30+#define CMD_NONE (0)
31+#define CMD_SKIP_TEST (1ULL << 1)
32+#define CMD_HOLE_PT (1ULL << 2)
33+#define CMD_HOLE_DATA (1ULL << 3)
34+#define CMD_CHECK_WRITE_IN_DIRTY_LOG (1ULL << 4)
35+#define CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG (1ULL << 5)
36+#define CMD_CHECK_NO_WRITE_IN_DIRTY_LOG (1ULL << 6)
37+#define CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG (1ULL << 7)
38+#define CMD_SET_PTE_AF (1ULL << 8)
39+
40+#define PREPARE_FN_NR 10
41+#define CHECK_FN_NR 10
42+
43+static struct event_cnt {
44+ int mmio_exits;
45+ int fail_vcpu_runs;
46+ int uffd_faults;
47+ /* uffd_faults is incremented from multiple threads. */
48+ pthread_mutex_t uffd_faults_mutex;
49+} events;
50+
51+struct test_desc {
52+ const char *name;
53+ uint64_t mem_mark_cmd;
54+ /* Skip the test if any prepare function returns false */
55+ bool (*guest_prepare[PREPARE_FN_NR])(void);
56+ void (*guest_test)(void);
57+ void (*guest_test_check[CHECK_FN_NR])(void);
58+ uffd_handler_t uffd_pt_handler;
59+ uffd_handler_t uffd_data_handler;
60+ void (*dabt_handler)(struct ex_regs *regs);
61+ void (*iabt_handler)(struct ex_regs *regs);
62+ void (*mmio_handler)(struct kvm_vm *vm, struct kvm_run *run);
63+ void (*fail_vcpu_run_handler)(int ret);
64+ uint32_t pt_memslot_flags;
65+ uint32_t data_memslot_flags;
66+ bool skip;
67+ struct event_cnt expected_events;
68+};
69+
70+struct test_params {
71+ enum vm_mem_backing_src_type src_type;
72+ struct test_desc *test_desc;
73+};
74+
75+static inline void flush_tlb_page(uint64_t vaddr)
76+{
77+ uint64_t page = vaddr >> 12;
78+
79+ dsb(ishst);
80+ asm volatile("tlbi vaae1is, %0" :: "r" (page));
81+ dsb(ish);
82+ isb();
83+}
84+
85+static void guest_write64(void)
86+{
87+ uint64_t val;
88+
89+ WRITE_ONCE(*guest_test_memory, TEST_DATA);
90+ val = READ_ONCE(*guest_test_memory);
91+ GUEST_ASSERT_EQ(val, TEST_DATA);
92+}
93+
94+/* Check the system for atomic instructions. */
95+static bool guest_check_lse(void)
96+{
97+ uint64_t isar0 = read_sysreg(id_aa64isar0_el1);
98+ uint64_t atomic;
99+
100+ atomic = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR0_ATOMICS), isar0);
101+ return atomic >= 2;
102+}
103+
104+static bool guest_check_dc_zva(void)
105+{
106+ uint64_t dczid = read_sysreg(dczid_el0);
107+ uint64_t dzp = FIELD_GET(ARM64_FEATURE_MASK(DCZID_DZP), dczid);
108+
109+ return dzp == 0;
110+}
111+
112+/* Compare and swap instruction. */
113+static void guest_cas(void)
114+{
115+ uint64_t val;
116+
117+ GUEST_ASSERT(guest_check_lse());
118+ asm volatile(".arch_extension lse\n"
119+ "casal %0, %1, [%2]\n"
120+ :: "r" (0), "r" (TEST_DATA), "r" (guest_test_memory));
121+ val = READ_ONCE(*guest_test_memory);
122+ GUEST_ASSERT_EQ(val, TEST_DATA);
123+}
124+
125+static void guest_read64(void)
126+{
127+ uint64_t val;
128+
129+ val = READ_ONCE(*guest_test_memory);
130+ GUEST_ASSERT_EQ(val, 0);
131+}
132+
133+/* Address translation instruction */
134+static void guest_at(void)
135+{
136+ uint64_t par;
137+
138+ asm volatile("at s1e1r, %0" :: "r" (guest_test_memory));
139+ par = read_sysreg(par_el1);
140+ isb();
141+
142+ /* Bit 1 indicates whether the AT was successful */
143+ GUEST_ASSERT_EQ(par & 1, 0);
144+}
145+
146+/*
147+ * The size of the block written by "dc zva" is guaranteed to be between (2 <<
148+ * 0) and (2 << 9), which is safe in our case as we need the write to happen
149+ * for at least a word, and not more than a page.
150+ */
151+static void guest_dc_zva(void)
152+{
153+ uint16_t val;
154+
155+ asm volatile("dc zva, %0" :: "r" (guest_test_memory));
156+ dsb(ish);
157+ val = READ_ONCE(*guest_test_memory);
158+ GUEST_ASSERT_EQ(val, 0);
159+}
160+
161+/*
162+ * Pre-indexing loads and stores don't have a valid syndrome (ESR_EL2.ISV==0).
163+ * And that's special because KVM must take special care with those: they
164+ * should still count as accesses for dirty logging or user-faulting, but
165+ * should be handled differently on mmio.
166+ */
167+static void guest_ld_preidx(void)
168+{
169+ uint64_t val;
170+ uint64_t addr = TEST_GVA - 8;
171+
172+ /*
173+ * This ends up accessing "TEST_GVA + 8 - 8", where "TEST_GVA - 8" is
174+ * in a gap between memslots not backing by anything.
175+ */
176+ asm volatile("ldr %0, [%1, #8]!"
177+ : "=r" (val), "+r" (addr));
178+ GUEST_ASSERT_EQ(val, 0);
179+ GUEST_ASSERT_EQ(addr, TEST_GVA);
180+}
181+
182+static void guest_st_preidx(void)
183+{
184+ uint64_t val = TEST_DATA;
185+ uint64_t addr = TEST_GVA - 8;
186+
187+ asm volatile("str %0, [%1, #8]!"
188+ : "+r" (val), "+r" (addr));
189+
190+ GUEST_ASSERT_EQ(addr, TEST_GVA);
191+ val = READ_ONCE(*guest_test_memory);
192+}
193+
194+static bool guest_set_ha(void)
195+{
196+ uint64_t mmfr1 = read_sysreg(id_aa64mmfr1_el1);
197+ uint64_t hadbs, tcr;
198+
199+ /* Skip if HA is not supported. */
200+ hadbs = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_HADBS), mmfr1);
201+ if (hadbs == 0)
202+ return false;
203+
204+ tcr = read_sysreg(tcr_el1) | TCR_EL1_HA;
205+ write_sysreg(tcr, tcr_el1);
206+ isb();
207+
208+ return true;
209+}
210+
211+static bool guest_clear_pte_af(void)
212+{
213+ *((uint64_t *)TEST_PTE_GVA) &= ~PTE_AF;
214+ flush_tlb_page(TEST_GVA);
215+
216+ return true;
217+}
218+
219+static void guest_check_pte_af(void)
220+{
221+ dsb(ish);
222+ GUEST_ASSERT_EQ(*((uint64_t *)TEST_PTE_GVA) & PTE_AF, PTE_AF);
223+}
224+
225+static void guest_check_write_in_dirty_log(void)
226+{
227+ GUEST_SYNC(CMD_CHECK_WRITE_IN_DIRTY_LOG);
228+}
229+
230+static void guest_check_no_write_in_dirty_log(void)
231+{
232+ GUEST_SYNC(CMD_CHECK_NO_WRITE_IN_DIRTY_LOG);
233+}
234+
235+static void guest_check_s1ptw_wr_in_dirty_log(void)
236+{
237+ GUEST_SYNC(CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG);
238+}
239+
240+static void guest_exec(void)
241+{
242+ int (*code)(void) = (int (*)(void))TEST_EXEC_GVA;
243+ int ret;
244+
245+ ret = code();
246+ GUEST_ASSERT_EQ(ret, 0x77);
247+}
248+
249+static bool guest_prepare(struct test_desc *test)
250+{
251+ bool (*prepare_fn)(void);
252+ int i;
253+
254+ for (i = 0; i < PREPARE_FN_NR; i++) {
255+ prepare_fn = test->guest_prepare[i];
256+ if (prepare_fn && !prepare_fn())
257+ return false;
258+ }
259+
260+ return true;
261+}
262+
263+static void guest_test_check(struct test_desc *test)
264+{
265+ void (*check_fn)(void);
266+ int i;
267+
268+ for (i = 0; i < CHECK_FN_NR; i++) {
269+ check_fn = test->guest_test_check[i];
270+ if (check_fn)
271+ check_fn();
272+ }
273+}
274+
275+static void guest_code(struct test_desc *test)
276+{
277+ if (!guest_prepare(test))
278+ GUEST_SYNC(CMD_SKIP_TEST);
279+
280+ GUEST_SYNC(test->mem_mark_cmd);
281+
282+ if (test->guest_test)
283+ test->guest_test();
284+
285+ guest_test_check(test);
286+ GUEST_DONE();
287+}
288+
289+static void no_dabt_handler(struct ex_regs *regs)
290+{
291+ GUEST_ASSERT_1(false, read_sysreg(far_el1));
292+}
293+
294+static void no_iabt_handler(struct ex_regs *regs)
295+{
296+ GUEST_ASSERT_1(false, regs->pc);
297+}
298+
299+static struct uffd_args {
300+ char *copy;
301+ void *hva;
302+ uint64_t paging_size;
303+} pt_args, data_args;
304+
305+/* Returns true to continue the test, and false if it should be skipped. */
306+static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg,
307+ struct uffd_args *args, bool expect_write)
308+{
309+ uint64_t addr = msg->arg.pagefault.address;
310+ uint64_t flags = msg->arg.pagefault.flags;
311+ struct uffdio_copy copy;
312+ int ret;
313+
314+ TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING,
315+ "The only expected UFFD mode is MISSING");
316+ ASSERT_EQ(!!(flags & UFFD_PAGEFAULT_FLAG_WRITE), expect_write);
317+ ASSERT_EQ(addr, (uint64_t)args->hva);
318+
319+ pr_debug("uffd fault: addr=%p write=%d\n",
320+ (void *)addr, !!(flags & UFFD_PAGEFAULT_FLAG_WRITE));
321+
322+ copy.src = (uint64_t)args->copy;
323+ copy.dst = addr;
324+ copy.len = args->paging_size;
325+ copy.mode = 0;
326+
327+ ret = ioctl(uffd, UFFDIO_COPY, &copy);
328+ if (ret == -1) {
329+ pr_info("Failed UFFDIO_COPY in 0x%lx with errno: %d\n",
330+ addr, errno);
331+ return ret;
332+ }
333+
334+ pthread_mutex_lock(&events.uffd_faults_mutex);
335+ events.uffd_faults += 1;
336+ pthread_mutex_unlock(&events.uffd_faults_mutex);
337+ return 0;
338+}
339+
340+static int uffd_pt_write_handler(int mode, int uffd, struct uffd_msg *msg)
341+{
342+ return uffd_generic_handler(mode, uffd, msg, &pt_args, true);
343+}
344+
345+static int uffd_data_write_handler(int mode, int uffd, struct uffd_msg *msg)
346+{
347+ return uffd_generic_handler(mode, uffd, msg, &data_args, true);
348+}
349+
350+static int uffd_data_read_handler(int mode, int uffd, struct uffd_msg *msg)
351+{
352+ return uffd_generic_handler(mode, uffd, msg, &data_args, false);
353+}
354+
355+static void setup_uffd_args(struct userspace_mem_region *region,
356+ struct uffd_args *args)
357+{
358+ args->hva = (void *)region->region.userspace_addr;
359+ args->paging_size = region->region.memory_size;
360+
361+ args->copy = malloc(args->paging_size);
362+ TEST_ASSERT(args->copy, "Failed to allocate data copy.");
363+ memcpy(args->copy, args->hva, args->paging_size);
364+}
365+
366+static void setup_uffd(struct kvm_vm *vm, struct test_params *p,
367+ struct uffd_desc **pt_uffd, struct uffd_desc **data_uffd)
368+{
369+ struct test_desc *test = p->test_desc;
370+ int uffd_mode = UFFDIO_REGISTER_MODE_MISSING;
371+
372+ setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_PT), &pt_args);
373+ setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_TEST_DATA), &data_args);
374+
375+ *pt_uffd = NULL;
376+ if (test->uffd_pt_handler)
377+ *pt_uffd = uffd_setup_demand_paging(uffd_mode, 0,
378+ pt_args.hva,
379+ pt_args.paging_size,
380+ test->uffd_pt_handler);
381+
382+ *data_uffd = NULL;
383+ if (test->uffd_data_handler)
384+ *data_uffd = uffd_setup_demand_paging(uffd_mode, 0,
385+ data_args.hva,
386+ data_args.paging_size,
387+ test->uffd_data_handler);
388+}
389+
390+static void free_uffd(struct test_desc *test, struct uffd_desc *pt_uffd,
391+ struct uffd_desc *data_uffd)
392+{
393+ if (test->uffd_pt_handler)
394+ uffd_stop_demand_paging(pt_uffd);
395+ if (test->uffd_data_handler)
396+ uffd_stop_demand_paging(data_uffd);
397+
398+ free(pt_args.copy);
399+ free(data_args.copy);
400+}
401+
402+static int uffd_no_handler(int mode, int uffd, struct uffd_msg *msg)
403+{
404+ TEST_FAIL("There was no UFFD fault expected.");
405+ return -1;
406+}
407+
408+/* Returns false if the test should be skipped. */
409+static bool punch_hole_in_backing_store(struct kvm_vm *vm,
410+ struct userspace_mem_region *region)
411+{
412+ void *hva = (void *)region->region.userspace_addr;
413+ uint64_t paging_size = region->region.memory_size;
414+ int ret, fd = region->fd;
415+
416+ if (fd != -1) {
417+ ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
418+ 0, paging_size);
419+ TEST_ASSERT(ret == 0, "fallocate failed\n");
420+ } else {
421+ ret = madvise(hva, paging_size, MADV_DONTNEED);
422+ TEST_ASSERT(ret == 0, "madvise failed\n");
423+ }
424+
425+ return true;
426+}
427+
428+static void mmio_on_test_gpa_handler(struct kvm_vm *vm, struct kvm_run *run)
429+{
430+ struct userspace_mem_region *region;
431+ void *hva;
432+
433+ region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
434+ hva = (void *)region->region.userspace_addr;
435+
436+ ASSERT_EQ(run->mmio.phys_addr, region->region.guest_phys_addr);
437+
438+ memcpy(hva, run->mmio.data, run->mmio.len);
439+ events.mmio_exits += 1;
440+}
441+
442+static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run)
443+{
444+ uint64_t data;
445+
446+ memcpy(&data, run->mmio.data, sizeof(data));
447+ pr_debug("addr=%lld len=%d w=%d data=%lx\n",
448+ run->mmio.phys_addr, run->mmio.len,
449+ run->mmio.is_write, data);
450+ TEST_FAIL("There was no MMIO exit expected.");
451+}
452+
453+static bool check_write_in_dirty_log(struct kvm_vm *vm,
454+ struct userspace_mem_region *region,
455+ uint64_t host_pg_nr)
456+{
457+ unsigned long *bmap;
458+ bool first_page_dirty;
459+ uint64_t size = region->region.memory_size;
460+
461+ /* getpage_size() is not always equal to vm->page_size */
462+ bmap = bitmap_zalloc(size / getpagesize());
463+ kvm_vm_get_dirty_log(vm, region->region.slot, bmap);
464+ first_page_dirty = test_bit(host_pg_nr, bmap);
465+ free(bmap);
466+ return first_page_dirty;
467+}
468+
469+/* Returns true to continue the test, and false if it should be skipped. */
470+static bool handle_cmd(struct kvm_vm *vm, int cmd)
471+{
472+ struct userspace_mem_region *data_region, *pt_region;
473+ bool continue_test = true;
474+
475+ data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
476+ pt_region = vm_get_mem_region(vm, MEM_REGION_PT);
477+
478+ if (cmd == CMD_SKIP_TEST)
479+ continue_test = false;
480+
481+ if (cmd & CMD_HOLE_PT)
482+ continue_test = punch_hole_in_backing_store(vm, pt_region);
483+ if (cmd & CMD_HOLE_DATA)
484+ continue_test = punch_hole_in_backing_store(vm, data_region);
485+ if (cmd & CMD_CHECK_WRITE_IN_DIRTY_LOG)
486+ TEST_ASSERT(check_write_in_dirty_log(vm, data_region, 0),
487+ "Missing write in dirty log");
488+ if (cmd & CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG)
489+ TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, 0),
490+ "Missing s1ptw write in dirty log");
491+ if (cmd & CMD_CHECK_NO_WRITE_IN_DIRTY_LOG)
492+ TEST_ASSERT(!check_write_in_dirty_log(vm, data_region, 0),
493+ "Unexpected write in dirty log");
494+ if (cmd & CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG)
495+ TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, 0),
496+ "Unexpected s1ptw write in dirty log");
497+
498+ return continue_test;
499+}
500+
501+void fail_vcpu_run_no_handler(int ret)
502+{
503+ TEST_FAIL("Unexpected vcpu run failure\n");
504+}
505+
506+void fail_vcpu_run_mmio_no_syndrome_handler(int ret)
507+{
508+ TEST_ASSERT(errno == ENOSYS,
509+ "The mmio handler should have returned not implemented.");
510+ events.fail_vcpu_runs += 1;
511+}
512+
513+typedef uint32_t aarch64_insn_t;
514+extern aarch64_insn_t __exec_test[2];
515+
516+noinline void __return_0x77(void)
517+{
518+ asm volatile("__exec_test: mov x0, #0x77\n"
519+ "ret\n");
520+}
521+
522+/*
523+ * Note that this function runs on the host before the test VM starts: there's
524+ * no need to sync the D$ and I$ caches.
525+ */
526+static void load_exec_code_for_test(struct kvm_vm *vm)
527+{
528+ uint64_t *code;
529+ struct userspace_mem_region *region;
530+ void *hva;
531+
532+ region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
533+ hva = (void *)region->region.userspace_addr;
534+
535+ assert(TEST_EXEC_GVA > TEST_GVA);
536+ code = hva + TEST_EXEC_GVA - TEST_GVA;
537+ memcpy(code, __exec_test, sizeof(__exec_test));
538+}
539+
540+static void setup_abort_handlers(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
541+ struct test_desc *test)
542+{
543+ vm_init_descriptor_tables(vm);
544+ vcpu_init_descriptor_tables(vcpu);
545+
546+ vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
547+ ESR_EC_DABT, no_dabt_handler);
548+ vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
549+ ESR_EC_IABT, no_iabt_handler);
550+}
551+
552+static void setup_gva_maps(struct kvm_vm *vm)
553+{
554+ struct userspace_mem_region *region;
555+ uint64_t pte_gpa;
556+
557+ region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
558+ /* Map TEST_GVA first. This will install a new PTE. */
559+ virt_pg_map(vm, TEST_GVA, region->region.guest_phys_addr);
560+ /* Then map TEST_PTE_GVA to the above PTE. */
561+ pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA));
562+ virt_pg_map(vm, TEST_PTE_GVA, pte_gpa);
563+}
564+
565+enum pf_test_memslots {
566+ CODE_AND_DATA_MEMSLOT,
567+ PAGE_TABLE_MEMSLOT,
568+ TEST_DATA_MEMSLOT,
569+};
570+
571+/*
572+ * Create a memslot for code and data at pfn=0, and test-data and PT ones
573+ * at max_gfn.
574+ */
575+static void setup_memslots(struct kvm_vm *vm, struct test_params *p)
576+{
577+ uint64_t backing_src_pagesz = get_backing_src_pagesz(p->src_type);
578+ uint64_t guest_page_size = vm->page_size;
579+ uint64_t max_gfn = vm_compute_max_gfn(vm);
580+ /* Enough for 2M of code when using 4K guest pages. */
581+ uint64_t code_npages = 512;
582+ uint64_t pt_size, data_size, data_gpa;
583+
584+ /*
585+ * This test requires 1 pgd, 2 pud, 4 pmd, and 6 pte pages when using
586+ * VM_MODE_P48V48_4K. Note that the .text takes ~1.6MBs. That's 13
587+ * pages. VM_MODE_P48V48_4K is the mode with most PT pages; let's use
588+ * twice that just in case.
589+ */
590+ pt_size = 26 * guest_page_size;
591+
592+ /* memslot sizes and gpa's must be aligned to the backing page size */
593+ pt_size = align_up(pt_size, backing_src_pagesz);
594+ data_size = align_up(guest_page_size, backing_src_pagesz);
595+ data_gpa = (max_gfn * guest_page_size) - data_size;
596+ data_gpa = align_down(data_gpa, backing_src_pagesz);
597+
598+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0,
599+ CODE_AND_DATA_MEMSLOT, code_npages, 0);
600+ vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT;
601+ vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT;
602+
603+ vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size,
604+ PAGE_TABLE_MEMSLOT, pt_size / guest_page_size,
605+ p->test_desc->pt_memslot_flags);
606+ vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT;
607+
608+ vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT,
609+ data_size / guest_page_size,
610+ p->test_desc->data_memslot_flags);
611+ vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT;
612+}
613+
614+static void setup_ucall(struct kvm_vm *vm)
615+{
616+ struct userspace_mem_region *region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
617+
618+ ucall_init(vm, region->region.guest_phys_addr + region->region.memory_size);
619+}
620+
621+static void setup_default_handlers(struct test_desc *test)
622+{
623+ if (!test->mmio_handler)
624+ test->mmio_handler = mmio_no_handler;
625+
626+ if (!test->fail_vcpu_run_handler)
627+ test->fail_vcpu_run_handler = fail_vcpu_run_no_handler;
628+}
629+
630+static void check_event_counts(struct test_desc *test)
631+{
632+ ASSERT_EQ(test->expected_events.uffd_faults, events.uffd_faults);
633+ ASSERT_EQ(test->expected_events.mmio_exits, events.mmio_exits);
634+ ASSERT_EQ(test->expected_events.fail_vcpu_runs, events.fail_vcpu_runs);
635+}
636+
637+static void print_test_banner(enum vm_guest_mode mode, struct test_params *p)
638+{
639+ struct test_desc *test = p->test_desc;
640+
641+ pr_debug("Test: %s\n", test->name);
642+ pr_debug("Testing guest mode: %s\n", vm_guest_mode_string(mode));
643+ pr_debug("Testing memory backing src type: %s\n",
644+ vm_mem_backing_src_alias(p->src_type)->name);
645+}
646+
647+static void reset_event_counts(void)
648+{
649+ memset(&events, 0, sizeof(events));
650+}
651+
652+/*
653+ * This function either succeeds, skips the test (after setting test->skip), or
654+ * fails with a TEST_FAIL that aborts all tests.
655+ */
656+static void vcpu_run_loop(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
657+ struct test_desc *test)
658+{
659+ struct kvm_run *run;
660+ struct ucall uc;
661+ int ret;
662+
663+ run = vcpu->run;
664+
665+ for (;;) {
666+ ret = _vcpu_run(vcpu);
667+ if (ret) {
668+ test->fail_vcpu_run_handler(ret);
669+ goto done;
670+ }
671+
672+ switch (get_ucall(vcpu, &uc)) {
673+ case UCALL_SYNC:
674+ if (!handle_cmd(vm, uc.args[1])) {
675+ test->skip = true;
676+ goto done;
677+ }
678+ break;
679+ case UCALL_ABORT:
680+ REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");
681+ break;
682+ case UCALL_DONE:
683+ goto done;
684+ case UCALL_NONE:
685+ if (run->exit_reason == KVM_EXIT_MMIO)
686+ test->mmio_handler(vm, run);
687+ break;
688+ default:
689+ TEST_FAIL("Unknown ucall %lu", uc.cmd);
690+ }
691+ }
692+
693+done:
694+ pr_debug(test->skip ? "Skipped.\n" : "Done.\n");
695+}
696+
697+static void run_test(enum vm_guest_mode mode, void *arg)
698+{
699+ struct test_params *p = (struct test_params *)arg;
700+ struct test_desc *test = p->test_desc;
701+ struct kvm_vm *vm;
702+ struct kvm_vcpu *vcpu;
703+ struct uffd_desc *pt_uffd, *data_uffd;
704+
705+ print_test_banner(mode, p);
706+
707+ vm = ____vm_create(mode);
708+ setup_memslots(vm, p);
709+ kvm_vm_elf_load(vm, program_invocation_name);
710+ setup_ucall(vm);
711+ vcpu = vm_vcpu_add(vm, 0, guest_code);
712+
713+ setup_gva_maps(vm);
714+
715+ reset_event_counts();
716+
717+ /*
718+ * Set some code in the data memslot for the guest to execute (only
719+ * applicable to the EXEC tests). This has to be done before
720+ * setup_uffd() as that function copies the memslot data for the uffd
721+ * handler.
722+ */
723+ load_exec_code_for_test(vm);
724+ setup_uffd(vm, p, &pt_uffd, &data_uffd);
725+ setup_abort_handlers(vm, vcpu, test);
726+ setup_default_handlers(test);
727+ vcpu_args_set(vcpu, 1, test);
728+
729+ vcpu_run_loop(vm, vcpu, test);
730+
731+ kvm_vm_free(vm);
732+ free_uffd(test, pt_uffd, data_uffd);
733+
734+ /*
735+ * Make sure we check the events after the uffd threads have exited,
736+ * which means they updated their respective event counters.
737+ */
738+ if (!test->skip)
739+ check_event_counts(test);
740+}
741+
742+static void help(char *name)
743+{
744+ puts("");
745+ printf("usage: %s [-h] [-s mem-type]\n", name);
746+ puts("");
747+ guest_modes_help();
748+ backing_src_help("-s");
749+ puts("");
750+}
751+
752+#define SNAME(s) #s
753+#define SCAT2(a, b) SNAME(a ## _ ## b)
754+#define SCAT3(a, b, c) SCAT2(a, SCAT2(b, c))
755+#define SCAT4(a, b, c, d) SCAT2(a, SCAT3(b, c, d))
756+
757+#define _CHECK(_test) _CHECK_##_test
758+#define _PREPARE(_test) _PREPARE_##_test
759+#define _PREPARE_guest_read64 NULL
760+#define _PREPARE_guest_ld_preidx NULL
761+#define _PREPARE_guest_write64 NULL
762+#define _PREPARE_guest_st_preidx NULL
763+#define _PREPARE_guest_exec NULL
764+#define _PREPARE_guest_at NULL
765+#define _PREPARE_guest_dc_zva guest_check_dc_zva
766+#define _PREPARE_guest_cas guest_check_lse
767+
768+/* With or without access flag checks */
769+#define _PREPARE_with_af guest_set_ha, guest_clear_pte_af
770+#define _PREPARE_no_af NULL
771+#define _CHECK_with_af guest_check_pte_af
772+#define _CHECK_no_af NULL
773+
774+/* Performs an access and checks that no faults were triggered. */
775+#define TEST_ACCESS(_access, _with_af, _mark_cmd) \
776+{ \
777+ .name = SCAT3(_access, _with_af, #_mark_cmd), \
778+ .guest_prepare = { _PREPARE(_with_af), \
779+ _PREPARE(_access) }, \
780+ .mem_mark_cmd = _mark_cmd, \
781+ .guest_test = _access, \
782+ .guest_test_check = { _CHECK(_with_af) }, \
783+ .expected_events = { 0 }, \
784+}
785+
786+#define TEST_UFFD(_access, _with_af, _mark_cmd, \
787+ _uffd_data_handler, _uffd_pt_handler, _uffd_faults) \
788+{ \
789+ .name = SCAT4(uffd, _access, _with_af, #_mark_cmd), \
790+ .guest_prepare = { _PREPARE(_with_af), \
791+ _PREPARE(_access) }, \
792+ .guest_test = _access, \
793+ .mem_mark_cmd = _mark_cmd, \
794+ .guest_test_check = { _CHECK(_with_af) }, \
795+ .uffd_data_handler = _uffd_data_handler, \
796+ .uffd_pt_handler = _uffd_pt_handler, \
797+ .expected_events = { .uffd_faults = _uffd_faults, }, \
798+}
799+
800+#define TEST_DIRTY_LOG(_access, _with_af, _test_check) \
801+{ \
802+ .name = SCAT3(dirty_log, _access, _with_af), \
803+ .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
804+ .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
805+ .guest_prepare = { _PREPARE(_with_af), \
806+ _PREPARE(_access) }, \
807+ .guest_test = _access, \
808+ .guest_test_check = { _CHECK(_with_af), _test_check, \
809+ guest_check_s1ptw_wr_in_dirty_log}, \
810+ .expected_events = { 0 }, \
811+}
812+
813+#define TEST_UFFD_AND_DIRTY_LOG(_access, _with_af, _uffd_data_handler, \
814+ _uffd_faults, _test_check) \
815+{ \
816+ .name = SCAT3(uffd_and_dirty_log, _access, _with_af), \
817+ .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
818+ .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
819+ .guest_prepare = { _PREPARE(_with_af), \
820+ _PREPARE(_access) }, \
821+ .guest_test = _access, \
822+ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
823+ .guest_test_check = { _CHECK(_with_af), _test_check }, \
824+ .uffd_data_handler = _uffd_data_handler, \
825+ .uffd_pt_handler = uffd_pt_write_handler, \
826+ .expected_events = { .uffd_faults = _uffd_faults, }, \
827+}
828+
829+#define TEST_RO_MEMSLOT(_access, _mmio_handler, _mmio_exits) \
830+{ \
831+ .name = SCAT3(ro_memslot, _access, _with_af), \
832+ .data_memslot_flags = KVM_MEM_READONLY, \
833+ .guest_prepare = { _PREPARE(_access) }, \
834+ .guest_test = _access, \
835+ .mmio_handler = _mmio_handler, \
836+ .expected_events = { .mmio_exits = _mmio_exits }, \
837+}
838+
839+#define TEST_RO_MEMSLOT_NO_SYNDROME(_access) \
840+{ \
841+ .name = SCAT2(ro_memslot_no_syndrome, _access), \
842+ .data_memslot_flags = KVM_MEM_READONLY, \
843+ .guest_test = _access, \
844+ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
845+ .expected_events = { .fail_vcpu_runs = 1 }, \
846+}
847+
848+#define TEST_RO_MEMSLOT_AND_DIRTY_LOG(_access, _mmio_handler, _mmio_exits, \
849+ _test_check) \
850+{ \
851+ .name = SCAT3(ro_memslot, _access, _with_af), \
852+ .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \
853+ .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
854+ .guest_prepare = { _PREPARE(_access) }, \
855+ .guest_test = _access, \
856+ .guest_test_check = { _test_check }, \
857+ .mmio_handler = _mmio_handler, \
858+ .expected_events = { .mmio_exits = _mmio_exits}, \
859+}
860+
861+#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(_access, _test_check) \
862+{ \
863+ .name = SCAT2(ro_memslot_no_syn_and_dlog, _access), \
864+ .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \
865+ .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
866+ .guest_test = _access, \
867+ .guest_test_check = { _test_check }, \
868+ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
869+ .expected_events = { .fail_vcpu_runs = 1 }, \
870+}
871+
872+#define TEST_RO_MEMSLOT_AND_UFFD(_access, _mmio_handler, _mmio_exits, \
873+ _uffd_data_handler, _uffd_faults) \
874+{ \
875+ .name = SCAT2(ro_memslot_uffd, _access), \
876+ .data_memslot_flags = KVM_MEM_READONLY, \
877+ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
878+ .guest_prepare = { _PREPARE(_access) }, \
879+ .guest_test = _access, \
880+ .uffd_data_handler = _uffd_data_handler, \
881+ .uffd_pt_handler = uffd_pt_write_handler, \
882+ .mmio_handler = _mmio_handler, \
883+ .expected_events = { .mmio_exits = _mmio_exits, \
884+ .uffd_faults = _uffd_faults }, \
885+}
886+
887+#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(_access, _uffd_data_handler, \
888+ _uffd_faults) \
889+{ \
890+ .name = SCAT2(ro_memslot_no_syndrome, _access), \
891+ .data_memslot_flags = KVM_MEM_READONLY, \
892+ .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
893+ .guest_test = _access, \
894+ .uffd_data_handler = _uffd_data_handler, \
895+ .uffd_pt_handler = uffd_pt_write_handler, \
896+ .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
897+ .expected_events = { .fail_vcpu_runs = 1, \
898+ .uffd_faults = _uffd_faults }, \
899+}
900+
901+static struct test_desc tests[] = {
902+
903+ /* Check that HW is setting the Access Flag (AF) (sanity checks). */
904+ TEST_ACCESS(guest_read64, with_af, CMD_NONE),
905+ TEST_ACCESS(guest_ld_preidx, with_af, CMD_NONE),
906+ TEST_ACCESS(guest_cas, with_af, CMD_NONE),
907+ TEST_ACCESS(guest_write64, with_af, CMD_NONE),
908+ TEST_ACCESS(guest_st_preidx, with_af, CMD_NONE),
909+ TEST_ACCESS(guest_dc_zva, with_af, CMD_NONE),
910+ TEST_ACCESS(guest_exec, with_af, CMD_NONE),
911+
912+ /*
913+ * Punch a hole in the data backing store, and then try multiple
914+ * accesses: reads should rturn zeroes, and writes should
915+ * re-populate the page. Moreover, the test also check that no
916+ * exception was generated in the guest. Note that this
917+ * reading/writing behavior is the same as reading/writing a
918+ * punched page (with fallocate(FALLOC_FL_PUNCH_HOLE)) from
919+ * userspace.
920+ */
921+ TEST_ACCESS(guest_read64, no_af, CMD_HOLE_DATA),
922+ TEST_ACCESS(guest_cas, no_af, CMD_HOLE_DATA),
923+ TEST_ACCESS(guest_ld_preidx, no_af, CMD_HOLE_DATA),
924+ TEST_ACCESS(guest_write64, no_af, CMD_HOLE_DATA),
925+ TEST_ACCESS(guest_st_preidx, no_af, CMD_HOLE_DATA),
926+ TEST_ACCESS(guest_at, no_af, CMD_HOLE_DATA),
927+ TEST_ACCESS(guest_dc_zva, no_af, CMD_HOLE_DATA),
928+
929+ /*
930+ * Punch holes in the data and PT backing stores and mark them for
931+ * userfaultfd handling. This should result in 2 faults: the access
932+ * on the data backing store, and its respective S1 page table walk
933+ * (S1PTW).
934+ */
935+ TEST_UFFD(guest_read64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
936+ uffd_data_read_handler, uffd_pt_write_handler, 2),
937+ /* no_af should also lead to a PT write. */
938+ TEST_UFFD(guest_read64, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,
939+ uffd_data_read_handler, uffd_pt_write_handler, 2),
940+ /* Note how that cas invokes the read handler. */
941+ TEST_UFFD(guest_cas, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
942+ uffd_data_read_handler, uffd_pt_write_handler, 2),
943+ /*
944+ * Can't test guest_at with_af as it's IMPDEF whether the AF is set.
945+ * The S1PTW fault should still be marked as a write.
946+ */
947+ TEST_UFFD(guest_at, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,
948+ uffd_data_read_handler, uffd_pt_write_handler, 1),
949+ TEST_UFFD(guest_ld_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
950+ uffd_data_read_handler, uffd_pt_write_handler, 2),
951+ TEST_UFFD(guest_write64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
952+ uffd_data_write_handler, uffd_pt_write_handler, 2),
953+ TEST_UFFD(guest_dc_zva, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
954+ uffd_data_write_handler, uffd_pt_write_handler, 2),
955+ TEST_UFFD(guest_st_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
956+ uffd_data_write_handler, uffd_pt_write_handler, 2),
957+ TEST_UFFD(guest_exec, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
958+ uffd_data_read_handler, uffd_pt_write_handler, 2),
959+
960+ /*
961+ * Try accesses when the data and PT memory regions are both
962+ * tracked for dirty logging.
963+ */
964+ TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log),
965+ /* no_af should also lead to a PT write. */
966+ TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log),
967+ TEST_DIRTY_LOG(guest_ld_preidx, with_af, guest_check_no_write_in_dirty_log),
968+ TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log),
969+ TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log),
970+ TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log),
971+ TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log),
972+ TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log),
973+ TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log),
974+
975+ /*
976+ * Access when the data and PT memory regions are both marked for
977+ * dirty logging and UFFD at the same time. The expected result is
978+ * that writes should mark the dirty log and trigger a userfaultfd
979+ * write fault. Reads/execs should result in a read userfaultfd
980+ * fault, and nothing in the dirty log. Any S1PTW should result in
981+ * a write in the dirty log and a userfaultfd write.
982+ */
983+ TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af, uffd_data_read_handler, 2,
984+ guest_check_no_write_in_dirty_log),
985+ /* no_af should also lead to a PT write. */
986+ TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af, uffd_data_read_handler, 2,
987+ guest_check_no_write_in_dirty_log),
988+ TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af, uffd_data_read_handler,
989+ 2, guest_check_no_write_in_dirty_log),
990+ TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, 0, 1,
991+ guest_check_no_write_in_dirty_log),
992+ TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af, uffd_data_read_handler, 2,
993+ guest_check_no_write_in_dirty_log),
994+ TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af, uffd_data_write_handler,
995+ 2, guest_check_write_in_dirty_log),
996+ TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af, uffd_data_read_handler, 2,
997+ guest_check_write_in_dirty_log),
998+ TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af, uffd_data_write_handler,
999+ 2, guest_check_write_in_dirty_log),
1000+ TEST_UFFD_AND_DIRTY_LOG(guest_st_preidx, with_af,
1001+ uffd_data_write_handler, 2,
1002+ guest_check_write_in_dirty_log),
1003+
1004+ /*
1005+ * Try accesses when the data memory region is marked read-only
1006+ * (with KVM_MEM_READONLY). Writes with a syndrome result in an
1007+ * MMIO exit, writes with no syndrome (e.g., CAS) result in a
1008+ * failed vcpu run, and reads/execs with and without syndroms do
1009+ * not fault.
1010+ */
1011+ TEST_RO_MEMSLOT(guest_read64, 0, 0),
1012+ TEST_RO_MEMSLOT(guest_ld_preidx, 0, 0),
1013+ TEST_RO_MEMSLOT(guest_at, 0, 0),
1014+ TEST_RO_MEMSLOT(guest_exec, 0, 0),
1015+ TEST_RO_MEMSLOT(guest_write64, mmio_on_test_gpa_handler, 1),
1016+ TEST_RO_MEMSLOT_NO_SYNDROME(guest_dc_zva),
1017+ TEST_RO_MEMSLOT_NO_SYNDROME(guest_cas),
1018+ TEST_RO_MEMSLOT_NO_SYNDROME(guest_st_preidx),
1019+
1020+ /*
1021+ * Access when both the data region is both read-only and marked
1022+ * for dirty logging at the same time. The expected result is that
1023+ * for writes there should be no write in the dirty log. The
1024+ * readonly handling is the same as if the memslot was not marked
1025+ * for dirty logging: writes with a syndrome result in an MMIO
1026+ * exit, and writes with no syndrome result in a failed vcpu run.
1027+ */
1028+ TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_read64, 0, 0,
1029+ guest_check_no_write_in_dirty_log),
1030+ TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_ld_preidx, 0, 0,
1031+ guest_check_no_write_in_dirty_log),
1032+ TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_at, 0, 0,
1033+ guest_check_no_write_in_dirty_log),
1034+ TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_exec, 0, 0,
1035+ guest_check_no_write_in_dirty_log),
1036+ TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_write64, mmio_on_test_gpa_handler,
1037+ 1, guest_check_no_write_in_dirty_log),
1038+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_dc_zva,
1039+ guest_check_no_write_in_dirty_log),
1040+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_cas,
1041+ guest_check_no_write_in_dirty_log),
1042+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_st_preidx,
1043+ guest_check_no_write_in_dirty_log),
1044+
1045+ /*
1046+ * Access when the data region is both read-only and punched with
1047+ * holes tracked with userfaultfd. The expected result is the
1048+ * union of both userfaultfd and read-only behaviors. For example,
1049+ * write accesses result in a userfaultfd write fault and an MMIO
1050+ * exit. Writes with no syndrome result in a failed vcpu run and
1051+ * no userfaultfd write fault. Reads result in userfaultfd getting
1052+ * triggered.
1053+ */
1054+ TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0,
1055+ uffd_data_read_handler, 2),
1056+ TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0,
1057+ uffd_data_read_handler, 2),
1058+ TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0,
1059+ uffd_no_handler, 1),
1060+ TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0,
1061+ uffd_data_read_handler, 2),
1062+ TEST_RO_MEMSLOT_AND_UFFD(guest_write64, mmio_on_test_gpa_handler, 1,
1063+ uffd_data_write_handler, 2),
1064+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas,
1065+ uffd_data_read_handler, 2),
1066+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva,
1067+ uffd_no_handler, 1),
1068+ TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx,
1069+ uffd_no_handler, 1),
1070+
1071+ { 0 }
1072+};
1073+
1074+static void for_each_test_and_guest_mode(enum vm_mem_backing_src_type src_type)
1075+{
1076+ struct test_desc *t;
1077+
1078+ for (t = &tests[0]; t->name; t++) {
1079+ if (t->skip)
1080+ continue;
1081+
1082+ struct test_params p = {
1083+ .src_type = src_type,
1084+ .test_desc = t,
1085+ };
1086+
1087+ for_each_guest_mode(run_test, &p);
1088+ }
1089+}
1090+
1091+int main(int argc, char *argv[])
1092+{
1093+ enum vm_mem_backing_src_type src_type;
1094+ int opt;
1095+
1096+ setbuf(stdout, NULL);
1097+
1098+ src_type = DEFAULT_VM_MEM_SRC;
1099+
1100+ while ((opt = getopt(argc, argv, "hm:s:")) != -1) {
1101+ switch (opt) {
1102+ case 'm':
1103+ guest_modes_cmdline(optarg);
1104+ break;
1105+ case 's':
1106+ src_type = parse_backing_src_type(optarg);
1107+ break;
1108+ case 'h':
1109+ default:
1110+ help(argv[0]);
1111+ exit(0);
1112+ }
1113+ }
1114+
1115+ for_each_test_and_guest_mode(src_type);
1116+ return 0;
1117+}
tools/testing/selftests/kvm/access_tracking_perf_test.c+1−7
@@ -58,9 +58,6 @@ static enum { · 57 unmodified lines
5858 ITERATION_MARK_IDLE,
5959 } iteration_work;
6060
61−/* Set to true when vCPU threads should exit. */
62−static bool done;
63−
6461 /* The iteration that was last completed by each vCPU. */
6562 static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
6663
@@ -211,7 +208,7 @@ static bool spin_wait_for_next_iteration(int *current_iteration) · 144 unmodified lines
211208 int last_iteration = *current_iteration;
212209
213210 do {
214− if (READ_ONCE(done))
211+ if (READ_ONCE(memstress_args.stop_vcpus))
215212 return false;
216213
217214 *current_iteration = READ_ONCE(iteration);
@@ -321,9 +318,6 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 103 unmodified lines
321318 mark_memory_idle(vm, nr_vcpus);
322319 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory");
323320
324− /* Set done to signal the vCPU threads to exit */
325− done = true;
326−
327321 memstress_join_vcpu_threads(nr_vcpus);
328322 memstress_destroy_vm(vm);
329323 }
tools/testing/selftests/kvm/demand_paging_test.c+30−198
@@ -22,23 +22,13 @@ · 21 unmodified lines
2222 #include "test_util.h"
2323 #include "memstress.h"
2424 #include "guest_modes.h"
25+#include "userfaultfd_util.h"
2526
2627 #ifdef __NR_userfaultfd
2728
28−#ifdef PRINT_PER_PAGE_UPDATES
29−#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
30−#else
31−#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
32−#endif
33−
34−#ifdef PRINT_PER_VCPU_UPDATES
35−#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
36−#else
37−#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
38−#endif
39−
4029 static int nr_vcpus = 1;
4130 static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
31+
4232 static size_t demand_paging_size;
4333 static char *guest_data_prototype;
4434
@@ -67,9 +57,11 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) · 22 unmodified lines
6757 ts_diff.tv_sec, ts_diff.tv_nsec);
6858 }
6959
70−static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr)
60+static int handle_uffd_page_request(int uffd_mode, int uffd,
61+ struct uffd_msg *msg)
7162 {
7263 pid_t tid = syscall(__NR_gettid);
64+ uint64_t addr = msg->arg.pagefault.address;
7365 struct timespec start;
7466 struct timespec ts_diff;
7567 int r;
@@ -116,174 +108,32 @@ static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr) · 40 unmodified lines
116108 return 0;
117109 }
118110
119−bool quit_uffd_thread;
120−
121−struct uffd_handler_args {
111+struct test_params {
122112 int uffd_mode;
123− int uffd;
124− int pipefd;
125− useconds_t delay;
113+ useconds_t uffd_delay;
114+ enum vm_mem_backing_src_type src_type;
115+ bool partition_vcpu_memory_access;
126116 };
127117
128−static void *uffd_handler_thread_fn(void *arg)
129−{
130− struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg;
131− int uffd = uffd_args->uffd;
132− int pipefd = uffd_args->pipefd;
133− useconds_t delay = uffd_args->delay;
134− int64_t pages = 0;
135− struct timespec start;
136− struct timespec ts_diff;
137−
138− clock_gettime(CLOCK_MONOTONIC, &start);
139− while (!quit_uffd_thread) {
140− struct uffd_msg msg;
141− struct pollfd pollfd[2];
142− char tmp_chr;
143− int r;
144− uint64_t addr;
145−
146− pollfd[0].fd = uffd;
147− pollfd[0].events = POLLIN;
148− pollfd[1].fd = pipefd;
149− pollfd[1].events = POLLIN;
150−
151− r = poll(pollfd, 2, -1);
152− switch (r) {
153− case -1:
154− pr_info("poll err");
155− continue;
156− case 0:
157− continue;
158− case 1:
159− break;
160− default:
161− pr_info("Polling uffd returned %d", r);
162− return NULL;
163− }
164−
165− if (pollfd[0].revents & POLLERR) {
166− pr_info("uffd revents has POLLERR");
167− return NULL;
168− }
169−
170− if (pollfd[1].revents & POLLIN) {
171− r = read(pollfd[1].fd, &tmp_chr, 1);
172− TEST_ASSERT(r == 1,
173− "Error reading pipefd in UFFD thread\n");
174− return NULL;
175− }
176−
177− if (!(pollfd[0].revents & POLLIN))
178− continue;
179−
180− r = read(uffd, &msg, sizeof(msg));
181− if (r == -1) {
182− if (errno == EAGAIN)
183− continue;
184− pr_info("Read of uffd got errno %d\n", errno);
185− return NULL;
186− }
187−
188− if (r != sizeof(msg)) {
189− pr_info("Read on uffd returned unexpected size: %d bytes", r);
190− return NULL;
191− }
192−
193− if (!(msg.event & UFFD_EVENT_PAGEFAULT))
194− continue;
195−
196− if (delay)
197− usleep(delay);
198− addr = msg.arg.pagefault.address;
199− r = handle_uffd_page_request(uffd_args->uffd_mode, uffd, addr);
200− if (r < 0)
201− return NULL;
202− pages++;
203− }
204−
205− ts_diff = timespec_elapsed(start);
206− PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
207− pages, ts_diff.tv_sec, ts_diff.tv_nsec,
208− pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
209−
210− return NULL;
211−}
212−
213−static void setup_demand_paging(struct kvm_vm *vm,
214− pthread_t *uffd_handler_thread, int pipefd,
215− int uffd_mode, useconds_t uffd_delay,
216− struct uffd_handler_args *uffd_args,
217− void *hva, void *alias, uint64_t len)
118+static void prefault_mem(void *alias, uint64_t len)
218119 {
219− bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);
220− int uffd;
221− struct uffdio_api uffdio_api;
222− struct uffdio_register uffdio_register;
223− uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
224− int ret;
120+ size_t p;
225121
226− PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
227− is_minor ? "MINOR" : "MISSING",
228− is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");
229−
230− /* In order to get minor faults, prefault via the alias. */
231− if (is_minor) {
232− size_t p;
233−
234− expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
235−
236− TEST_ASSERT(alias != NULL, "Alias required for minor faults");
237− for (p = 0; p < (len / demand_paging_size); ++p) {
238− memcpy(alias + (p * demand_paging_size),
239− guest_data_prototype, demand_paging_size);
240− }
122+ TEST_ASSERT(alias != NULL, "Alias required for minor faults");
123+ for (p = 0; p < (len / demand_paging_size); ++p) {
124+ memcpy(alias + (p * demand_paging_size),
125+ guest_data_prototype, demand_paging_size);
241126 }
242−
243− uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
244− TEST_ASSERT(uffd >= 0, __KVM_SYSCALL_ERROR("userfaultfd()", uffd));
245−
246− uffdio_api.api = UFFD_API;
247− uffdio_api.features = 0;
248− ret = ioctl(uffd, UFFDIO_API, &uffdio_api);
249− TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_API", ret));
250−
251− uffdio_register.range.start = (uint64_t)hva;
252− uffdio_register.range.len = len;
253− uffdio_register.mode = uffd_mode;
254− ret = ioctl(uffd, UFFDIO_REGISTER, &uffdio_register);
255− TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_REGISTER", ret));
256− TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
257− expected_ioctls, "missing userfaultfd ioctls");
258−
259− uffd_args->uffd_mode = uffd_mode;
260− uffd_args->uffd = uffd;
261− uffd_args->pipefd = pipefd;
262− uffd_args->delay = uffd_delay;
263− pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn,
264− uffd_args);
265−
266− PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
267− hva, hva + len);
268127 }
269128
270−struct test_params {
271− int uffd_mode;
272− useconds_t uffd_delay;
273− enum vm_mem_backing_src_type src_type;
274− bool partition_vcpu_memory_access;
275−};
276−
277129 static void run_test(enum vm_guest_mode mode, void *arg)
278130 {
279131 struct test_params *p = arg;
280− pthread_t *uffd_handler_threads = NULL;
281− struct uffd_handler_args *uffd_args = NULL;
132+ struct uffd_desc **uffd_descs = NULL;
282133 struct timespec start;
283134 struct timespec ts_diff;
284− int *pipefds = NULL;
285135 struct kvm_vm *vm;
286− int r, i;
136+ int i;
287137
288138 vm = memstress_create_vm(mode, nr_vcpus, guest_percpu_mem_size, 1,
289139 p->src_type, p->partition_vcpu_memory_access);
@@ -296,15 +146,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
296146 memset(guest_data_prototype, 0xAB, demand_paging_size);
297147
298148 if (p->uffd_mode) {
299− uffd_handler_threads =
300− malloc(nr_vcpus * sizeof(*uffd_handler_threads));
301− TEST_ASSERT(uffd_handler_threads, "Memory allocation failed");
302−
303− uffd_args = malloc(nr_vcpus * sizeof(*uffd_args));
304− TEST_ASSERT(uffd_args, "Memory allocation failed");
305−
306− pipefds = malloc(sizeof(int) * nr_vcpus * 2);
307− TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd");
149+ uffd_descs = malloc(nr_vcpus * sizeof(struct uffd_desc *));
150+ TEST_ASSERT(uffd_descs, "Memory allocation failed");
308151
309152 for (i = 0; i < nr_vcpus; i++) {
310153 struct memstress_vcpu_args *vcpu_args;
@@ -317,19 +160,17 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
317160 vcpu_hva = addr_gpa2hva(vm, vcpu_args->gpa);
318161 vcpu_alias = addr_gpa2alias(vm, vcpu_args->gpa);
319162
163+ prefault_mem(vcpu_alias,
164+ vcpu_args->pages * memstress_args.guest_page_size);
165+
320166 /*
321167 * Set up user fault fd to handle demand paging
322168 * requests.
323169 */
324− r = pipe2(&pipefds[i * 2],
325− O_CLOEXEC | O_NONBLOCK);
326− TEST_ASSERT(!r, "Failed to set up pipefd");
327−
328− setup_demand_paging(vm, &uffd_handler_threads[i],
329− pipefds[i * 2], p->uffd_mode,
330− p->uffd_delay, &uffd_args[i],
331− vcpu_hva, vcpu_alias,
332− vcpu_args->pages * memstress_args.guest_page_size);
170+ uffd_descs[i] = uffd_setup_demand_paging(
171+ p->uffd_mode, p->uffd_delay, vcpu_hva,
172+ vcpu_args->pages * memstress_args.guest_page_size,
173+ &handle_uffd_page_request);
333174 }
334175 }
335176
@@ -344,15 +185,9 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 8 unmodified lines
344185 pr_info("All vCPU threads joined\n");
345186
346187 if (p->uffd_mode) {
347− char c;
348−
349188 /* Tell the user fault fd handler threads to quit */
350− for (i = 0; i < nr_vcpus; i++) {
351− r = write(pipefds[i * 2 + 1], &c, 1);
352− TEST_ASSERT(r == 1, "Unable to write to pipefd");
353−
354− pthread_join(uffd_handler_threads[i], NULL);
355− }
189+ for (i = 0; i < nr_vcpus; i++)
190+ uffd_stop_demand_paging(uffd_descs[i]);
356191 }
357192
358193 pr_info("Total guest execution time: %ld.%.9lds\n",
@@ -364,11 +199,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 5 unmodified lines
364199 memstress_destroy_vm(vm);
365200
366201 free(guest_data_prototype);
367− if (p->uffd_mode) {
368− free(uffd_handler_threads);
369− free(uffd_args);
370− free(pipefds);
371− }
202+ if (p->uffd_mode)
203+ free(uffd_descs);
372204 }
373205
374206 static void help(char *name)
tools/testing/selftests/kvm/dirty_log_test.c+39−14
@@ -24,6 +24,9 @@ · 23 unmodified lines
2424 #include "guest_modes.h"
2525 #include "processor.h"
2626
27+#define DIRTY_MEM_BITS 30 /* 1G */
28+#define PAGE_SHIFT_4K 12
29+
2730 /* The memory slot index to track dirty pages */
2831 #define TEST_MEM_SLOT_INDEX 1
2932
@@ -226,13 +229,15 @@ static void clear_log_create_vm_done(struct kvm_vm *vm) · 196 unmodified lines
226229 }
227230
228231 static void dirty_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
229− void *bitmap, uint32_t num_pages)
232+ void *bitmap, uint32_t num_pages,
233+ uint32_t *unused)
230234 {
231235 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
232236 }
233237
234238 static void clear_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
235− void *bitmap, uint32_t num_pages)
239+ void *bitmap, uint32_t num_pages,
240+ uint32_t *unused)
236241 {
237242 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
238243 kvm_vm_clear_dirty_log(vcpu->vm, slot, bitmap, 0, num_pages);
@@ -271,6 +276,24 @@ static bool dirty_ring_supported(void) · 32 unmodified lines
271276
272277 static void dirty_ring_create_vm_done(struct kvm_vm *vm)
273278 {
279+ uint64_t pages;
280+ uint32_t limit;
281+
282+ /*
283+ * We rely on vcpu exit due to full dirty ring state. Adjust
284+ * the ring buffer size to ensure we're able to reach the
285+ * full dirty ring state.
286+ */
287+ pages = (1ul << (DIRTY_MEM_BITS - vm->page_shift)) + 3;
288+ pages = vm_adjust_num_guest_pages(vm->mode, pages);
289+ if (vm->page_size < getpagesize())
290+ pages = vm_num_host_pages(vm->mode, pages);
291+
292+ limit = 1 << (31 - __builtin_clz(pages));
293+ test_dirty_ring_count = 1 << (31 - __builtin_clz(test_dirty_ring_count));
294+ test_dirty_ring_count = min(limit, test_dirty_ring_count);
295+ pr_info("dirty ring count: 0x%x\n", test_dirty_ring_count);
296+
274297 /*
275298 * Switch to dirty ring mode after VM creation but before any
276299 * of the vcpu creation.
@@ -329,10 +352,9 @@ static void dirty_ring_continue_vcpu(void) · 52 unmodified lines
329352 }
330353
331354 static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
332− void *bitmap, uint32_t num_pages)
355+ void *bitmap, uint32_t num_pages,
356+ uint32_t *ring_buf_idx)
333357 {
334− /* We only have one vcpu */
335− static uint32_t fetch_index = 0;
336358 uint32_t count = 0, cleared;
337359 bool continued_vcpu = false;
338360
@@ -349,7 +371,8 @@ static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot, · 10 unmodified lines
349371
350372 /* Only have one vcpu */
351373 count = dirty_ring_collect_one(vcpu_map_dirty_ring(vcpu),
352− slot, bitmap, num_pages, &fetch_index);
374+ slot, bitmap, num_pages,
375+ ring_buf_idx);
353376
354377 cleared = kvm_vm_reset_dirty_ring(vcpu->vm);
355378
@@ -406,7 +429,8 @@ struct log_mode { · 50 unmodified lines
406429 void (*create_vm_done)(struct kvm_vm *vm);
407430 /* Hook to collect the dirty pages into the bitmap provided */
408431 void (*collect_dirty_pages) (struct kvm_vcpu *vcpu, int slot,
409− void *bitmap, uint32_t num_pages);
432+ void *bitmap, uint32_t num_pages,
433+ uint32_t *ring_buf_idx);
410434 /* Hook to call when after each vcpu run */
411435 void (*after_vcpu_run)(struct kvm_vcpu *vcpu, int ret, int err);
412436 void (*before_vcpu_join) (void);
@@ -471,13 +495,14 @@ static void log_mode_create_vm_done(struct kvm_vm *vm) · 58 unmodified lines
471495 }
472496
473497 static void log_mode_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
474− void *bitmap, uint32_t num_pages)
498+ void *bitmap, uint32_t num_pages,
499+ uint32_t *ring_buf_idx)
475500 {
476501 struct log_mode *mode = &log_modes[host_log_mode];
477502
478503 TEST_ASSERT(mode->collect_dirty_pages != NULL,
479504 "collect_dirty_pages() is required for any log mode!");
480− mode->collect_dirty_pages(vcpu, slot, bitmap, num_pages);
505+ mode->collect_dirty_pages(vcpu, slot, bitmap, num_pages, ring_buf_idx);
481506 }
482507
483508 static void log_mode_after_vcpu_run(struct kvm_vcpu *vcpu, int ret, int err)
@@ -681,9 +706,6 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu, · 197 unmodified lines
681706 return vm;
682707 }
683708
684−#define DIRTY_MEM_BITS 30 /* 1G */
685−#define PAGE_SHIFT_4K 12
686−
687709 struct test_params {
688710 unsigned long iterations;
689711 unsigned long interval;
@@ -696,6 +718,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
696718 struct kvm_vcpu *vcpu;
697719 struct kvm_vm *vm;
698720 unsigned long *bmap;
721+ uint32_t ring_buf_idx = 0;
699722
700723 if (!log_mode_supported()) {
701724 print_skip("Log mode '%s' not supported",
@@ -769,6 +792,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 67 unmodified lines
769792 host_dirty_count = 0;
770793 host_clear_count = 0;
771794 host_track_next_count = 0;
795+ WRITE_ONCE(dirty_ring_vcpu_ring_full, false);
772796
773797 pthread_create(&vcpu_thread, NULL, vcpu_worker, vcpu);
774798
@@ -776,7 +800,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 1 unmodified line
776800 /* Give the vcpu thread some time to dirty some pages */
777801 usleep(p->interval * 1000);
778802 log_mode_collect_dirty_pages(vcpu, TEST_MEM_SLOT_INDEX,
779− bmap, host_num_pages);
803+ bmap, host_num_pages,
804+ &ring_buf_idx);
780805
781806 /*
782807 * See vcpu_sync_stop_requested definition for details on why
@@ -820,7 +845,7 @@ static void help(char *name) · 37 unmodified lines
820845 printf("usage: %s [-h] [-i iterations] [-I interval] "
821846 "[-p offset] [-m mode]\n", name);
822847 puts("");
823− printf(" -c: specify dirty ring size, in number of entries\n");
848+ printf(" -c: hint to dirty ring size, in number of entries\n");
824849 printf(" (only useful for dirty-ring test; default: %"PRIu32")\n",
825850 TEST_DIRTY_RING_COUNT);
826851 printf(" -i: specify iteration counts (default: %"PRIu64")\n",
tools/testing/selftests/kvm/include/aarch64/processor.h+29−6
@@ -38,12 +38,25 @@ · 37 unmodified lines
3838 * NORMAL 4 1111:1111
3939 * NORMAL_WT 5 1011:1011
4040 */
41−#define DEFAULT_MAIR_EL1 ((0x00ul << (0 * 8)) | \
42− (0x04ul << (1 * 8)) | \
43− (0x0cul << (2 * 8)) | \
44− (0x44ul << (3 * 8)) | \
45− (0xfful << (4 * 8)) | \
46− (0xbbul << (5 * 8)))
41+
42+/* Linux doesn't use these memory types, so let's define them. */
43+#define MAIR_ATTR_DEVICE_GRE UL(0x0c)
44+#define MAIR_ATTR_NORMAL_WT UL(0xbb)
45+
46+#define MT_DEVICE_nGnRnE 0
47+#define MT_DEVICE_nGnRE 1
48+#define MT_DEVICE_GRE 2
49+#define MT_NORMAL_NC 3
50+#define MT_NORMAL 4
51+#define MT_NORMAL_WT 5
52+
53+#define DEFAULT_MAIR_EL1 \
54+ (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
55+ MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \
56+ MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \
57+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \
58+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
59+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT))
4760
4861 #define MPIDR_HWID_BITMASK (0xff00fffffful)
4962
@@ -92,11 +105,19 @@ enum { · 42 unmodified lines
92105 #define ESR_EC_MASK (ESR_EC_NUM - 1)
93106
94107 #define ESR_EC_SVC64 0x15
108+#define ESR_EC_IABT 0x21
109+#define ESR_EC_DABT 0x25
95110 #define ESR_EC_HW_BP_CURRENT 0x31
96111 #define ESR_EC_SSTEP_CURRENT 0x33
97112 #define ESR_EC_WP_CURRENT 0x35
98113 #define ESR_EC_BRK_INS 0x3c
99114
115+/* Access flag */
116+#define PTE_AF (1ULL << 10)
117+
118+/* Access flag update enable/disable */
119+#define TCR_EL1_HA (1ULL << 39)
120+
100121 void aarch64_get_supported_page_sizes(uint32_t ipa,
101122 bool *ps4k, bool *ps16k, bool *ps64k);
102123
@@ -109,6 +130,8 @@ void vm_install_exception_handler(struct kvm_vm *vm, · 6 unmodified lines
109130 void vm_install_sync_handler(struct kvm_vm *vm,
110131 int vector, int ec, handler_fn handler);
111132
133+uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva);
134+
112135 static inline void cpu_relax(void)
113136 {
114137 asm volatile("yield" ::: "memory");
tools/testing/selftests/kvm/include/kvm_util_base.h+29−2
@@ -35,6 +35,7 @@ struct userspace_mem_region { · 34 unmodified lines
3535 struct sparsebit *unused_phy_pages;
3636 int fd;
3737 off_t offset;
38+ enum vm_mem_backing_src_type backing_src_type;
3839 void *host_mem;
3940 void *host_alias;
4041 void *mmap_start;
@@ -65,6 +66,14 @@ struct userspace_mem_regions { · 24 unmodified lines
6566 DECLARE_HASHTABLE(slot_hash, 9);
6667 };
6768
69+enum kvm_mem_region_type {
70+ MEM_REGION_CODE,
71+ MEM_REGION_DATA,
72+ MEM_REGION_PT,
73+ MEM_REGION_TEST_DATA,
74+ NR_MEM_REGIONS,
75+};
76+
6877 struct kvm_vm {
6978 int mode;
7079 unsigned long type;
@@ -94,6 +103,13 @@ struct kvm_vm { · 23 unmodified lines
94103 int stats_fd;
95104 struct kvm_stats_header stats_header;
96105 struct kvm_stats_desc *stats_desc;
106+
107+ /*
108+ * KVM region slots. These are the default memslots used by page
109+ * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
110+ * memslot.
111+ */
112+ uint32_t memslots[NR_MEM_REGIONS];
97113 };
98114
99115
@@ -106,6 +122,13 @@ struct kvm_vm { · 6 unmodified lines
106122 struct userspace_mem_region *
107123 memslot2region(struct kvm_vm *vm, uint32_t memslot);
108124
125+static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
126+ enum kvm_mem_region_type type)
127+{
128+ assert(type < NR_MEM_REGIONS);
129+ return memslot2region(vm, vm->memslots[type]);
130+}
131+
109132 /* Minimum allocated guest virtual and physical addresses */
110133 #define KVM_UTIL_MIN_VADDR 0x2000
111134 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000
@@ -387,7 +410,11 @@ void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); · 275 unmodified lines
387410 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
388411 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
389412 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
413+vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
414+ enum kvm_mem_region_type type);
390415 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
416+vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
417+ enum kvm_mem_region_type type);
391418 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
392419
393420 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
@@ -649,13 +676,13 @@ vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); · 255 unmodified lines
649676 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
650677 * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
651678 */
652−struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages);
679+struct kvm_vm *____vm_create(enum vm_guest_mode mode);
653680 struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
654681 uint64_t nr_extra_pages);
655682
656683 static inline struct kvm_vm *vm_create_barebones(void)
657684 {
658− return ____vm_create(VM_MODE_DEFAULT, 0);
685+ return ____vm_create(VM_MODE_DEFAULT);
659686 }
660687
661688 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
tools/testing/selftests/kvm/include/memstress.h+3−0
@@ -47,6 +47,9 @@ struct memstress_args { · 46 unmodified lines
4747 /* The vCPU=>pCPU pinning map. Only valid if pin_vcpus is true. */
4848 uint32_t vcpu_to_pcpu[KVM_MAX_VCPUS];
4949
50+ /* Test is done, stop running vCPUs. */
51+ bool stop_vcpus;
52+
5053 struct memstress_vcpu_args vcpu_args[KVM_MAX_VCPUS];
5154 };
5255
tools/testing/selftests/kvm/include/userfaultfd_util.h+45−0
@@ -0,0 +1,45 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * KVM userfaultfd util
4+ *
5+ * Copyright (C) 2018, Red Hat, Inc.
6+ * Copyright (C) 2019-2022 Google LLC
7+ */
8+
9+#define _GNU_SOURCE /* for pipe2 */
10+
11+#include <inttypes.h>
12+#include <time.h>
13+#include <pthread.h>
14+#include <linux/userfaultfd.h>
15+
16+#include "test_util.h"
17+
18+typedef int (*uffd_handler_t)(int uffd_mode, int uffd, struct uffd_msg *msg);
19+
20+struct uffd_desc {
21+ int uffd_mode;
22+ int uffd;
23+ int pipefds[2];
24+ useconds_t delay;
25+ uffd_handler_t handler;
26+ pthread_t thread;
27+};
28+
29+struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
30+ void *hva, uint64_t len,
31+ uffd_handler_t handler);
32+
33+void uffd_stop_demand_paging(struct uffd_desc *uffd);
34+
35+#ifdef PRINT_PER_PAGE_UPDATES
36+#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
37+#else
38+#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
39+#endif
40+
41+#ifdef PRINT_PER_VCPU_UPDATES
42+#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
43+#else
44+#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
45+#endif
tools/testing/selftests/kvm/lib/aarch64/processor.c+34−21
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #include "guest_modes.h"
1212 #include "kvm_util.h"
1313 #include "processor.h"
14+#include <linux/bitfield.h>
1415
1516 #define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000
1617
@@ -76,13 +77,15 @@ static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm) · 59 unmodified lines
7677
7778 void virt_arch_pgd_alloc(struct kvm_vm *vm)
7879 {
79− if (!vm->pgd_created) {
80− vm_paddr_t paddr = vm_phy_pages_alloc(vm,
81− page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size,
82− KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
83− vm->pgd = paddr;
84− vm->pgd_created = true;
85− }
80+ size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size;
81+
82+ if (vm->pgd_created)
83+ return;
84+
85+ vm->pgd = vm_phy_pages_alloc(vm, nr_pages,
86+ KVM_GUEST_PAGE_TABLE_MIN_PADDR,
87+ vm->memslots[MEM_REGION_PT]);
88+ vm->pgd_created = true;
8689 }
8790
8891 static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
@@ -133,12 +136,12 @@ static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, · 44 unmodified lines
133136
134137 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
135138 {
136− uint64_t attr_idx = 4; /* NORMAL (See DEFAULT_MAIR_EL1) */
139+ uint64_t attr_idx = MT_NORMAL;
137140
138141 _virt_pg_map(vm, vaddr, paddr, attr_idx);
139142 }
140143
141−vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
144+uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva)
142145 {
143146 uint64_t *ptep;
144147
@@ -169,11 +172,18 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) · 24 unmodified lines
169172 TEST_FAIL("Page table levels must be 2, 3, or 4");
170173 }
171174
172− return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
175+ return ptep;
173176
174177 unmapped_gva:
175178 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
176− exit(1);
179+ exit(EXIT_FAILURE);
180+}
181+
182+vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
183+{
184+ uint64_t *ptep = virt_get_pte_hva(vm, gva);
185+
186+ return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
177187 }
178188
179189 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
@@ -318,13 +328,16 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) · 138 unmodified lines
318328 struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
319329 struct kvm_vcpu_init *init, void *guest_code)
320330 {
321− size_t stack_size = vm->page_size == 4096 ?
322− DEFAULT_STACK_PGS * vm->page_size :
323− vm->page_size;
324− uint64_t stack_vaddr = vm_vaddr_alloc(vm, stack_size,
325− DEFAULT_ARM64_GUEST_STACK_VADDR_MIN);
331+ size_t stack_size;
332+ uint64_t stack_vaddr;
326333 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
327334
335+ stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
336+ vm->page_size;
337+ stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
338+ DEFAULT_ARM64_GUEST_STACK_VADDR_MIN,
339+ MEM_REGION_DATA);
340+
328341 aarch64_vcpu_setup(vcpu, init);
329342
330343 vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
@@ -428,8 +441,8 @@ unexpected_exception: · 97 unmodified lines
428441
429442 void vm_init_descriptor_tables(struct kvm_vm *vm)
430443 {
431− vm->handlers = vm_vaddr_alloc(vm, sizeof(struct handlers),
432− vm->page_size);
444+ vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
445+ vm->page_size, MEM_REGION_DATA);
433446
434447 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
435448 }
@@ -486,9 +499,9 @@ void aarch64_get_supported_page_sizes(uint32_t ipa, · 50 unmodified lines
486499 err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);
487500 TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd));
488501
489− *ps4k = ((val >> 28) & 0xf) != 0xf;
490− *ps64k = ((val >> 24) & 0xf) == 0;
491− *ps16k = ((val >> 20) & 0xf) != 0;
502+ *ps4k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN4), val) != 0xf;
503+ *ps64k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN64), val) == 0;
504+ *ps16k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN16), val) != 0;
492505
493506 close(vcpu_fd);
494507 close(vm_fd);
tools/testing/selftests/kvm/lib/elf.c+2−1
@@ -161,7 +161,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) · 160 unmodified lines
161161 seg_vend |= vm->page_size - 1;
162162 size_t seg_size = seg_vend - seg_vstart + 1;
163163
164− vm_vaddr_t vaddr = vm_vaddr_alloc(vm, seg_size, seg_vstart);
164+ vm_vaddr_t vaddr = __vm_vaddr_alloc(vm, seg_size, seg_vstart,
165+ MEM_REGION_CODE);
165166 TEST_ASSERT(vaddr == seg_vstart, "Unable to allocate "
166167 "virtual memory for segment at requested min addr,\n"
167168 " segment idx: %u\n"
tools/testing/selftests/kvm/lib/kvm_util.c+53−31
@@ -186,13 +186,10 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { · 185 unmodified lines
186186 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
187187 "Missing new mode params?");
188188
189−struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages)
189+struct kvm_vm *____vm_create(enum vm_guest_mode mode)
190190 {
191191 struct kvm_vm *vm;
192192
193− pr_debug("%s: mode='%s' pages='%ld'\n", __func__,
194− vm_guest_mode_string(mode), nr_pages);
195−
196193 vm = calloc(1, sizeof(*vm));
197194 TEST_ASSERT(vm != NULL, "Insufficient Memory");
198195
@@ -288,9 +285,6 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages) · 89 unmodified lines
288285
289286 /* Allocate and setup memory for guest. */
290287 vm->vpages_mapped = sparsebit_alloc();
291− if (nr_pages != 0)
292− vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
293− 0, 0, nr_pages, 0);
294288
295289 return vm;
296290 }
@@ -337,8 +331,16 @@ struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus, · 40 unmodified lines
337331 nr_extra_pages);
338332 struct userspace_mem_region *slot0;
339333 struct kvm_vm *vm;
334+ int i;
335+
336+ pr_debug("%s: mode='%s' pages='%ld'\n", __func__,
337+ vm_guest_mode_string(mode), nr_pages);
340338
341− vm = ____vm_create(mode, nr_pages);
339+ vm = ____vm_create(mode);
340+
341+ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);
342+ for (i = 0; i < NR_MEM_REGIONS; i++)
343+ vm->memslots[i] = 0;
342344
343345 kvm_vm_elf_load(vm, program_invocation_name);
344346
@@ -649,6 +651,12 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, · 304 unmodified lines
649651 sparsebit_free(&region->unused_phy_pages);
650652 ret = munmap(region->mmap_start, region->mmap_size);
651653 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
654+ if (region->fd >= 0) {
655+ /* There's an extra map when using shared memory. */
656+ ret = munmap(region->mmap_alias, region->mmap_size);
657+ TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
658+ close(region->fd);
659+ }
652660
653661 free(region);
654662 }
@@ -986,6 +994,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, · 331 unmodified lines
986994 vm_mem_backing_src_alias(src_type)->name);
987995 }
988996
997+ region->backing_src_type = src_type;
989998 region->unused_phy_pages = sparsebit_alloc();
990999 sparsebit_set_num(region->unused_phy_pages,
9911000 guest_paddr >> vm->page_shift, npages);
@@ -1280,32 +1289,15 @@ va_found: · 288 unmodified lines
12801289 return pgidx_start * vm->page_size;
12811290 }
12821291
1283−/*
1284− * VM Virtual Address Allocate
1285− *
1286− * Input Args:
1287− * vm - Virtual Machine
1288− * sz - Size in bytes
1289− * vaddr_min - Minimum starting virtual address
1290− *
1291− * Output Args: None
1292− *
1293− * Return:
1294− * Starting guest virtual address
1295− *
1296− * Allocates at least sz bytes within the virtual address space of the vm
1297− * given by vm. The allocated bytes are mapped to a virtual address >=
1298− * the address given by vaddr_min. Note that each allocation uses a
1299− * a unique set of pages, with the minimum real allocation being at least
1300− * a page.
1301− */
1302−vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1292+vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1293+ enum kvm_mem_region_type type)
13031294 {
13041295 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
13051296
13061297 virt_pgd_alloc(vm);
13071298 vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,
1308− KVM_UTIL_MIN_PFN * vm->page_size, 0);
1299+ KVM_UTIL_MIN_PFN * vm->page_size,
1300+ vm->memslots[type]);
13091301
13101302 /*
13111303 * Find an unused range of virtual page addresses of at least
@@ -1325,6 +1317,30 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min) · 13 unmodified lines
13251317 return vaddr_start;
13261318 }
13271319
1320+/*
1321+ * VM Virtual Address Allocate
1322+ *
1323+ * Input Args:
1324+ * vm - Virtual Machine
1325+ * sz - Size in bytes
1326+ * vaddr_min - Minimum starting virtual address
1327+ *
1328+ * Output Args: None
1329+ *
1330+ * Return:
1331+ * Starting guest virtual address
1332+ *
1333+ * Allocates at least sz bytes within the virtual address space of the vm
1334+ * given by vm. The allocated bytes are mapped to a virtual address >=
1335+ * the address given by vaddr_min. Note that each allocation uses a
1336+ * a unique set of pages, with the minimum real allocation being at least
1337+ * a page. The allocated physical space comes from the TEST_DATA memory region.
1338+ */
1339+vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1340+{
1341+ return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
1342+}
1343+
13281344 /*
13291345 * VM Virtual Address Allocate Pages
13301346 *
@@ -1344,6 +1360,11 @@ vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages) · 13 unmodified lines
13441360 return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
13451361 }
13461362
1363+vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
1364+{
1365+ return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
1366+}
1367+
13471368 /*
13481369 * VM Virtual Address Allocate Page
13491370 *
@@ -1570,7 +1591,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu) · 220 unmodified lines
15701591
15711592 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
15721593 {
1573− uint32_t page_size = vcpu->vm->page_size;
1594+ uint32_t page_size = getpagesize();
15741595 uint32_t size = vcpu->vm->dirty_ring_size;
15751596
15761597 TEST_ASSERT(size > 0, "Should enable dirty ring first");
@@ -1911,7 +1932,8 @@ vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, · 334 unmodified lines
19111932
19121933 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
19131934 {
1914− return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
1935+ return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
1936+ vm->memslots[MEM_REGION_PT]);
19151937 }
19161938
19171939 /*
tools/testing/selftests/kvm/lib/memstress.c+3−0
@@ -292,6 +292,7 @@ void memstress_start_vcpu_threads(int nr_vcpus, · 291 unmodified lines
292292
293293 vcpu_thread_fn = vcpu_fn;
294294 WRITE_ONCE(all_vcpu_threads_running, false);
295+ WRITE_ONCE(memstress_args.stop_vcpus, false);
295296
296297 for (i = 0; i < nr_vcpus; i++) {
297298 struct vcpu_thread *vcpu = &vcpu_threads[i];
@@ -314,6 +315,8 @@ void memstress_join_vcpu_threads(int nr_vcpus) · 16 unmodified lines
314315 {
315316 int i;
316317
318+ WRITE_ONCE(memstress_args.stop_vcpus, true);
319+
317320 for (i = 0; i < nr_vcpus; i++)
318321 pthread_join(vcpu_threads[i].thread, NULL);
319322 }
tools/testing/selftests/kvm/lib/riscv/processor.c+17−12
@@ -55,13 +55,15 @@ static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level) · 54 unmodified lines
5555
5656 void virt_arch_pgd_alloc(struct kvm_vm *vm)
5757 {
58− if (!vm->pgd_created) {
59− vm_paddr_t paddr = vm_phy_pages_alloc(vm,
60− page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size,
61− KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
62− vm->pgd = paddr;
63− vm->pgd_created = true;
64− }
58+ size_t nr_pages = page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size;
59+
60+ if (vm->pgd_created)
61+ return;
62+
63+ vm->pgd = vm_phy_pages_alloc(vm, nr_pages,
64+ KVM_GUEST_PAGE_TABLE_MIN_PADDR,
65+ vm->memslots[MEM_REGION_PT]);
66+ vm->pgd_created = true;
6567 }
6668
6769 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
@@ -279,15 +281,18 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 211 unmodified lines
279281 void *guest_code)
280282 {
281283 int r;
282− size_t stack_size = vm->page_size == 4096 ?
283− DEFAULT_STACK_PGS * vm->page_size :
284− vm->page_size;
285− unsigned long stack_vaddr = vm_vaddr_alloc(vm, stack_size,
286− DEFAULT_RISCV_GUEST_STACK_VADDR_MIN);
284+ size_t stack_size;
285+ unsigned long stack_vaddr;
287286 unsigned long current_gp = 0;
288287 struct kvm_mp_state mps;
289288 struct kvm_vcpu *vcpu;
290289
290+ stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
291+ vm->page_size;
292+ stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
293+ DEFAULT_RISCV_GUEST_STACK_VADDR_MIN,
294+ MEM_REGION_DATA);
295+
291296 vcpu = __vm_vcpu_add(vm, vcpu_id);
292297 riscv_vcpu_mmu_setup(vcpu);
293298
tools/testing/selftests/kvm/lib/s390x/processor.c+5−3
@@ -21,7 +21,8 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) · 20 unmodified lines
2121 return;
2222
2323 paddr = vm_phy_pages_alloc(vm, PAGES_PER_REGION,
24− KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
24+ KVM_GUEST_PAGE_TABLE_MIN_PADDR,
25+ vm->memslots[MEM_REGION_PT]);
2526 memset(addr_gpa2hva(vm, paddr), 0xff, PAGES_PER_REGION * vm->page_size);
2627
2728 vm->pgd = paddr;
@@ -167,8 +168,9 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 139 unmodified lines
167168 TEST_ASSERT(vm->page_size == 4096, "Unsupported page size: 0x%x",
168169 vm->page_size);
169170
170− stack_vaddr = vm_vaddr_alloc(vm, stack_size,
171− DEFAULT_GUEST_STACK_VADDR_MIN);
171+ stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
172+ DEFAULT_GUEST_STACK_VADDR_MIN,
173+ MEM_REGION_DATA);
172174
173175 vcpu = __vm_vcpu_add(vm, vcpu_id);
174176
tools/testing/selftests/kvm/lib/userfaultfd_util.c+186−0
@@ -0,0 +1,186 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * KVM userfaultfd util
4+ * Adapted from demand_paging_test.c
5+ *
6+ * Copyright (C) 2018, Red Hat, Inc.
7+ * Copyright (C) 2019-2022 Google LLC
8+ */
9+
10+#define _GNU_SOURCE /* for pipe2 */
11+
12+#include <inttypes.h>
13+#include <stdio.h>
14+#include <stdlib.h>
15+#include <time.h>
16+#include <poll.h>
17+#include <pthread.h>
18+#include <linux/userfaultfd.h>
19+#include <sys/syscall.h>
20+
21+#include "kvm_util.h"
22+#include "test_util.h"
23+#include "memstress.h"
24+#include "userfaultfd_util.h"
25+
26+#ifdef __NR_userfaultfd
27+
28+static void *uffd_handler_thread_fn(void *arg)
29+{
30+ struct uffd_desc *uffd_desc = (struct uffd_desc *)arg;
31+ int uffd = uffd_desc->uffd;
32+ int pipefd = uffd_desc->pipefds[0];
33+ useconds_t delay = uffd_desc->delay;
34+ int64_t pages = 0;
35+ struct timespec start;
36+ struct timespec ts_diff;
37+
38+ clock_gettime(CLOCK_MONOTONIC, &start);
39+ while (1) {
40+ struct uffd_msg msg;
41+ struct pollfd pollfd[2];
42+ char tmp_chr;
43+ int r;
44+
45+ pollfd[0].fd = uffd;
46+ pollfd[0].events = POLLIN;
47+ pollfd[1].fd = pipefd;
48+ pollfd[1].events = POLLIN;
49+
50+ r = poll(pollfd, 2, -1);
51+ switch (r) {
52+ case -1:
53+ pr_info("poll err");
54+ continue;
55+ case 0:
56+ continue;
57+ case 1:
58+ break;
59+ default:
60+ pr_info("Polling uffd returned %d", r);
61+ return NULL;
62+ }
63+
64+ if (pollfd[0].revents & POLLERR) {
65+ pr_info("uffd revents has POLLERR");
66+ return NULL;
67+ }
68+
69+ if (pollfd[1].revents & POLLIN) {
70+ r = read(pollfd[1].fd, &tmp_chr, 1);
71+ TEST_ASSERT(r == 1,
72+ "Error reading pipefd in UFFD thread\n");
73+ return NULL;
74+ }
75+
76+ if (!(pollfd[0].revents & POLLIN))
77+ continue;
78+
79+ r = read(uffd, &msg, sizeof(msg));
80+ if (r == -1) {
81+ if (errno == EAGAIN)
82+ continue;
83+ pr_info("Read of uffd got errno %d\n", errno);
84+ return NULL;
85+ }
86+
87+ if (r != sizeof(msg)) {
88+ pr_info("Read on uffd returned unexpected size: %d bytes", r);
89+ return NULL;
90+ }
91+
92+ if (!(msg.event & UFFD_EVENT_PAGEFAULT))
93+ continue;
94+
95+ if (delay)
96+ usleep(delay);
97+ r = uffd_desc->handler(uffd_desc->uffd_mode, uffd, &msg);
98+ if (r < 0)
99+ return NULL;
100+ pages++;
101+ }
102+
103+ ts_diff = timespec_elapsed(start);
104+ PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
105+ pages, ts_diff.tv_sec, ts_diff.tv_nsec,
106+ pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
107+
108+ return NULL;
109+}
110+
111+struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
112+ void *hva, uint64_t len,
113+ uffd_handler_t handler)
114+{
115+ struct uffd_desc *uffd_desc;
116+ bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);
117+ int uffd;
118+ struct uffdio_api uffdio_api;
119+ struct uffdio_register uffdio_register;
120+ uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
121+ int ret;
122+
123+ PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
124+ is_minor ? "MINOR" : "MISSING",
125+ is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");
126+
127+ uffd_desc = malloc(sizeof(struct uffd_desc));
128+ TEST_ASSERT(uffd_desc, "malloc failed");
129+
130+ /* In order to get minor faults, prefault via the alias. */
131+ if (is_minor)
132+ expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
133+
134+ uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
135+ TEST_ASSERT(uffd >= 0, "uffd creation failed, errno: %d", errno);
136+
137+ uffdio_api.api = UFFD_API;
138+ uffdio_api.features = 0;
139+ TEST_ASSERT(ioctl(uffd, UFFDIO_API, &uffdio_api) != -1,
140+ "ioctl UFFDIO_API failed: %" PRIu64,
141+ (uint64_t)uffdio_api.api);
142+
143+ uffdio_register.range.start = (uint64_t)hva;
144+ uffdio_register.range.len = len;
145+ uffdio_register.mode = uffd_mode;
146+ TEST_ASSERT(ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) != -1,
147+ "ioctl UFFDIO_REGISTER failed");
148+ TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
149+ expected_ioctls, "missing userfaultfd ioctls");
150+
151+ ret = pipe2(uffd_desc->pipefds, O_CLOEXEC | O_NONBLOCK);
152+ TEST_ASSERT(!ret, "Failed to set up pipefd");
153+
154+ uffd_desc->uffd_mode = uffd_mode;
155+ uffd_desc->uffd = uffd;
156+ uffd_desc->delay = delay;
157+ uffd_desc->handler = handler;
158+ pthread_create(&uffd_desc->thread, NULL, uffd_handler_thread_fn,
159+ uffd_desc);
160+
161+ PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
162+ hva, hva + len);
163+
164+ return uffd_desc;
165+}
166+
167+void uffd_stop_demand_paging(struct uffd_desc *uffd)
168+{
169+ char c = 0;
170+ int ret;
171+
172+ ret = write(uffd->pipefds[1], &c, 1);
173+ TEST_ASSERT(ret == 1, "Unable to write to pipefd");
174+
175+ ret = pthread_join(uffd->thread, NULL);
176+ TEST_ASSERT(ret == 0, "Pthread_join failed.");
177+
178+ close(uffd->uffd);
179+
180+ close(uffd->pipefds[1]);
181+ close(uffd->pipefds[0]);
182+
183+ free(uffd);
184+}
185+
186+#endif /* __NR_userfaultfd */
tools/testing/selftests/kvm/lib/x86_64/processor.c+7−6
@@ -499,7 +499,7 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) · 498 unmodified lines
499499 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
500500 {
501501 if (!vm->gdt)
502− vm->gdt = vm_vaddr_alloc_page(vm);
502+ vm->gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
503503
504504 dt->base = vm->gdt;
505505 dt->limit = getpagesize();
@@ -509,7 +509,7 @@ static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp, · 3 unmodified lines
509509 int selector)
510510 {
511511 if (!vm->tss)
512− vm->tss = vm_vaddr_alloc_page(vm);
512+ vm->tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
513513
514514 memset(segp, 0, sizeof(*segp));
515515 segp->base = vm->tss;
@@ -599,8 +599,9 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 83 unmodified lines
599599 vm_vaddr_t stack_vaddr;
600600 struct kvm_vcpu *vcpu;
601601
602− stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
603− DEFAULT_GUEST_STACK_VADDR_MIN);
602+ stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
603+ DEFAULT_GUEST_STACK_VADDR_MIN,
604+ MEM_REGION_DATA);
604605
605606 vcpu = __vm_vcpu_add(vm, vcpu_id);
606607 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
@@ -1093,8 +1094,8 @@ void vm_init_descriptor_tables(struct kvm_vm *vm) · 486 unmodified lines
10931094 extern void *idt_handlers;
10941095 int i;
10951096
1096− vm->idt = vm_vaddr_alloc_page(vm);
1097− vm->handlers = vm_vaddr_alloc_page(vm);
1097+ vm->idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1098+ vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
10981099 /* Handlers have the same address in both address spaces.*/
10991100 for (i = 0; i < NUM_INTERRUPTS; i++)
11001101 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
tools/testing/selftests/kvm/memslot_modification_stress_test.c+1−5
@@ -34,8 +34,6 @@ · 33 unmodified lines
3434 static int nr_vcpus = 1;
3535 static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
3636
37−static bool run_vcpus = true;
38−
3937 static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
4038 {
4139 struct kvm_vcpu *vcpu = vcpu_args->vcpu;
@@ -45,7 +43,7 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) · 3 unmodified lines
4543 run = vcpu->run;
4644
4745 /* Let the guest access its memory until a stop signal is received */
48− while (READ_ONCE(run_vcpus)) {
46+ while (!READ_ONCE(memstress_args.stop_vcpus)) {
4947 ret = _vcpu_run(vcpu);
5048 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
5149
@@ -109,8 +107,6 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 57 unmodified lines
109107
110108 add_remove_memslot(vm, p->delay, p->nr_iterations);
111109
112− run_vcpus = false;
113−
114110 memstress_join_vcpu_threads(nr_vcpus);
115111 pr_info("All vCPU threads joined\n");
116112
tools/testing/selftests/kvm/memslot_perf_test.c+207−106
@@ -20,20 +20,20 @@ · 19 unmodified lines
2020 #include <unistd.h>
2121
2222 #include <linux/compiler.h>
23+#include <linux/sizes.h>
2324
2425 #include <test_util.h>
2526 #include <kvm_util.h>
2627 #include <processor.h>
2728
28−#define MEM_SIZE ((512U << 20) + 4096)
29−#define MEM_SIZE_PAGES (MEM_SIZE / 4096)
30−#define MEM_GPA 0x10000000UL
29+#define MEM_EXTRA_SIZE SZ_64K
30+
31+#define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE)
32+#define MEM_GPA SZ_256M
3133 #define MEM_AUX_GPA MEM_GPA
3234 #define MEM_SYNC_GPA MEM_AUX_GPA
33−#define MEM_TEST_GPA (MEM_AUX_GPA + 4096)
34−#define MEM_TEST_SIZE (MEM_SIZE - 4096)
35−static_assert(MEM_SIZE % 4096 == 0, "invalid mem size");
36−static_assert(MEM_TEST_SIZE % 4096 == 0, "invalid mem test size");
35+#define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE)
36+#define MEM_TEST_SIZE (MEM_SIZE - MEM_EXTRA_SIZE)
3737
3838 /*
3939 * 32 MiB is max size that gets well over 100 iterations on 509 slots.
@@ -41,44 +41,38 @@ static_assert(MEM_TEST_SIZE % 4096 == 0, "invalid mem test size"); · 1 unmodified line
4141 * 8194 slots in use can then be tested (although with slightly
4242 * limited resolution).
4343 */
44−#define MEM_SIZE_MAP ((32U << 20) + 4096)
45−#define MEM_SIZE_MAP_PAGES (MEM_SIZE_MAP / 4096)
46−#define MEM_TEST_MAP_SIZE (MEM_SIZE_MAP - 4096)
47−#define MEM_TEST_MAP_SIZE_PAGES (MEM_TEST_MAP_SIZE / 4096)
48−static_assert(MEM_SIZE_MAP % 4096 == 0, "invalid map test region size");
49−static_assert(MEM_TEST_MAP_SIZE % 4096 == 0, "invalid map test region size");
50−static_assert(MEM_TEST_MAP_SIZE_PAGES % 2 == 0, "invalid map test region size");
51−static_assert(MEM_TEST_MAP_SIZE_PAGES > 2, "invalid map test region size");
44+#define MEM_SIZE_MAP (SZ_32M + MEM_EXTRA_SIZE)
45+#define MEM_TEST_MAP_SIZE (MEM_SIZE_MAP - MEM_EXTRA_SIZE)
5246
5347 /*
5448 * 128 MiB is min size that fills 32k slots with at least one page in each
5549 * while at the same time gets 100+ iterations in such test
50+ *
51+ * 2 MiB chunk size like a typical huge page
5652 */
57−#define MEM_TEST_UNMAP_SIZE (128U << 20)
58−#define MEM_TEST_UNMAP_SIZE_PAGES (MEM_TEST_UNMAP_SIZE / 4096)
59−/* 2 MiB chunk size like a typical huge page */
60−#define MEM_TEST_UNMAP_CHUNK_PAGES (2U << (20 - 12))
61−static_assert(MEM_TEST_UNMAP_SIZE <= MEM_TEST_SIZE,
62− "invalid unmap test region size");
63−static_assert(MEM_TEST_UNMAP_SIZE % 4096 == 0,
64− "invalid unmap test region size");
65−static_assert(MEM_TEST_UNMAP_SIZE_PAGES %
66− (2 * MEM_TEST_UNMAP_CHUNK_PAGES) == 0,
67− "invalid unmap test region size");
53+#define MEM_TEST_UNMAP_SIZE SZ_128M
54+#define MEM_TEST_UNMAP_CHUNK_SIZE SZ_2M
6855
6956 /*
7057 * For the move active test the middle of the test area is placed on
7158 * a memslot boundary: half lies in the memslot being moved, half in
7259 * other memslot(s).
7360 *
74− * When running this test with 32k memslots (32764, really) each memslot
75− * contains 4 pages.
76− * The last one additionally contains the remaining 21 pages of memory,
77− * for the total size of 25 pages.
78− * Hence, the maximum size here is 50 pages.
61+ * We have different number of memory slots, excluding the reserved
62+ * memory slot 0, on various architectures and configurations. The
63+ * memory size in this test is calculated by picking the maximal
64+ * last memory slot's memory size, with alignment to the largest
65+ * supported page size (64KB). In this way, the selected memory
66+ * size for this test is compatible with test_memslot_move_prepare().
67+ *
68+ * architecture slots memory-per-slot memory-on-last-slot
69+ * --------------------------------------------------------------
70+ * x86-4KB 32763 16KB 160KB
71+ * arm64-4KB 32766 16KB 112KB
72+ * arm64-16KB 32766 16KB 112KB
73+ * arm64-64KB 8192 64KB 128KB
7974 */
80−#define MEM_TEST_MOVE_SIZE_PAGES (50)
81−#define MEM_TEST_MOVE_SIZE (MEM_TEST_MOVE_SIZE_PAGES * 4096)
75+#define MEM_TEST_MOVE_SIZE (3 * SZ_64K)
8276 #define MEM_TEST_MOVE_GPA_DEST (MEM_GPA + MEM_SIZE)
8377 static_assert(MEM_TEST_MOVE_SIZE <= MEM_TEST_SIZE,
8478 "invalid move test region size");
@@ -100,6 +94,7 @@ struct vm_data { · 15 unmodified lines
10094 };
10195
10296 struct sync_area {
97+ uint32_t guest_page_size;
10398 atomic_bool start_flag;
10499 atomic_bool exit_flag;
105100 atomic_bool sync_flag;
@@ -192,14 +187,15 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages) · 86 unmodified lines
192187 uint64_t gpage, pgoffs;
193188 uint32_t slot, slotoffs;
194189 void *base;
190+ uint32_t guest_page_size = data->vm->page_size;
195191
196192 TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");
197− TEST_ASSERT(gpa < MEM_GPA + data->npages * 4096,
193+ TEST_ASSERT(gpa < MEM_GPA + data->npages * guest_page_size,
198194 "Too high gpa to translate");
199195 gpa -= MEM_GPA;
200196
201− gpage = gpa / 4096;
202− pgoffs = gpa % 4096;
197+ gpage = gpa / guest_page_size;
198+ pgoffs = gpa % guest_page_size;
203199 slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);
204200 slotoffs = gpage - (slot * data->pages_per_slot);
205201
@@ -217,14 +213,16 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages) · 11 unmodified lines
217213 }
218214
219215 base = data->hva_slots[slot];
220− return (uint8_t *)base + slotoffs * 4096 + pgoffs;
216+ return (uint8_t *)base + slotoffs * guest_page_size + pgoffs;
221217 }
222218
223219 static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)
224220 {
221+ uint32_t guest_page_size = data->vm->page_size;
222+
225223 TEST_ASSERT(slot < data->nslots, "Too high slot number");
226224
227− return MEM_GPA + slot * data->pages_per_slot * 4096;
225+ return MEM_GPA + slot * data->pages_per_slot * guest_page_size;
228226 }
229227
230228 static struct vm_data *alloc_vm(void)
@@ -241,81 +239,111 @@ static struct vm_data *alloc_vm(void) · 10 unmodified lines
241239 return data;
242240 }
243241
242+static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
243+ uint64_t pages_per_slot, uint64_t rempages)
244+{
245+ if (!pages_per_slot)
246+ return false;
247+
248+ if ((pages_per_slot * guest_page_size) % host_page_size)
249+ return false;
250+
251+ if ((rempages * guest_page_size) % host_page_size)
252+ return false;
253+
254+ return true;
255+}
256+
257+
258+static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
259+{
260+ uint32_t guest_page_size = data->vm->page_size;
261+ uint64_t mempages, pages_per_slot, rempages;
262+ uint64_t slots;
263+
264+ mempages = data->npages;
265+ slots = data->nslots;
266+ while (--slots > 1) {
267+ pages_per_slot = mempages / slots;
268+ rempages = mempages % pages_per_slot;
269+ if (check_slot_pages(host_page_size, guest_page_size,
270+ pages_per_slot, rempages))
271+ return slots + 1; /* slot 0 is reserved */
272+ }
273+
274+ return 0;
275+}
276+
244277 static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
245− void *guest_code, uint64_t mempages,
278+ void *guest_code, uint64_t mem_size,
246279 struct timespec *slot_runtime)
247280 {
248− uint32_t max_mem_slots;
249− uint64_t rempages;
281+ uint64_t mempages, rempages;
250282 uint64_t guest_addr;
251− uint32_t slot;
283+ uint32_t slot, host_page_size, guest_page_size;
252284 struct timespec tstart;
253285 struct sync_area *sync;
254286
255− max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
256− TEST_ASSERT(max_mem_slots > 1,
257− "KVM_CAP_NR_MEMSLOTS should be greater than 1");
258− TEST_ASSERT(nslots > 1 || nslots == -1,
259− "Slot count cap should be greater than 1");
260− if (nslots != -1)
261− max_mem_slots = min(max_mem_slots, (uint32_t)nslots);
262− pr_info_v("Allowed number of memory slots: %"PRIu32"\n", max_mem_slots);
287+ host_page_size = getpagesize();
288+ guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
289+ mempages = mem_size / guest_page_size;
263290
264− TEST_ASSERT(mempages > 1,
265− "Can't test without any memory");
291+ data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
292+ TEST_ASSERT(data->vm->page_size == guest_page_size, "Invalid VM page size");
266293
267294 data->npages = mempages;
268− data->nslots = max_mem_slots - 1;
269− data->pages_per_slot = mempages / data->nslots;
270− if (!data->pages_per_slot) {
271− *maxslots = mempages + 1;
295+ TEST_ASSERT(data->npages > 1, "Can't test without any memory");
296+ data->nslots = nslots;
297+ data->pages_per_slot = data->npages / data->nslots;
298+ rempages = data->npages % data->nslots;
299+ if (!check_slot_pages(host_page_size, guest_page_size,
300+ data->pages_per_slot, rempages)) {
301+ *maxslots = get_max_slots(data, host_page_size);
272302 return false;
273303 }
274304
275− rempages = mempages % data->nslots;
276305 data->hva_slots = malloc(sizeof(*data->hva_slots) * data->nslots);
277306 TEST_ASSERT(data->hva_slots, "malloc() fail");
278307
279308 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
280309
281310 pr_info_v("Adding slots 1..%i, each slot with %"PRIu64" pages + %"PRIu64" extra pages last\n",
282− max_mem_slots - 1, data->pages_per_slot, rempages);
311+ data->nslots, data->pages_per_slot, rempages);
283312
284313 clock_gettime(CLOCK_MONOTONIC, &tstart);
285− for (slot = 1, guest_addr = MEM_GPA; slot < max_mem_slots; slot++) {
314+ for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
286315 uint64_t npages;
287316
288317 npages = data->pages_per_slot;
289− if (slot == max_mem_slots - 1)
318+ if (slot == data->nslots)
290319 npages += rempages;
291320
292321 vm_userspace_mem_region_add(data->vm, VM_MEM_SRC_ANONYMOUS,
293322 guest_addr, slot, npages,
294323 0);
295− guest_addr += npages * 4096;
324+ guest_addr += npages * guest_page_size;
296325 }
297326 *slot_runtime = timespec_elapsed(tstart);
298327
299− for (slot = 0, guest_addr = MEM_GPA; slot < max_mem_slots - 1; slot++) {
328+ for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
300329 uint64_t npages;
301330 uint64_t gpa;
302331
303332 npages = data->pages_per_slot;
304− if (slot == max_mem_slots - 2)
333+ if (slot == data->nslots)
305334 npages += rempages;
306335
307− gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr,
308− slot + 1);
336+ gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot);
309337 TEST_ASSERT(gpa == guest_addr,
310338 "vm_phy_pages_alloc() failed\n");
311339
312− data->hva_slots[slot] = addr_gpa2hva(data->vm, guest_addr);
313− memset(data->hva_slots[slot], 0, npages * 4096);
340+ data->hva_slots[slot - 1] = addr_gpa2hva(data->vm, guest_addr);
341+ memset(data->hva_slots[slot - 1], 0, npages * guest_page_size);
314342
315− guest_addr += npages * 4096;
343+ guest_addr += npages * guest_page_size;
316344 }
317345
318− virt_map(data->vm, MEM_GPA, MEM_GPA, mempages);
346+ virt_map(data->vm, MEM_GPA, MEM_GPA, data->npages);
319347
320348 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
321349 atomic_init(&sync->start_flag, false);
@@ -414,6 +442,7 @@ static bool guest_perform_sync(void) · 92 unmodified lines
414442 static void guest_code_test_memslot_move(void)
415443 {
416444 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
445+ uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
417446 uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);
418447
419448 GUEST_SYNC(0);
@@ -424,7 +453,7 @@ static void guest_code_test_memslot_move(void) · 4 unmodified lines
424453 uintptr_t ptr;
425454
426455 for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;
427− ptr += 4096)
456+ ptr += page_size)
428457 *(uint64_t *)ptr = MEM_TEST_VAL_1;
429458
430459 /*
@@ -442,6 +471,7 @@ static void guest_code_test_memslot_move(void) · 11 unmodified lines
442471 static void guest_code_test_memslot_map(void)
443472 {
444473 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
474+ uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
445475
446476 GUEST_SYNC(0);
447477
@@ -451,14 +481,16 @@ static void guest_code_test_memslot_map(void) · 3 unmodified lines
451481 uintptr_t ptr;
452482
453483 for (ptr = MEM_TEST_GPA;
454− ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2; ptr += 4096)
484+ ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
485+ ptr += page_size)
455486 *(uint64_t *)ptr = MEM_TEST_VAL_1;
456487
457488 if (!guest_perform_sync())
458489 break;
459490
460491 for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
461− ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE; ptr += 4096)
492+ ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE;
493+ ptr += page_size)
462494 *(uint64_t *)ptr = MEM_TEST_VAL_2;
463495
464496 if (!guest_perform_sync())
@@ -505,6 +537,9 @@ static void guest_code_test_memslot_unmap(void) · 40 unmodified lines
505537
506538 static void guest_code_test_memslot_rw(void)
507539 {
540+ struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
541+ uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
542+
508543 GUEST_SYNC(0);
509544
510545 guest_spin_until_start();
@@ -513,14 +548,14 @@ static void guest_code_test_memslot_rw(void) · 2 unmodified lines
513548 uintptr_t ptr;
514549
515550 for (ptr = MEM_TEST_GPA;
516− ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += 4096)
551+ ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size)
517552 *(uint64_t *)ptr = MEM_TEST_VAL_1;
518553
519554 if (!guest_perform_sync())
520555 break;
521556
522− for (ptr = MEM_TEST_GPA + 4096 / 2;
523− ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += 4096) {
557+ for (ptr = MEM_TEST_GPA + page_size / 2;
558+ ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size) {
524559 uint64_t val = *(uint64_t *)ptr;
525560
526561 GUEST_ASSERT_1(val == MEM_TEST_VAL_2, val);
@@ -538,6 +573,7 @@ static bool test_memslot_move_prepare(struct vm_data *data, · 11 unmodified lines
538573 struct sync_area *sync,
539574 uint64_t *maxslots, bool isactive)
540575 {
576+ uint32_t guest_page_size = data->vm->page_size;
541577 uint64_t movesrcgpa, movetestgpa;
542578
543579 movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
@@ -546,7 +582,7 @@ static bool test_memslot_move_prepare(struct vm_data *data, · 2 unmodified lines
546582 uint64_t lastpages;
547583
548584 vm_gpa2hva(data, movesrcgpa, &lastpages);
549− if (lastpages < MEM_TEST_MOVE_SIZE_PAGES / 2) {
585+ if (lastpages * guest_page_size < MEM_TEST_MOVE_SIZE / 2) {
550586 *maxslots = 0;
551587 return false;
552588 }
@@ -592,8 +628,9 @@ static void test_memslot_do_unmap(struct vm_data *data, · 39 unmodified lines
592628 uint64_t offsp, uint64_t count)
593629 {
594630 uint64_t gpa, ctr;
631+ uint32_t guest_page_size = data->vm->page_size;
595632
596− for (gpa = MEM_TEST_GPA + offsp * 4096, ctr = 0; ctr < count; ) {
633+ for (gpa = MEM_TEST_GPA + offsp * guest_page_size, ctr = 0; ctr < count; ) {
597634 uint64_t npages;
598635 void *hva;
599636 int ret;
@@ -601,12 +638,12 @@ static void test_memslot_do_unmap(struct vm_data *data, · 1 unmodified line
601638 hva = vm_gpa2hva(data, gpa, &npages);
602639 TEST_ASSERT(npages, "Empty memory slot at gptr 0x%"PRIx64, gpa);
603640 npages = min(npages, count - ctr);
604− ret = madvise(hva, npages * 4096, MADV_DONTNEED);
641+ ret = madvise(hva, npages * guest_page_size, MADV_DONTNEED);
605642 TEST_ASSERT(!ret,
606643 "madvise(%p, MADV_DONTNEED) on VM memory should not fail for gptr 0x%"PRIx64,
607644 hva, gpa);
608645 ctr += npages;
609− gpa += npages * 4096;
646+ gpa += npages * guest_page_size;
610647 }
611648 TEST_ASSERT(ctr == count,
612649 "madvise(MADV_DONTNEED) should exactly cover all of the requested area");
@@ -617,11 +654,12 @@ static void test_memslot_map_unmap_check(struct vm_data *data, · 4 unmodified lines
617654 {
618655 uint64_t gpa;
619656 uint64_t *val;
657+ uint32_t guest_page_size = data->vm->page_size;
620658
621659 if (!map_unmap_verify)
622660 return;
623661
624− gpa = MEM_TEST_GPA + offsp * 4096;
662+ gpa = MEM_TEST_GPA + offsp * guest_page_size;
625663 val = (typeof(val))vm_gpa2hva(data, gpa, NULL);
626664 TEST_ASSERT(*val == valexp,
627665 "Guest written values should read back correctly before unmap (%"PRIu64" vs %"PRIu64" @ %"PRIx64")",
@@ -631,12 +669,14 @@ static void test_memslot_map_unmap_check(struct vm_data *data, · 3 unmodified lines
631669
632670 static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
633671 {
672+ uint32_t guest_page_size = data->vm->page_size;
673+ uint64_t guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
674+
634675 /*
635676 * Unmap the second half of the test area while guest writes to (maps)
636677 * the first half.
637678 */
638− test_memslot_do_unmap(data, MEM_TEST_MAP_SIZE_PAGES / 2,
639− MEM_TEST_MAP_SIZE_PAGES / 2);
679+ test_memslot_do_unmap(data, guest_pages / 2, guest_pages / 2);
640680
641681 /*
642682 * Wait for the guest to finish writing the first half of the test
@@ -647,10 +687,8 @@ static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync) · 4 unmodified lines
647687 */
648688 host_perform_sync(sync);
649689 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
650− test_memslot_map_unmap_check(data,
651− MEM_TEST_MAP_SIZE_PAGES / 2 - 1,
652− MEM_TEST_VAL_1);
653− test_memslot_do_unmap(data, 0, MEM_TEST_MAP_SIZE_PAGES / 2);
690+ test_memslot_map_unmap_check(data, guest_pages / 2 - 1, MEM_TEST_VAL_1);
691+ test_memslot_do_unmap(data, 0, guest_pages / 2);
654692
655693
656694 /*
@@ -663,16 +701,16 @@ static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync) · 6 unmodified lines
663701 * the test area.
664702 */
665703 host_perform_sync(sync);
666− test_memslot_map_unmap_check(data, MEM_TEST_MAP_SIZE_PAGES / 2,
667− MEM_TEST_VAL_2);
668− test_memslot_map_unmap_check(data, MEM_TEST_MAP_SIZE_PAGES - 1,
669− MEM_TEST_VAL_2);
704+ test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
705+ test_memslot_map_unmap_check(data, guest_pages - 1, MEM_TEST_VAL_2);
670706 }
671707
672708 static void test_memslot_unmap_loop_common(struct vm_data *data,
673709 struct sync_area *sync,
674710 uint64_t chunk)
675711 {
712+ uint32_t guest_page_size = data->vm->page_size;
713+ uint64_t guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
676714 uint64_t ctr;
677715
678716 /*
@@ -684,42 +722,49 @@ static void test_memslot_unmap_loop_common(struct vm_data *data, · 5 unmodified lines
684722 */
685723 host_perform_sync(sync);
686724 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
687− for (ctr = 0; ctr < MEM_TEST_UNMAP_SIZE_PAGES / 2; ctr += chunk)
725+ for (ctr = 0; ctr < guest_pages / 2; ctr += chunk)
688726 test_memslot_do_unmap(data, ctr, chunk);
689727
690728 /* Likewise, but for the opposite host / guest areas */
691729 host_perform_sync(sync);
692− test_memslot_map_unmap_check(data, MEM_TEST_UNMAP_SIZE_PAGES / 2,
693− MEM_TEST_VAL_2);
694− for (ctr = MEM_TEST_UNMAP_SIZE_PAGES / 2;
695− ctr < MEM_TEST_UNMAP_SIZE_PAGES; ctr += chunk)
730+ test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
731+ for (ctr = guest_pages / 2; ctr < guest_pages; ctr += chunk)
696732 test_memslot_do_unmap(data, ctr, chunk);
697733 }
698734
699735 static void test_memslot_unmap_loop(struct vm_data *data,
700736 struct sync_area *sync)
701737 {
702− test_memslot_unmap_loop_common(data, sync, 1);
738+ uint32_t host_page_size = getpagesize();
739+ uint32_t guest_page_size = data->vm->page_size;
740+ uint64_t guest_chunk_pages = guest_page_size >= host_page_size ?
741+ 1 : host_page_size / guest_page_size;
742+
743+ test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
703744 }
704745
705746 static void test_memslot_unmap_loop_chunked(struct vm_data *data,
706747 struct sync_area *sync)
707748 {
708− test_memslot_unmap_loop_common(data, sync, MEM_TEST_UNMAP_CHUNK_PAGES);
749+ uint32_t guest_page_size = data->vm->page_size;
750+ uint64_t guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
751+
752+ test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
709753 }
710754
711755 static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
712756 {
713757 uint64_t gptr;
758+ uint32_t guest_page_size = data->vm->page_size;
714759
715− for (gptr = MEM_TEST_GPA + 4096 / 2;
716− gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += 4096)
760+ for (gptr = MEM_TEST_GPA + guest_page_size / 2;
761+ gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size)
717762 *(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
718763
719764 host_perform_sync(sync);
720765
721766 for (gptr = MEM_TEST_GPA;
722− gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += 4096) {
767+ gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size) {
723768 uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
724769 uint64_t val = *vptr;
725770
@@ -748,7 +793,7 @@ static bool test_execute(int nslots, uint64_t *maxslots, · 22 unmodified lines
748793 struct timespec *slot_runtime,
749794 struct timespec *guest_runtime)
750795 {
751− uint64_t mem_size = tdata->mem_size ? : MEM_SIZE_PAGES;
796+ uint64_t mem_size = tdata->mem_size ? : MEM_SIZE;
752797 struct vm_data *data;
753798 struct sync_area *sync;
754799 struct timespec tstart;
@@ -763,6 +808,7 @@ static bool test_execute(int nslots, uint64_t *maxslots, · 8 unmodified lines
763808
764809 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
765810
811+ sync->guest_page_size = data->vm->page_size;
766812 if (tdata->prepare &&
767813 !tdata->prepare(data, sync, maxslots)) {
768814 ret = false;
@@ -796,19 +842,19 @@ exit_free: · 27 unmodified lines
796842 static const struct test_data tests[] = {
797843 {
798844 .name = "map",
799− .mem_size = MEM_SIZE_MAP_PAGES,
845+ .mem_size = MEM_SIZE_MAP,
800846 .guest_code = guest_code_test_memslot_map,
801847 .loop = test_memslot_map_loop,
802848 },
803849 {
804850 .name = "unmap",
805− .mem_size = MEM_TEST_UNMAP_SIZE_PAGES + 1,
851+ .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
806852 .guest_code = guest_code_test_memslot_unmap,
807853 .loop = test_memslot_unmap_loop,
808854 },
809855 {
810856 .name = "unmap chunked",
811− .mem_size = MEM_TEST_UNMAP_SIZE_PAGES + 1,
857+ .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
812858 .guest_code = guest_code_test_memslot_unmap,
813859 .loop = test_memslot_unmap_loop_chunked,
814860 },
@@ -866,9 +912,46 @@ static void help(char *name, struct test_args *targs) · 51 unmodified lines
866912 pr_info("%d: %s\n", ctr, tests[ctr].name);
867913 }
868914
915+static bool check_memory_sizes(void)
916+{
917+ uint32_t host_page_size = getpagesize();
918+ uint32_t guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
919+
920+ if (host_page_size > SZ_64K || guest_page_size > SZ_64K) {
921+ pr_info("Unsupported page size on host (0x%x) or guest (0x%x)\n",
922+ host_page_size, guest_page_size);
923+ return false;
924+ }
925+
926+ if (MEM_SIZE % guest_page_size ||
927+ MEM_TEST_SIZE % guest_page_size) {
928+ pr_info("invalid MEM_SIZE or MEM_TEST_SIZE\n");
929+ return false;
930+ }
931+
932+ if (MEM_SIZE_MAP % guest_page_size ||
933+ MEM_TEST_MAP_SIZE % guest_page_size ||
934+ (MEM_TEST_MAP_SIZE / guest_page_size) <= 2 ||
935+ (MEM_TEST_MAP_SIZE / guest_page_size) % 2) {
936+ pr_info("invalid MEM_SIZE_MAP or MEM_TEST_MAP_SIZE\n");
937+ return false;
938+ }
939+
940+ if (MEM_TEST_UNMAP_SIZE > MEM_TEST_SIZE ||
941+ MEM_TEST_UNMAP_SIZE % guest_page_size ||
942+ (MEM_TEST_UNMAP_SIZE / guest_page_size) %
943+ (2 * MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size)) {
944+ pr_info("invalid MEM_TEST_UNMAP_SIZE or MEM_TEST_UNMAP_CHUNK_SIZE\n");
945+ return false;
946+ }
947+
948+ return true;
949+}
950+
869951 static bool parse_args(int argc, char *argv[],
870952 struct test_args *targs)
871953 {
954+ uint32_t max_mem_slots;
872955 int opt;
873956
874957 while ((opt = getopt(argc, argv, "hvds:f:e:l:r:")) != -1) {
@@ -885,8 +968,8 @@ static bool parse_args(int argc, char *argv[], · 10 unmodified lines
885968 break;
886969 case 's':
887970 targs->nslots = atoi_paranoid(optarg);
888− if (targs->nslots <= 0 && targs->nslots != -1) {
889− pr_info("Slot count cap has to be positive or -1 for no cap\n");
971+ if (targs->nslots <= 1 && targs->nslots != -1) {
972+ pr_info("Slot count cap must be larger than 1 or -1 for no cap\n");
890973 return false;
891974 }
892975 break;
@@ -920,6 +1003,21 @@ static bool parse_args(int argc, char *argv[], · 27 unmodified lines
9201003 return false;
9211004 }
9221005
1006+ max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
1007+ if (max_mem_slots <= 1) {
1008+ pr_info("KVM_CAP_NR_MEMSLOTS should be greater than 1\n");
1009+ return false;
1010+ }
1011+
1012+ /* Memory slot 0 is reserved */
1013+ if (targs->nslots == -1)
1014+ targs->nslots = max_mem_slots - 1;
1015+ else
1016+ targs->nslots = min_t(int, targs->nslots, max_mem_slots) - 1;
1017+
1018+ pr_info_v("Allowed Number of memory slots: %"PRIu32"\n",
1019+ targs->nslots + 1);
1020+
9231021 return true;
9241022 }
9251023
@@ -994,6 +1092,9 @@ int main(int argc, char *argv[]) · 68 unmodified lines
9941092 struct test_result rbestslottime;
9951093 int tctr;
9961094
1095+ if (!check_memory_sizes())
1096+ return -1;
1097+
9971098 if (!parse_args(argc, argv, &targs))
9981099 return -1;
9991100
virt/kvm/Kconfig+6−0
@@ -33,6 +33,12 @@ config HAVE_KVM_DIRTY_RING_ACQ_REL · 32 unmodified lines
3333 bool
3434 select HAVE_KVM_DIRTY_RING
3535
36+# Allow enabling both the dirty bitmap and dirty ring. Only architectures
37+# that need to dirty memory outside of a vCPU context should select this.
38+config NEED_KVM_DIRTY_RING_WITH_BITMAP
39+ bool
40+ depends on HAVE_KVM_DIRTY_RING
41+
3642 config HAVE_KVM_EVENTFD
3743 bool
3844 select EVENTFD
virt/kvm/dirty_ring.c+44−2
@@ -21,12 +21,26 @@ u32 kvm_dirty_ring_get_rsvd_entries(void) · 20 unmodified lines
2121 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
2222 }
2323
24+bool kvm_use_dirty_bitmap(struct kvm *kvm)
25+{
26+ lockdep_assert_held(&kvm->slots_lock);
27+
28+ return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap;
29+}
30+
31+#ifndef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
32+bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
33+{
34+ return false;
35+}
36+#endif
37+
2438 static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
2539 {
2640 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
2741 }
2842
29−bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
43+static bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
3044 {
3145 return kvm_dirty_ring_used(ring) >= ring->soft_limit;
3246 }
@@ -142,13 +156,19 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring) · 109 unmodified lines
142156
143157 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
144158
159+ /*
160+ * The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared
161+ * by the VCPU thread next time when it enters the guest.
162+ */
163+
145164 trace_kvm_dirty_ring_reset(ring);
146165
147166 return count;
148167 }
149168
150−void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset)
169+void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset)
151170 {
171+ struct kvm_dirty_ring *ring = &vcpu->dirty_ring;
152172 struct kvm_dirty_gfn *entry;
153173
154174 /* It should never get full */
@@ -166,6 +186,28 @@ void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset) · 11 unmodified lines
166186 kvm_dirty_gfn_set_dirtied(entry);
167187 ring->dirty_index++;
168188 trace_kvm_dirty_ring_push(ring, slot, offset);
189+
190+ if (kvm_dirty_ring_soft_full(ring))
191+ kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
192+}
193+
194+bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu)
195+{
196+ /*
197+ * The VCPU isn't runnable when the dirty ring becomes soft full.
198+ * The KVM_REQ_DIRTY_RING_SOFT_FULL event is always set to prevent
199+ * the VCPU from running until the dirty pages are harvested and
200+ * the dirty ring is reset by userspace.
201+ */
202+ if (kvm_check_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu) &&
203+ kvm_dirty_ring_soft_full(&vcpu->dirty_ring)) {
204+ kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
205+ vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
206+ trace_kvm_dirty_ring_exit(vcpu);
207+ return true;
208+ }
209+
210+ return false;
169211 }
170212
171213 struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)
virt/kvm/kvm_main.c+53−12
@@ -1617,7 +1617,7 @@ static int kvm_prepare_memory_region(struct kvm *kvm, · 1616 unmodified lines
16171617 new->dirty_bitmap = NULL;
16181618 else if (old && old->dirty_bitmap)
16191619 new->dirty_bitmap = old->dirty_bitmap;
1620− else if (!kvm->dirty_ring_size) {
1620+ else if (kvm_use_dirty_bitmap(kvm)) {
16211621 r = kvm_alloc_dirty_bitmap(new);
16221622 if (r)
16231623 return r;
@@ -2068,8 +2068,8 @@ int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, · 444 unmodified lines
20682068 unsigned long n;
20692069 unsigned long any = 0;
20702070
2071− /* Dirty ring tracking is exclusive to dirty log tracking */
2072− if (kvm->dirty_ring_size)
2071+ /* Dirty ring tracking may be exclusive to dirty log tracking */
2072+ if (!kvm_use_dirty_bitmap(kvm))
20732073 return -ENXIO;
20742074
20752075 *memslot = NULL;
@@ -2133,8 +2133,8 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) · 57 unmodified lines
21332133 unsigned long *dirty_bitmap_buffer;
21342134 bool flush;
21352135
2136− /* Dirty ring tracking is exclusive to dirty log tracking */
2137− if (kvm->dirty_ring_size)
2136+ /* Dirty ring tracking may be exclusive to dirty log tracking */
2137+ if (!kvm_use_dirty_bitmap(kvm))
21382138 return -ENXIO;
21392139
21402140 as_id = log->slot >> 16;
@@ -2245,8 +2245,8 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 104 unmodified lines
22452245 unsigned long *dirty_bitmap_buffer;
22462246 bool flush;
22472247
2248− /* Dirty ring tracking is exclusive to dirty log tracking */
2249− if (kvm->dirty_ring_size)
2248+ /* Dirty ring tracking may be exclusive to dirty log tracking */
2249+ if (!kvm_use_dirty_bitmap(kvm))
22502250 return -ENXIO;
22512251
22522252 as_id = log->slot >> 16;
@@ -3321,18 +3321,19 @@ void mark_page_dirty_in_slot(struct kvm *kvm, · 1068 unmodified lines
33213321 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
33223322
33233323 #ifdef CONFIG_HAVE_KVM_DIRTY_RING
3324− if (WARN_ON_ONCE(!vcpu) || WARN_ON_ONCE(vcpu->kvm != kvm))
3324+ if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
33253325 return;
3326+
3327+ WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
33263328 #endif
33273329
33283330 if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
33293331 unsigned long rel_gfn = gfn - memslot->base_gfn;
33303332 u32 slot = (memslot->as_id << 16) | memslot->id;
33313333
3332− if (kvm->dirty_ring_size)
3333− kvm_dirty_ring_push(&vcpu->dirty_ring,
3334− slot, rel_gfn);
3335− else
3334+ if (kvm->dirty_ring_size && vcpu)
3335+ kvm_dirty_ring_push(vcpu, slot, rel_gfn);
3336+ else if (memslot->dirty_bitmap)
33363337 set_bit_le(rel_gfn, memslot->dirty_bitmap);
33373338 }
33383339 }
@@ -4499,6 +4500,9 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) · 1160 unmodified lines
44994500 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
45004501 #else
45014502 return 0;
4503+#endif
4504+#ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
4505+ case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
45024506 #endif
45034507 case KVM_CAP_BINARY_STATS_FD:
45044508 case KVM_CAP_SYSTEM_EVENT_DATA:
@@ -4575,6 +4579,20 @@ int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, · 70 unmodified lines
45754579 return -EINVAL;
45764580 }
45774581
4582+static bool kvm_are_all_memslots_empty(struct kvm *kvm)
4583+{
4584+ int i;
4585+
4586+ lockdep_assert_held(&kvm->slots_lock);
4587+
4588+ for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4589+ if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
4590+ return false;
4591+ }
4592+
4593+ return true;
4594+}
4595+
45784596 static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
45794597 struct kvm_enable_cap *cap)
45804598 {
@@ -4605,6 +4623,29 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, · 24 unmodified lines
46054623 return -EINVAL;
46064624
46074625 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
4626+ case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
4627+ int r = -EINVAL;
4628+
4629+ if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
4630+ !kvm->dirty_ring_size || cap->flags)
4631+ return r;
4632+
4633+ mutex_lock(&kvm->slots_lock);
4634+
4635+ /*
4636+ * For simplicity, allow enabling ring+bitmap if and only if
4637+ * there are no memslots, e.g. to ensure all memslots allocate
4638+ * a bitmap after the capability is enabled.
4639+ */
4640+ if (kvm_are_all_memslots_empty(kvm)) {
4641+ kvm->dirty_ring_with_bitmap = true;
4642+ r = 0;
4643+ }
4644+
4645+ mutex_unlock(&kvm->slots_lock);
4646+
4647+ return r;
4648+ }
46084649 default:
46094650 return kvm_vm_ioctl_enable_cap(kvm, cap);
46104651 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t