History

Merge tag 'kvmarm-6.2' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 updates for 6.2 - Enable the per-vcpu dirty-ring tracking mechanism, together with an option to keep the good old dirty log around for pages that are dirtied by something other than a vcpu. - Switch to the relaxed parallel fault handling, using RCU to delay page table reclaim and giving better performance under load. - Relax the MTE ABI, allowing a VMM to use the MAP_SHARED mapping option, which multi-process VMMs such as crosvm rely on. - Merge the pKVM shadow vcpu state tracking that allows the hypervisor to have its own view of a vcpu, keeping that state private. - Add support for the PMUv3p5 architecture revision, bringing support for 64bit counters on systems that support it, and fix the no-quite-compliant CHAIN-ed counter support for the machines that actually exist out there. - Fix a handful of minor issues around 52bit VA/PA support (64kB pages only) as a prefix of the oncoming support for 4kB and 16kB pages. - Add/Enable/Fix a bunch of selftests covering memslots, breakpoints, stage-2 faults and access tracking. You name it, we got it, we probably broke it. - Pick a small set of documentation and spelling fixes, because no good merge window would be complete without those. As a side effect, this tag also drags: - The 'kvmarm-fixes-6.1-3' tag as a dependency to the dirty-ring series - A shared branch with the arm64 tree that repaints all the system registers to match the ARM ARM's naming, and resulting in interesting conflicts

Paolo Bonzini committed 3 years ago

Commit eb561892 parents 1e79a9e 753d73491 files · +6077 −1770Browse files

Unified · Split

91 files
  1. Documentation/virt/kvm/api.rst +31 −10
  2. Documentation/virt/kvm/arm/pvtime.rst +8 −6
  3. Documentation/virt/kvm/devices/arm-vgic-its.rst +4 −1
  4. Documentation/virt/kvm/devices/vcpu.rst +2 −0
  5. arch/arm64/Kconfig +1 −0
  6. arch/arm64/include/asm/kvm_arm.h +6 −2
  7. arch/arm64/include/asm/kvm_asm.h +5 −2
  8. arch/arm64/include/asm/kvm_host.h +74 −2
  9. arch/arm64/include/asm/kvm_hyp.h +3 −0
  10. arch/arm64/include/asm/kvm_mmu.h +1 −1
  11. arch/arm64/include/asm/kvm_pgtable.h +147 −28
  12. arch/arm64/include/asm/kvm_pkvm.h +38 −0
  13. arch/arm64/include/asm/mte.h +64 −1
  14. arch/arm64/include/asm/pgtable.h +2 −2
  15. arch/arm64/include/asm/sysreg.h +0 −138
  16. arch/arm64/include/uapi/asm/kvm.h +1 −0
  17. arch/arm64/kernel/cpufeature.c +107 −105
  18. arch/arm64/kernel/elfcore.c +1 −1
  19. arch/arm64/kernel/hibernate.c +1 −1
  20. arch/arm64/kernel/image-vars.h +0 −15
  21. arch/arm64/kernel/mte.c +10 −11
  22. arch/arm64/kvm/Kconfig +2 −0
  23. arch/arm64/kvm/arm.c +51 −39
  24. arch/arm64/kvm/guest.c +11 −7
  25. arch/arm64/kvm/hyp/hyp-constants.c +3 −0
  26. arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +21 −4
  27. arch/arm64/kvm/hyp/include/nvhe/memory.h +27 −0
  28. arch/arm64/kvm/hyp/include/nvhe/mm.h +5 −13
  29. arch/arm64/kvm/hyp/include/nvhe/pkvm.h +68 −0
  30. arch/arm64/kvm/hyp/include/nvhe/spinlock.h +9 −1
  31. arch/arm64/kvm/hyp/nvhe/cache.S +11 −0
  32. arch/arm64/kvm/hyp/nvhe/hyp-main.c +108 −2
  33. arch/arm64/kvm/hyp/nvhe/hyp-smp.c +2 −0
  34. arch/arm64/kvm/hyp/nvhe/mem_protect.c +476 −45
  35. arch/arm64/kvm/hyp/nvhe/mm.c +163 −4
  36. arch/arm64/kvm/hyp/nvhe/page_alloc.c +7 −22
  37. arch/arm64/kvm/hyp/nvhe/pkvm.c +436 −0
  38. arch/arm64/kvm/hyp/nvhe/setup.c +61 −37
  39. arch/arm64/kvm/hyp/pgtable.c +343 −309
  40. arch/arm64/kvm/hyp/vhe/Makefile +1 −1
  41. arch/arm64/kvm/mmu.c +132 −61
  42. arch/arm64/kvm/pkvm.c +122 −16
  43. arch/arm64/kvm/pmu-emul.c +196 −286
  44. arch/arm64/kvm/reset.c +0 −29
  45. arch/arm64/kvm/sys_regs.c +135 −22
  46. arch/arm64/kvm/vgic/vgic-its.c +20 −0
  47. arch/arm64/lib/mte.S +1 −1
  48. arch/arm64/mm/copypage.c +5 −2
  49. arch/arm64/mm/fault.c +3 −1
  50. arch/arm64/mm/mteswap.c +6 −10
  51. arch/arm64/tools/gen-sysreg.awk +1 −1
  52. arch/arm64/tools/sysreg +754 −0
  53. arch/x86/include/asm/kvm_host.h +0 −2
  54. arch/x86/kvm/x86.c +6 −9
  55. fs/proc/page.c +2 −1
  56. include/kvm/arm_pmu.h +13 −2
  57. include/kvm/arm_vgic.h +1 −0
  58. include/linux/kernel-page-flags.h +1 −0
  59. include/linux/kvm_dirty_ring.h +12 −8
  60. include/linux/kvm_host.h +6 −4
  61. include/linux/page-flags.h +2 −1
  62. include/trace/events/mmflags.h +5 −4
  63. include/uapi/linux/kvm.h +1 −0
  64. mm/Kconfig +8 −0
  65. mm/huge_memory.c +2 −1
  66. tools/include/linux/bitfield.h +176 −0
  67. tools/testing/selftests/kvm/.gitignore +1 −0
  68. tools/testing/selftests/kvm/Makefile +3 −0
  69. tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c +2 −1
  70. tools/testing/selftests/kvm/aarch64/debug-exceptions.c +239 −72
  71. tools/testing/selftests/kvm/aarch64/page_fault_test.c +1117 −0
  72. tools/testing/selftests/kvm/access_tracking_perf_test.c +1 −7
  73. tools/testing/selftests/kvm/demand_paging_test.c +30 −198
  74. tools/testing/selftests/kvm/dirty_log_test.c +39 −14
  75. tools/testing/selftests/kvm/include/aarch64/processor.h +29 −6
  76. tools/testing/selftests/kvm/include/kvm_util_base.h +29 −2
  77. tools/testing/selftests/kvm/include/memstress.h +3 −0
  78. tools/testing/selftests/kvm/include/userfaultfd_util.h +45 −0
  79. tools/testing/selftests/kvm/lib/aarch64/processor.c +34 −21
  80. tools/testing/selftests/kvm/lib/elf.c +2 −1
  81. tools/testing/selftests/kvm/lib/kvm_util.c +53 −31
  82. tools/testing/selftests/kvm/lib/memstress.c +3 −0
  83. tools/testing/selftests/kvm/lib/riscv/processor.c +17 −12
  84. tools/testing/selftests/kvm/lib/s390x/processor.c +5 −3
  85. tools/testing/selftests/kvm/lib/userfaultfd_util.c +186 −0
  86. tools/testing/selftests/kvm/lib/x86_64/processor.c +7 −6
  87. tools/testing/selftests/kvm/memslot_modification_stress_test.c +1 −5
  88. tools/testing/selftests/kvm/memslot_perf_test.c +207 −106
  89. virt/kvm/Kconfig +6 −0
  90. virt/kvm/dirty_ring.c +44 −2
  91. virt/kvm/kvm_main.c +53 −12
Documentation/virt/kvm/api.rst+31−10
@@ -7418,8 +7418,9 @@ hibernation of the host; however the VMM needs to manually save/restore the · 7417 unmodified lines
7418tags as appropriate if the VM is migrated.7418tags as appropriate if the VM is migrated.
7419 7419
7420When this capability is enabled all memory in memslots must be mapped as7420When this capability is enabled all memory in memslots must be mapped as
7421not-shareable (no MAP_SHARED), attempts to create a memslot with a7421``MAP_ANONYMOUS`` or with a RAM-based file mapping (``tmpfs``, ``memfd``),
7422MAP_SHARED mmap will result in an -EINVAL return.7422attempts to create a memslot with an invalid mmap will result in an
7423-EINVAL return.
7423 7424
7424When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to7425When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to
7425perform a bulk copy of tags to/from the guest.7426perform a bulk copy of tags to/from the guest.
@@ -7954,7 +7955,7 @@ regardless of what has actually been exposed through the CPUID leaf. · 528 unmodified lines
79548.29 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL79558.29 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL
7955----------------------------------------------------------7956----------------------------------------------------------
7956 7957
7957:Architectures: x867958:Architectures: x86, arm64
7958:Parameters: args[0] - size of the dirty log ring7959:Parameters: args[0] - size of the dirty log ring
7959 7960
7960KVM is capable of tracking dirty memory using ring buffers that are7961KVM is capable of tracking dirty memory using ring buffers that are
@@ -8036,13 +8037,6 @@ flushing is done by the KVM_GET_DIRTY_LOG ioctl). To achieve that, one · 75 unmodified lines
8036needs to kick the vcpu out of KVM_RUN using a signal. The resulting8037needs to kick the vcpu out of KVM_RUN using a signal. The resulting
8037vmexit ensures that all dirty GFNs are flushed to the dirty rings.8038vmexit ensures that all dirty GFNs are flushed to the dirty rings.
8038 8039
8039NOTE: the capability KVM_CAP_DIRTY_LOG_RING and the corresponding
8040ioctl KVM_RESET_DIRTY_RINGS are mutual exclusive to the existing ioctls
8041KVM_GET_DIRTY_LOG and KVM_CLEAR_DIRTY_LOG. After enabling
8042KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual
8043machine will switch to ring-buffer dirty page tracking and further
8044KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail.
8045
8046NOTE: KVM_CAP_DIRTY_LOG_RING_ACQ_REL is the only capability that8040NOTE: KVM_CAP_DIRTY_LOG_RING_ACQ_REL is the only capability that
8047should be exposed by weakly ordered architecture, in order to indicate8041should be exposed by weakly ordered architecture, in order to indicate
8048the additional memory ordering requirements imposed on userspace when8042the additional memory ordering requirements imposed on userspace when
@@ -8051,6 +8045,33 @@ Architecture with TSO-like ordering (such as x86) are allowed to · 2 unmodified lines
8051expose both KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL8045expose both KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL
8052to userspace.8046to userspace.
8053 8047
8048After enabling the dirty rings, the userspace needs to detect the
8049capability of KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP to see whether the
8050ring structures can be backed by per-slot bitmaps. With this capability
8051advertised, it means the architecture can dirty guest pages without
8052vcpu/ring context, so that some of the dirty information will still be
8053maintained in the bitmap structure. KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP
8054can't be enabled if the capability of KVM_CAP_DIRTY_LOG_RING_ACQ_REL
8055hasn't been enabled, or any memslot has been existing.
8056
8057Note that the bitmap here is only a backup of the ring structure. The
8058use of the ring and bitmap combination is only beneficial if there is
8059only a very small amount of memory that is dirtied out of vcpu/ring
8060context. Otherwise, the stand-alone per-slot bitmap mechanism needs to
8061be considered.
8062
8063To collect dirty bits in the backup bitmap, userspace can use the same
8064KVM_GET_DIRTY_LOG ioctl. KVM_CLEAR_DIRTY_LOG isn't needed as long as all
8065the generation of the dirty bits is done in a single pass. Collecting
8066the dirty bitmap should be the very last thing that the VMM does before
8067considering the state as complete. VMM needs to ensure that the dirty
8068state is final and avoid missing dirty pages from another ioctl ordered
8069after the bitmap collection.
8070
8071NOTE: One example of using the backup bitmap is saving arm64 vgic/its
8072tables through KVM_DEV_ARM_{VGIC_GRP_CTRL, ITS_SAVE_TABLES} command on
8073KVM device "kvm-arm-vgic-its" when dirty ring is enabled.
8074
80548.30 KVM_CAP_XEN_HVM80758.30 KVM_CAP_XEN_HVM
8055--------------------8076--------------------
8056 8077
Documentation/virt/kvm/arm/pvtime.rst+8−6
@@ -23,21 +23,23 @@ the PV_TIME_FEATURES hypercall should be probed using the SMCCC 1.1 · 22 unmodified lines
23ARCH_FEATURES mechanism before calling it.23ARCH_FEATURES mechanism before calling it.
24 24
25PV_TIME_FEATURES25PV_TIME_FEATURES
26 ============= ======== ==========26
27 ============= ======== =================================================
27 Function ID: (uint32) 0xC500002028 Function ID: (uint32) 0xC5000020
28 PV_call_id: (uint32) The function to query for support.29 PV_call_id: (uint32) The function to query for support.
29 Currently only PV_TIME_ST is supported.30 Currently only PV_TIME_ST is supported.
30 Return value: (int64) NOT_SUPPORTED (-1) or SUCCESS (0) if the relevant31 Return value: (int64) NOT_SUPPORTED (-1) or SUCCESS (0) if the relevant
31 PV-time feature is supported by the hypervisor.32 PV-time feature is supported by the hypervisor.
32 ============= ======== ==========33 ============= ======== =================================================
33 34
34PV_TIME_ST35PV_TIME_ST
35 ============= ======== ==========36
37 ============= ======== ==============================================
36 Function ID: (uint32) 0xC500002138 Function ID: (uint32) 0xC5000021
37 Return value: (int64) IPA of the stolen time data structure for this39 Return value: (int64) IPA of the stolen time data structure for this
38 VCPU. On failure:40 VCPU. On failure:
39 NOT_SUPPORTED (-1)41 NOT_SUPPORTED (-1)
40 ============= ======== ==========42 ============= ======== ==============================================
41 43
42The IPA returned by PV_TIME_ST should be mapped by the guest as normal memory44The IPA returned by PV_TIME_ST should be mapped by the guest as normal memory
43with inner and outer write back caching attributes, in the inner shareable45with inner and outer write back caching attributes, in the inner shareable
@@ -76,5 +78,5 @@ It is advisable that one or more 64k pages are set aside for the purpose of · 32 unmodified lines
76these structures and not used for other purposes, this enables the guest to map78these structures and not used for other purposes, this enables the guest to map
77the region using 64k pages and avoids conflicting attributes with other memory.79the region using 64k pages and avoids conflicting attributes with other memory.
78 80
79For the user space interface see Documentation/virt/kvm/devices/vcpu.rst81For the user space interface see
80section "3. GROUP: KVM_ARM_VCPU_PVTIME_CTRL".82:ref:`Documentation/virt/kvm/devices/vcpu.rst <kvm_arm_vcpu_pvtime_ctrl>`.
Documentation/virt/kvm/devices/arm-vgic-its.rst+4−1
@@ -52,7 +52,10 @@ KVM_DEV_ARM_VGIC_GRP_CTRL · 51 unmodified lines
52 52
53 KVM_DEV_ARM_ITS_SAVE_TABLES53 KVM_DEV_ARM_ITS_SAVE_TABLES
54 save the ITS table data into guest RAM, at the location provisioned54 save the ITS table data into guest RAM, at the location provisioned
55 by the guest in corresponding registers/table entries.55 by the guest in corresponding registers/table entries. Should userspace
56 require a form of dirty tracking to identify which pages are modified
57 by the saving process, it should use a bitmap even if using another
58 mechanism to track the memory dirtied by the vCPUs.
56 59
57 The layout of the tables in guest memory defines an ABI. The entries60 The layout of the tables in guest memory defines an ABI. The entries
58 are laid out in little endian format as described in the last paragraph.61 are laid out in little endian format as described in the last paragraph.
Documentation/virt/kvm/devices/vcpu.rst+2−0
@@ -171,6 +171,8 @@ configured values on other VCPUs. Userspace should configure the interrupt · 170 unmodified lines
171numbers on at least one VCPU after creating all VCPUs and before running any171numbers on at least one VCPU after creating all VCPUs and before running any
172VCPUs.172VCPUs.
173 173
174.. _kvm_arm_vcpu_pvtime_ctrl:
175
1743. GROUP: KVM_ARM_VCPU_PVTIME_CTRL1763. GROUP: KVM_ARM_VCPU_PVTIME_CTRL
175==================================177==================================
176 178
arch/arm64/Kconfig+1−0
@@ -1965,6 +1965,7 @@ config ARM64_MTE · 1964 unmodified lines
1965 depends on ARM64_PAN1965 depends on ARM64_PAN
1966 select ARCH_HAS_SUBPAGE_FAULTS1966 select ARCH_HAS_SUBPAGE_FAULTS
1967 select ARCH_USES_HIGH_VMA_FLAGS1967 select ARCH_USES_HIGH_VMA_FLAGS
1968 select ARCH_USES_PG_ARCH_X
1968 help1969 help
1969 Memory Tagging (part of the ARMv8.5 Extensions) provides1970 Memory Tagging (part of the ARMv8.5 Extensions) provides
1970 architectural support for run-time, always-on detection of1971 architectural support for run-time, always-on detection of
arch/arm64/include/asm/kvm_arm.h+6−2
@@ -135,7 +135,7 @@ · 134 unmodified lines
135 * 40 bits wide (T0SZ = 24). Systems with a PARange smaller than 40 bits are135 * 40 bits wide (T0SZ = 24). Systems with a PARange smaller than 40 bits are
136 * not known to exist and will break with this configuration.136 * not known to exist and will break with this configuration.
137 *137 *
138 * The VTCR_EL2 is configured per VM and is initialised in kvm_arm_setup_stage2().138 * The VTCR_EL2 is configured per VM and is initialised in kvm_init_stage2_mmu.
139 *139 *
140 * Note that when using 4K pages, we concatenate two first level page tables140 * Note that when using 4K pages, we concatenate two first level page tables
141 * together. With 16K pages, we concatenate 16 first level page tables.141 * together. With 16K pages, we concatenate 16 first level page tables.
@@ -340,9 +340,13 @@ · 198 unmodified lines
340 * We have340 * We have
341 * PAR [PA_Shift - 1 : 12] = PA [PA_Shift - 1 : 12]341 * PAR [PA_Shift - 1 : 12] = PA [PA_Shift - 1 : 12]
342 * HPFAR [PA_Shift - 9 : 4] = FIPA [PA_Shift - 1 : 12]342 * HPFAR [PA_Shift - 9 : 4] = FIPA [PA_Shift - 1 : 12]
343 *
344 * Always assume 52 bit PA since at this point, we don't know how many PA bits
345 * the page table has been set up for. This should be safe since unused address
346 * bits in PAR are res0.
343 */347 */
344#define PAR_TO_HPFAR(par) \348#define PAR_TO_HPFAR(par) \
345 (((par) & GENMASK_ULL(PHYS_MASK_SHIFT - 1, 12)) >> 8)349 (((par) & GENMASK_ULL(52 - 1, 12)) >> 8)
346 350
347#define ECN(x) { ESR_ELx_EC_##x, #x }351#define ECN(x) { ESR_ELx_EC_##x, #x }
348 352
arch/arm64/include/asm/kvm_asm.h+5−2
@@ -76,6 +76,9 @@ enum __kvm_host_smccc_func { · 75 unmodified lines
76 __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,76 __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
77 __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs,77 __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs,
78 __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps,78 __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_init_traps,
79 __KVM_HOST_SMCCC_FUNC___pkvm_init_vm,
80 __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu,
81 __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm,
79};82};
80 83
81#define DECLARE_KVM_VHE_SYM(sym) extern char sym[]84#define DECLARE_KVM_VHE_SYM(sym) extern char sym[]
@@ -106,7 +109,7 @@ enum __kvm_host_smccc_func { · 24 unmodified lines
106#define per_cpu_ptr_nvhe_sym(sym, cpu) \109#define per_cpu_ptr_nvhe_sym(sym, cpu) \
107 ({ \110 ({ \
108 unsigned long base, off; \111 unsigned long base, off; \
109 base = kvm_arm_hyp_percpu_base[cpu]; \112 base = kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu]; \
110 off = (unsigned long)&CHOOSE_NVHE_SYM(sym) - \113 off = (unsigned long)&CHOOSE_NVHE_SYM(sym) - \
111 (unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start); \114 (unsigned long)&CHOOSE_NVHE_SYM(__per_cpu_start); \
112 base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL; \115 base ? (typeof(CHOOSE_NVHE_SYM(sym))*)(base + off) : NULL; \
@@ -211,7 +214,7 @@ DECLARE_KVM_HYP_SYM(__kvm_hyp_vector); · 98 unmodified lines
211#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)214#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
212#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)215#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
213 216
214extern unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];217extern unsigned long kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[];
215DECLARE_KVM_NVHE_SYM(__per_cpu_start);218DECLARE_KVM_NVHE_SYM(__per_cpu_start);
216DECLARE_KVM_NVHE_SYM(__per_cpu_end);219DECLARE_KVM_NVHE_SYM(__per_cpu_end);
217 220
arch/arm64/include/asm/kvm_host.h+74−2
@@ -73,6 +73,63 @@ u32 __attribute_const__ kvm_target_cpu(void); · 72 unmodified lines
73int kvm_reset_vcpu(struct kvm_vcpu *vcpu);73int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
74void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu);74void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu);
75 75
76struct kvm_hyp_memcache {
77 phys_addr_t head;
78 unsigned long nr_pages;
79};
80
81static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc,
82 phys_addr_t *p,
83 phys_addr_t (*to_pa)(void *virt))
84{
85 *p = mc->head;
86 mc->head = to_pa(p);
87 mc->nr_pages++;
88}
89
90static inline void *pop_hyp_memcache(struct kvm_hyp_memcache *mc,
91 void *(*to_va)(phys_addr_t phys))
92{
93 phys_addr_t *p = to_va(mc->head);
94
95 if (!mc->nr_pages)
96 return NULL;
97
98 mc->head = *p;
99 mc->nr_pages--;
100
101 return p;
102}
103
104static inline int __topup_hyp_memcache(struct kvm_hyp_memcache *mc,
105 unsigned long min_pages,
106 void *(*alloc_fn)(void *arg),
107 phys_addr_t (*to_pa)(void *virt),
108 void *arg)
109{
110 while (mc->nr_pages < min_pages) {
111 phys_addr_t *p = alloc_fn(arg);
112
113 if (!p)
114 return -ENOMEM;
115 push_hyp_memcache(mc, p, to_pa);
116 }
117
118 return 0;
119}
120
121static inline void __free_hyp_memcache(struct kvm_hyp_memcache *mc,
122 void (*free_fn)(void *virt, void *arg),
123 void *(*to_va)(phys_addr_t phys),
124 void *arg)
125{
126 while (mc->nr_pages)
127 free_fn(pop_hyp_memcache(mc, to_va), arg);
128}
129
130void free_hyp_memcache(struct kvm_hyp_memcache *mc);
131int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages);
132
76struct kvm_vmid {133struct kvm_vmid {
77 atomic64_t id;134 atomic64_t id;
78};135};
@@ -115,6 +172,13 @@ struct kvm_smccc_features { · 36 unmodified lines
115 unsigned long vendor_hyp_bmap;172 unsigned long vendor_hyp_bmap;
116};173};
117 174
175typedef unsigned int pkvm_handle_t;
176
177struct kvm_protected_vm {
178 pkvm_handle_t handle;
179 struct kvm_hyp_memcache teardown_mc;
180};
181
118struct kvm_arch {182struct kvm_arch {
119 struct kvm_s2_mmu mmu;183 struct kvm_s2_mmu mmu;
120 184
@@ -163,9 +227,19 @@ struct kvm_arch { · 42 unmodified lines
163 227
164 u8 pfr0_csv2;228 u8 pfr0_csv2;
165 u8 pfr0_csv3;229 u8 pfr0_csv3;
230 struct {
231 u8 imp:4;
232 u8 unimp:4;
233 } dfr0_pmuver;
166 234
167 /* Hypercall features firmware registers' descriptor */235 /* Hypercall features firmware registers' descriptor */
168 struct kvm_smccc_features smccc_feat;236 struct kvm_smccc_features smccc_feat;
237
238 /*
239 * For an untrusted host VM, 'pkvm.handle' is used to lookup
240 * the associated pKVM instance in the hypervisor.
241 */
242 struct kvm_protected_vm pkvm;
169};243};
170 244
171struct kvm_vcpu_fault_info {245struct kvm_vcpu_fault_info {
@@ -915,8 +989,6 @@ int kvm_set_ipa_limit(void); · 743 unmodified lines
915#define __KVM_HAVE_ARCH_VM_ALLOC989#define __KVM_HAVE_ARCH_VM_ALLOC
916struct kvm *kvm_arch_alloc_vm(void);990struct kvm *kvm_arch_alloc_vm(void);
917 991
918int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type);
919
920static inline bool kvm_vm_is_protected(struct kvm *kvm)992static inline bool kvm_vm_is_protected(struct kvm *kvm)
921{993{
922 return false;994 return false;
arch/arm64/include/asm/kvm_hyp.h+3−0
@@ -123,4 +123,7 @@ extern u64 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val); · 122 unmodified lines
123extern u64 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val);123extern u64 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val);
124extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val);124extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val);
125 125
126extern unsigned long kvm_nvhe_sym(__icache_flags);
127extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
128
126#endif /* __ARM64_KVM_HYP_H__ */129#endif /* __ARM64_KVM_HYP_H__ */
arch/arm64/include/asm/kvm_mmu.h+1−1
@@ -166,7 +166,7 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, · 165 unmodified lines
166void free_hyp_pgds(void);166void free_hyp_pgds(void);
167 167
168void stage2_unmap_vm(struct kvm *kvm);168void stage2_unmap_vm(struct kvm *kvm);
169int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu);169int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type);
170void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);170void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu);
171int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,171int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
172 phys_addr_t pa, unsigned long size, bool writable);172 phys_addr_t pa, unsigned long size, bool writable);
arch/arm64/include/asm/kvm_pgtable.h+147−28
@@ -42,6 +42,8 @@ typedef u64 kvm_pte_t; · 41 unmodified lines
42#define KVM_PTE_ADDR_MASK GENMASK(47, PAGE_SHIFT)42#define KVM_PTE_ADDR_MASK GENMASK(47, PAGE_SHIFT)
43#define KVM_PTE_ADDR_51_48 GENMASK(15, 12)43#define KVM_PTE_ADDR_51_48 GENMASK(15, 12)
44 44
45#define KVM_PHYS_INVALID (-1ULL)
46
45static inline bool kvm_pte_valid(kvm_pte_t pte)47static inline bool kvm_pte_valid(kvm_pte_t pte)
46{48{
47 return pte & KVM_PTE_VALID;49 return pte & KVM_PTE_VALID;
@@ -57,6 +59,18 @@ static inline u64 kvm_pte_to_phys(kvm_pte_t pte) · 9 unmodified lines
57 return pa;59 return pa;
58}60}
59 61
62static inline kvm_pte_t kvm_phys_to_pte(u64 pa)
63{
64 kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK;
65
66 if (PAGE_SHIFT == 16) {
67 pa &= GENMASK(51, 48);
68 pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48);
69 }
70
71 return pte;
72}
73
60static inline u64 kvm_granule_shift(u32 level)74static inline u64 kvm_granule_shift(u32 level)
61{75{
62 /* Assumes KVM_PGTABLE_MAX_LEVELS is 4 */76 /* Assumes KVM_PGTABLE_MAX_LEVELS is 4 */
@@ -85,6 +99,8 @@ static inline bool kvm_level_supports_block_mapping(u32 level) · 22 unmodified lines
85 * allocation is physically contiguous.99 * allocation is physically contiguous.
86 * @free_pages_exact: Free an exact number of memory pages previously100 * @free_pages_exact: Free an exact number of memory pages previously
87 * allocated by zalloc_pages_exact.101 * allocated by zalloc_pages_exact.
102 * @free_removed_table: Free a removed paging structure by unlinking and
103 * dropping references.
88 * @get_page: Increment the refcount on a page.104 * @get_page: Increment the refcount on a page.
89 * @put_page: Decrement the refcount on a page. When the105 * @put_page: Decrement the refcount on a page. When the
90 * refcount reaches 0 the page is automatically106 * refcount reaches 0 the page is automatically
@@ -103,6 +119,7 @@ struct kvm_pgtable_mm_ops { · 12 unmodified lines
103 void* (*zalloc_page)(void *arg);119 void* (*zalloc_page)(void *arg);
104 void* (*zalloc_pages_exact)(size_t size);120 void* (*zalloc_pages_exact)(size_t size);
105 void (*free_pages_exact)(void *addr, size_t size);121 void (*free_pages_exact)(void *addr, size_t size);
122 void (*free_removed_table)(void *addr, u32 level);
106 void (*get_page)(void *addr);123 void (*get_page)(void *addr);
107 void (*put_page)(void *addr);124 void (*put_page)(void *addr);
108 int (*page_count)(void *addr);125 int (*page_count)(void *addr);
@@ -161,29 +178,6 @@ enum kvm_pgtable_prot { · 52 unmodified lines
161typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,178typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
162 enum kvm_pgtable_prot prot);179 enum kvm_pgtable_prot prot);
163 180
164/**
165 * struct kvm_pgtable - KVM page-table.
166 * @ia_bits: Maximum input address size, in bits.
167 * @start_level: Level at which the page-table walk starts.
168 * @pgd: Pointer to the first top-level entry of the page-table.
169 * @mm_ops: Memory management callbacks.
170 * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
171 * @flags: Stage-2 page-table flags.
172 * @force_pte_cb: Function that returns true if page level mappings must
173 * be used instead of block mappings.
174 */
175struct kvm_pgtable {
176 u32 ia_bits;
177 u32 start_level;
178 kvm_pte_t *pgd;
179 struct kvm_pgtable_mm_ops *mm_ops;
180
181 /* Stage-2 only */
182 struct kvm_s2_mmu *mmu;
183 enum kvm_pgtable_stage2_flags flags;
184 kvm_pgtable_force_pte_cb_t force_pte_cb;
185};
186
187/**181/**
188 * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.182 * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.
189 * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid183 * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid
@@ -192,17 +186,34 @@ struct kvm_pgtable { · 2 unmodified lines
192 * children.186 * children.
193 * @KVM_PGTABLE_WALK_TABLE_POST: Visit table entries after their187 * @KVM_PGTABLE_WALK_TABLE_POST: Visit table entries after their
194 * children.188 * children.
189 * @KVM_PGTABLE_WALK_SHARED: Indicates the page-tables may be shared
190 * with other software walkers.
195 */191 */
196enum kvm_pgtable_walk_flags {192enum kvm_pgtable_walk_flags {
197 KVM_PGTABLE_WALK_LEAF = BIT(0),193 KVM_PGTABLE_WALK_LEAF = BIT(0),
198 KVM_PGTABLE_WALK_TABLE_PRE = BIT(1),194 KVM_PGTABLE_WALK_TABLE_PRE = BIT(1),
199 KVM_PGTABLE_WALK_TABLE_POST = BIT(2),195 KVM_PGTABLE_WALK_TABLE_POST = BIT(2),
196 KVM_PGTABLE_WALK_SHARED = BIT(3),
197};
198
199struct kvm_pgtable_visit_ctx {
200 kvm_pte_t *ptep;
201 kvm_pte_t old;
202 void *arg;
203 struct kvm_pgtable_mm_ops *mm_ops;
204 u64 addr;
205 u64 end;
206 u32 level;
207 enum kvm_pgtable_walk_flags flags;
200};208};
201 209
202typedef int (*kvm_pgtable_visitor_fn_t)(u64 addr, u64 end, u32 level,210typedef int (*kvm_pgtable_visitor_fn_t)(const struct kvm_pgtable_visit_ctx *ctx,
203 kvm_pte_t *ptep,211 enum kvm_pgtable_walk_flags visit);
204 enum kvm_pgtable_walk_flags flag,212
205 void * const arg);213static inline bool kvm_pgtable_walk_shared(const struct kvm_pgtable_visit_ctx *ctx)
214{
215 return ctx->flags & KVM_PGTABLE_WALK_SHARED;
216}
206 217
207/**218/**
208 * struct kvm_pgtable_walker - Hook into a page-table walk.219 * struct kvm_pgtable_walker - Hook into a page-table walk.
@@ -217,6 +228,94 @@ struct kvm_pgtable_walker { · 8 unmodified lines
217 const enum kvm_pgtable_walk_flags flags;228 const enum kvm_pgtable_walk_flags flags;
218};229};
219 230
231/*
232 * RCU cannot be used in a non-kernel context such as the hyp. As such, page
233 * table walkers used in hyp do not call into RCU and instead use other
234 * synchronization mechanisms (such as a spinlock).
235 */
236#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
237
238typedef kvm_pte_t *kvm_pteref_t;
239
240static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
241 kvm_pteref_t pteref)
242{
243 return pteref;
244}
245
246static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
247{
248 /*
249 * Due to the lack of RCU (or a similar protection scheme), only
250 * non-shared table walkers are allowed in the hypervisor.
251 */
252 if (walker->flags & KVM_PGTABLE_WALK_SHARED)
253 return -EPERM;
254
255 return 0;
256}
257
258static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker) {}
259
260static inline bool kvm_pgtable_walk_lock_held(void)
261{
262 return true;
263}
264
265#else
266
267typedef kvm_pte_t __rcu *kvm_pteref_t;
268
269static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walker,
270 kvm_pteref_t pteref)
271{
272 return rcu_dereference_check(pteref, !(walker->flags & KVM_PGTABLE_WALK_SHARED));
273}
274
275static inline int kvm_pgtable_walk_begin(struct kvm_pgtable_walker *walker)
276{
277 if (walker->flags & KVM_PGTABLE_WALK_SHARED)
278 rcu_read_lock();
279
280 return 0;
281}
282
283static inline void kvm_pgtable_walk_end(struct kvm_pgtable_walker *walker)
284{
285 if (walker->flags & KVM_PGTABLE_WALK_SHARED)
286 rcu_read_unlock();
287}
288
289static inline bool kvm_pgtable_walk_lock_held(void)
290{
291 return rcu_read_lock_held();
292}
293
294#endif
295
296/**
297 * struct kvm_pgtable - KVM page-table.
298 * @ia_bits: Maximum input address size, in bits.
299 * @start_level: Level at which the page-table walk starts.
300 * @pgd: Pointer to the first top-level entry of the page-table.
301 * @mm_ops: Memory management callbacks.
302 * @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
303 * @flags: Stage-2 page-table flags.
304 * @force_pte_cb: Function that returns true if page level mappings must
305 * be used instead of block mappings.
306 */
307struct kvm_pgtable {
308 u32 ia_bits;
309 u32 start_level;
310 kvm_pteref_t pgd;
311 struct kvm_pgtable_mm_ops *mm_ops;
312
313 /* Stage-2 only */
314 struct kvm_s2_mmu *mmu;
315 enum kvm_pgtable_stage2_flags flags;
316 kvm_pgtable_force_pte_cb_t force_pte_cb;
317};
318
220/**319/**
221 * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.320 * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
222 * @pgt: Uninitialised page-table structure to initialise.321 * @pgt: Uninitialised page-table structure to initialise.
@@ -296,6 +395,14 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); · 73 unmodified lines
296 */395 */
297u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift);396u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift);
298 397
398/**
399 * kvm_pgtable_stage2_pgd_size() - Helper to compute size of a stage-2 PGD
400 * @vtcr: Content of the VTCR register.
401 *
402 * Return: the size (in bytes) of the stage-2 PGD
403 */
404size_t kvm_pgtable_stage2_pgd_size(u64 vtcr);
405
299/**406/**
300 * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.407 * __kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
301 * @pgt: Uninitialised page-table structure to initialise.408 * @pgt: Uninitialised page-table structure to initialise.
@@ -324,6 +431,17 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 22 unmodified lines
324 */431 */
325void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);432void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
326 433
434/**
435 * kvm_pgtable_stage2_free_removed() - Free a removed stage-2 paging structure.
436 * @mm_ops: Memory management callbacks.
437 * @pgtable: Unlinked stage-2 paging structure to be freed.
438 * @level: Level of the stage-2 paging structure to be freed.
439 *
440 * The page-table is assumed to be unreachable by any hardware walkers prior to
441 * freeing and therefore no TLB invalidation is performed.
442 */
443void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level);
444
327/**445/**
328 * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table.446 * kvm_pgtable_stage2_map() - Install a mapping in a guest stage-2 page-table.
329 * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init*().447 * @pgt: Page-table structure initialised by kvm_pgtable_stage2_init*().
@@ -333,6 +451,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); · 3 unmodified lines
333 * @prot: Permissions and attributes for the mapping.451 * @prot: Permissions and attributes for the mapping.
334 * @mc: Cache of pre-allocated and zeroed memory from which to allocate452 * @mc: Cache of pre-allocated and zeroed memory from which to allocate
335 * page-table pages.453 * page-table pages.
454 * @flags: Flags to control the page-table walk (ex. a shared walk)
336 *455 *
337 * The offset of @addr within a page is ignored, @size is rounded-up to456 * The offset of @addr within a page is ignored, @size is rounded-up to
338 * the next page boundary and @phys is rounded-down to the previous page457 * the next page boundary and @phys is rounded-down to the previous page
@@ -354,7 +473,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt); · 15 unmodified lines
354 */473 */
355int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,474int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
356 u64 phys, enum kvm_pgtable_prot prot,475 u64 phys, enum kvm_pgtable_prot prot,
357 void *mc);476 void *mc, enum kvm_pgtable_walk_flags flags);
358 477
359/**478/**
360 * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to479 * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to
arch/arm64/include/asm/kvm_pkvm.h+38−0
@@ -9,11 +9,49 @@ · 8 unmodified lines
9#include <linux/memblock.h>9#include <linux/memblock.h>
10#include <asm/kvm_pgtable.h>10#include <asm/kvm_pgtable.h>
11 11
12/* Maximum number of VMs that can co-exist under pKVM. */
13#define KVM_MAX_PVMS 255
14
12#define HYP_MEMBLOCK_REGIONS 12815#define HYP_MEMBLOCK_REGIONS 128
13 16
17int pkvm_init_host_vm(struct kvm *kvm);
18int pkvm_create_hyp_vm(struct kvm *kvm);
19void pkvm_destroy_hyp_vm(struct kvm *kvm);
20
14extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];21extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
15extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);22extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
16 23
24static inline unsigned long
25hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size)
26{
27 unsigned long nr_pages = reg->size >> PAGE_SHIFT;
28 unsigned long start, end;
29
30 start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size;
31 end = start + nr_pages * vmemmap_entry_size;
32 start = ALIGN_DOWN(start, PAGE_SIZE);
33 end = ALIGN(end, PAGE_SIZE);
34
35 return end - start;
36}
37
38static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size)
39{
40 unsigned long res = 0, i;
41
42 for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
43 res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i],
44 vmemmap_entry_size);
45 }
46
47 return res >> PAGE_SHIFT;
48}
49
50static inline unsigned long hyp_vm_table_pages(void)
51{
52 return PAGE_ALIGN(KVM_MAX_PVMS * sizeof(void *)) >> PAGE_SHIFT;
53}
54
17static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)55static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
18{56{
19 unsigned long total = 0, i;57 unsigned long total = 0, i;
arch/arm64/include/asm/mte.h+64−1
@@ -25,7 +25,7 @@ unsigned long mte_copy_tags_to_user(void __user *to, void *from, · 24 unmodified lines
25 unsigned long n);25 unsigned long n);
26int mte_save_tags(struct page *page);26int mte_save_tags(struct page *page);
27void mte_save_page_tags(const void *page_addr, void *tag_storage);27void mte_save_page_tags(const void *page_addr, void *tag_storage);
28bool mte_restore_tags(swp_entry_t entry, struct page *page);28void mte_restore_tags(swp_entry_t entry, struct page *page);
29void mte_restore_page_tags(void *page_addr, const void *tag_storage);29void mte_restore_page_tags(void *page_addr, const void *tag_storage);
30void mte_invalidate_tags(int type, pgoff_t offset);30void mte_invalidate_tags(int type, pgoff_t offset);
31void mte_invalidate_tags_area(int type);31void mte_invalidate_tags_area(int type);
@@ -36,6 +36,58 @@ void mte_free_tag_storage(char *storage); · 4 unmodified lines
36 36
37/* track which pages have valid allocation tags */37/* track which pages have valid allocation tags */
38#define PG_mte_tagged PG_arch_238#define PG_mte_tagged PG_arch_2
39/* simple lock to avoid multiple threads tagging the same page */
40#define PG_mte_lock PG_arch_3
41
42static inline void set_page_mte_tagged(struct page *page)
43{
44 /*
45 * Ensure that the tags written prior to this function are visible
46 * before the page flags update.
47 */
48 smp_wmb();
49 set_bit(PG_mte_tagged, &page->flags);
50}
51
52static inline bool page_mte_tagged(struct page *page)
53{
54 bool ret = test_bit(PG_mte_tagged, &page->flags);
55
56 /*
57 * If the page is tagged, ensure ordering with a likely subsequent
58 * read of the tags.
59 */
60 if (ret)
61 smp_rmb();
62 return ret;
63}
64
65/*
66 * Lock the page for tagging and return 'true' if the page can be tagged,
67 * 'false' if already tagged. PG_mte_tagged is never cleared and therefore the
68 * locking only happens once for page initialisation.
69 *
70 * The page MTE lock state:
71 *
72 * Locked: PG_mte_lock && !PG_mte_tagged
73 * Unlocked: !PG_mte_lock || PG_mte_tagged
74 *
75 * Acquire semantics only if the page is tagged (returning 'false').
76 */
77static inline bool try_page_mte_tagging(struct page *page)
78{
79 if (!test_and_set_bit(PG_mte_lock, &page->flags))
80 return true;
81
82 /*
83 * The tags are either being initialised or may have been initialised
84 * already. Check if the PG_mte_tagged flag has been set or wait
85 * otherwise.
86 */
87 smp_cond_load_acquire(&page->flags, VAL & (1UL << PG_mte_tagged));
88
89 return false;
90}
39 91
40void mte_zero_clear_page_tags(void *addr);92void mte_zero_clear_page_tags(void *addr);
41void mte_sync_tags(pte_t old_pte, pte_t pte);93void mte_sync_tags(pte_t old_pte, pte_t pte);
@@ -56,6 +108,17 @@ size_t mte_probe_user_range(const char __user *uaddr, size_t size); · 14 unmodified lines
56/* unused if !CONFIG_ARM64_MTE, silence the compiler */108/* unused if !CONFIG_ARM64_MTE, silence the compiler */
57#define PG_mte_tagged 0109#define PG_mte_tagged 0
58 110
111static inline void set_page_mte_tagged(struct page *page)
112{
113}
114static inline bool page_mte_tagged(struct page *page)
115{
116 return false;
117}
118static inline bool try_page_mte_tagging(struct page *page)
119{
120 return false;
121}
59static inline void mte_zero_clear_page_tags(void *addr)122static inline void mte_zero_clear_page_tags(void *addr)
60{123{
61}124}
arch/arm64/include/asm/pgtable.h+2−2
@@ -1049,8 +1049,8 @@ static inline void arch_swap_invalidate_area(int type) · 1048 unmodified lines
1049#define __HAVE_ARCH_SWAP_RESTORE1049#define __HAVE_ARCH_SWAP_RESTORE
1050static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)1050static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
1051{1051{
1052 if (system_supports_mte() && mte_restore_tags(entry, &folio->page))1052 if (system_supports_mte())
1053 set_bit(PG_mte_tagged, &folio->flags);1053 mte_restore_tags(entry, &folio->page);
1054}1054}
1055 1055
1056#endif /* CONFIG_ARM64_MTE */1056#endif /* CONFIG_ARM64_MTE */
arch/arm64/include/asm/sysreg.h+0−138
@@ -165,31 +165,6 @@ · 164 unmodified lines
165#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)165#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
166#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)166#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
167 167
168#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
169#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
170#define SYS_ID_PFR2_EL1 sys_reg(3, 0, 0, 3, 4)
171#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
172#define SYS_ID_DFR1_EL1 sys_reg(3, 0, 0, 3, 5)
173#define SYS_ID_AFR0_EL1 sys_reg(3, 0, 0, 1, 3)
174#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
175#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
176#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
177#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
178#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
179#define SYS_ID_MMFR5_EL1 sys_reg(3, 0, 0, 3, 6)
180
181#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
182#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
183#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
184#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
185#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
186#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
187#define SYS_ID_ISAR6_EL1 sys_reg(3, 0, 0, 2, 7)
188
189#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
190#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
191#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
192
193#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)168#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
194#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)169#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
195#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)170#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
@@ -692,112 +667,6 @@ · 496 unmodified lines
692#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48667#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48
693#endif668#endif
694 669
695#define ID_DFR0_PERFMON_SHIFT 24
696
697#define ID_DFR0_PERFMON_8_0 0x3
698#define ID_DFR0_PERFMON_8_1 0x4
699#define ID_DFR0_PERFMON_8_4 0x5
700#define ID_DFR0_PERFMON_8_5 0x6
701
702#define ID_ISAR4_SWP_FRAC_SHIFT 28
703#define ID_ISAR4_PSR_M_SHIFT 24
704#define ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT 20
705#define ID_ISAR4_BARRIER_SHIFT 16
706#define ID_ISAR4_SMC_SHIFT 12
707#define ID_ISAR4_WRITEBACK_SHIFT 8
708#define ID_ISAR4_WITHSHIFTS_SHIFT 4
709#define ID_ISAR4_UNPRIV_SHIFT 0
710
711#define ID_DFR1_MTPMU_SHIFT 0
712
713#define ID_ISAR0_DIVIDE_SHIFT 24
714#define ID_ISAR0_DEBUG_SHIFT 20
715#define ID_ISAR0_COPROC_SHIFT 16
716#define ID_ISAR0_CMPBRANCH_SHIFT 12
717#define ID_ISAR0_BITFIELD_SHIFT 8
718#define ID_ISAR0_BITCOUNT_SHIFT 4
719#define ID_ISAR0_SWAP_SHIFT 0
720
721#define ID_ISAR5_RDM_SHIFT 24
722#define ID_ISAR5_CRC32_SHIFT 16
723#define ID_ISAR5_SHA2_SHIFT 12
724#define ID_ISAR5_SHA1_SHIFT 8
725#define ID_ISAR5_AES_SHIFT 4
726#define ID_ISAR5_SEVL_SHIFT 0
727
728#define ID_ISAR6_I8MM_SHIFT 24
729#define ID_ISAR6_BF16_SHIFT 20
730#define ID_ISAR6_SPECRES_SHIFT 16
731#define ID_ISAR6_SB_SHIFT 12
732#define ID_ISAR6_FHM_SHIFT 8
733#define ID_ISAR6_DP_SHIFT 4
734#define ID_ISAR6_JSCVT_SHIFT 0
735
736#define ID_MMFR0_INNERSHR_SHIFT 28
737#define ID_MMFR0_FCSE_SHIFT 24
738#define ID_MMFR0_AUXREG_SHIFT 20
739#define ID_MMFR0_TCM_SHIFT 16
740#define ID_MMFR0_SHARELVL_SHIFT 12
741#define ID_MMFR0_OUTERSHR_SHIFT 8
742#define ID_MMFR0_PMSA_SHIFT 4
743#define ID_MMFR0_VMSA_SHIFT 0
744
745#define ID_MMFR4_EVT_SHIFT 28
746#define ID_MMFR4_CCIDX_SHIFT 24
747#define ID_MMFR4_LSM_SHIFT 20
748#define ID_MMFR4_HPDS_SHIFT 16
749#define ID_MMFR4_CNP_SHIFT 12
750#define ID_MMFR4_XNX_SHIFT 8
751#define ID_MMFR4_AC2_SHIFT 4
752#define ID_MMFR4_SPECSEI_SHIFT 0
753
754#define ID_MMFR5_ETS_SHIFT 0
755
756#define ID_PFR0_DIT_SHIFT 24
757#define ID_PFR0_CSV2_SHIFT 16
758#define ID_PFR0_STATE3_SHIFT 12
759#define ID_PFR0_STATE2_SHIFT 8
760#define ID_PFR0_STATE1_SHIFT 4
761#define ID_PFR0_STATE0_SHIFT 0
762
763#define ID_DFR0_PERFMON_SHIFT 24
764#define ID_DFR0_MPROFDBG_SHIFT 20
765#define ID_DFR0_MMAPTRC_SHIFT 16
766#define ID_DFR0_COPTRC_SHIFT 12
767#define ID_DFR0_MMAPDBG_SHIFT 8
768#define ID_DFR0_COPSDBG_SHIFT 4
769#define ID_DFR0_COPDBG_SHIFT 0
770
771#define ID_PFR2_SSBS_SHIFT 4
772#define ID_PFR2_CSV3_SHIFT 0
773
774#define MVFR0_FPROUND_SHIFT 28
775#define MVFR0_FPSHVEC_SHIFT 24
776#define MVFR0_FPSQRT_SHIFT 20
777#define MVFR0_FPDIVIDE_SHIFT 16
778#define MVFR0_FPTRAP_SHIFT 12
779#define MVFR0_FPDP_SHIFT 8
780#define MVFR0_FPSP_SHIFT 4
781#define MVFR0_SIMD_SHIFT 0
782
783#define MVFR1_SIMDFMAC_SHIFT 28
784#define MVFR1_FPHP_SHIFT 24
785#define MVFR1_SIMDHP_SHIFT 20
786#define MVFR1_SIMDSP_SHIFT 16
787#define MVFR1_SIMDINT_SHIFT 12
788#define MVFR1_SIMDLS_SHIFT 8
789#define MVFR1_FPDNAN_SHIFT 4
790#define MVFR1_FPFTZ_SHIFT 0
791
792#define ID_PFR1_GIC_SHIFT 28
793#define ID_PFR1_VIRT_FRAC_SHIFT 24
794#define ID_PFR1_SEC_FRAC_SHIFT 20
795#define ID_PFR1_GENTIMER_SHIFT 16
796#define ID_PFR1_VIRTUALIZATION_SHIFT 12
797#define ID_PFR1_MPROGMOD_SHIFT 8
798#define ID_PFR1_SECURITY_SHIFT 4
799#define ID_PFR1_PROGMOD_SHIFT 0
800
801#if defined(CONFIG_ARM64_4K_PAGES)670#if defined(CONFIG_ARM64_4K_PAGES)
802#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT671#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT
803#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN672#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
@@ -815,9 +684,6 @@ · 11 unmodified lines
815#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT684#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
816#endif685#endif
817 686
818#define MVFR2_FPMISC_SHIFT 4
819#define MVFR2_SIMDMISC_SHIFT 0
820
821#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */687#define CPACR_EL1_FPEN_EL1EN (BIT(20)) /* enable EL1 access */
822#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */688#define CPACR_EL1_FPEN_EL0EN (BIT(21)) /* enable EL0 access, if EL1EN set */
823 689
@@ -851,10 +717,6 @@ · 27 unmodified lines
851#define SYS_RGSR_EL1_SEED_SHIFT 8717#define SYS_RGSR_EL1_SEED_SHIFT 8
852#define SYS_RGSR_EL1_SEED_MASK 0xffffUL718#define SYS_RGSR_EL1_SEED_MASK 0xffffUL
853 719
854/* GMID_EL1 field definitions */
855#define GMID_EL1_BS_SHIFT 0
856#define GMID_EL1_BS_SIZE 4
857
858/* TFSR{,E0}_EL1 bit definitions */720/* TFSR{,E0}_EL1 bit definitions */
859#define SYS_TFSR_EL1_TF0_SHIFT 0721#define SYS_TFSR_EL1_TF0_SHIFT 0
860#define SYS_TFSR_EL1_TF1_SHIFT 1722#define SYS_TFSR_EL1_TF1_SHIFT 1
arch/arm64/include/uapi/asm/kvm.h+1−0
@@ -43,6 +43,7 @@ · 42 unmodified lines
43#define __KVM_HAVE_VCPU_EVENTS43#define __KVM_HAVE_VCPU_EVENTS
44 44
45#define KVM_COALESCED_MMIO_PAGE_OFFSET 145#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
46#define KVM_DIRTY_LOG_PAGE_OFFSET 64
46 47
47#define KVM_REG_SIZE(id) \48#define KVM_REG_SIZE(id) \
48 (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))49 (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
arch/arm64/kernel/cpufeature.c+107−105
@@ -402,14 +402,14 @@ struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { · 401 unmodified lines
402};402};
403 403
404static const struct arm64_ftr_bits ftr_id_mmfr0[] = {404static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
405 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_INNERSHR_SHIFT, 4, 0xf),405 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
406 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_FCSE_SHIFT, 4, 0),406 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
407 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_AUXREG_SHIFT, 4, 0),407 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
408 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_TCM_SHIFT, 4, 0),408 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
409 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_SHARELVL_SHIFT, 4, 0),409 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
410 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_OUTERSHR_SHIFT, 4, 0xf),410 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
411 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_PMSA_SHIFT, 4, 0),411 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
412 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_VMSA_SHIFT, 4, 0),412 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
413 ARM64_FTR_END,413 ARM64_FTR_END,
414};414};
415 415
@@ -429,32 +429,32 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { · 13 unmodified lines
429};429};
430 430
431static const struct arm64_ftr_bits ftr_mvfr0[] = {431static const struct arm64_ftr_bits ftr_mvfr0[] = {
432 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPROUND_SHIFT, 4, 0),432 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
433 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSHVEC_SHIFT, 4, 0),433 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
434 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSQRT_SHIFT, 4, 0),434 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
435 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDIVIDE_SHIFT, 4, 0),435 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
436 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPTRAP_SHIFT, 4, 0),436 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
437 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDP_SHIFT, 4, 0),437 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
438 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSP_SHIFT, 4, 0),438 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
439 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_SIMD_SHIFT, 4, 0),439 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
440 ARM64_FTR_END,440 ARM64_FTR_END,
441};441};
442 442
443static const struct arm64_ftr_bits ftr_mvfr1[] = {443static const struct arm64_ftr_bits ftr_mvfr1[] = {
444 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDFMAC_SHIFT, 4, 0),444 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
445 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPHP_SHIFT, 4, 0),445 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
446 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDHP_SHIFT, 4, 0),446 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
447 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDSP_SHIFT, 4, 0),447 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
448 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDINT_SHIFT, 4, 0),448 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
449 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDLS_SHIFT, 4, 0),449 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
450 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPDNAN_SHIFT, 4, 0),450 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
451 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPFTZ_SHIFT, 4, 0),451 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
452 ARM64_FTR_END,452 ARM64_FTR_END,
453};453};
454 454
455static const struct arm64_ftr_bits ftr_mvfr2[] = {455static const struct arm64_ftr_bits ftr_mvfr2[] = {
456 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0),456 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
457 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0),457 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
458 ARM64_FTR_END,458 ARM64_FTR_END,
459};459};
460 460
@@ -470,34 +470,34 @@ static const struct arm64_ftr_bits ftr_gmid[] = { · 9 unmodified lines
470};470};
471 471
472static const struct arm64_ftr_bits ftr_id_isar0[] = {472static const struct arm64_ftr_bits ftr_id_isar0[] = {
473 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),473 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
474 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),474 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
475 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_COPROC_SHIFT, 4, 0),475 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
476 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_CMPBRANCH_SHIFT, 4, 0),476 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
477 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITFIELD_SHIFT, 4, 0),477 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
478 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITCOUNT_SHIFT, 4, 0),478 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
479 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_SWAP_SHIFT, 4, 0),479 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
480 ARM64_FTR_END,480 ARM64_FTR_END,
481};481};
482 482
483static const struct arm64_ftr_bits ftr_id_isar5[] = {483static const struct arm64_ftr_bits ftr_id_isar5[] = {
484 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),484 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
485 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),485 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
486 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),486 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
487 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),487 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
488 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),488 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
489 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),489 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
490 ARM64_FTR_END,490 ARM64_FTR_END,
491};491};
492 492
493static const struct arm64_ftr_bits ftr_id_mmfr4[] = {493static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
494 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EVT_SHIFT, 4, 0),494 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
495 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CCIDX_SHIFT, 4, 0),495 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
496 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_LSM_SHIFT, 4, 0),496 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
497 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_HPDS_SHIFT, 4, 0),497 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
498 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CNP_SHIFT, 4, 0),498 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
499 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_XNX_SHIFT, 4, 0),499 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
500 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_AC2_SHIFT, 4, 0),500 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
501 501
502 /*502 /*
503 * SpecSEI = 1 indicates that the PE might generate an SError on an503 * SpecSEI = 1 indicates that the PE might generate an SError on an
@@ -505,80 +505,80 @@ static const struct arm64_ftr_bits ftr_id_mmfr4[] = { · 1 unmodified line
505 * SError might be generated than it will not be. Hence it has been505 * SError might be generated than it will not be. Hence it has been
506 * classified as FTR_HIGHER_SAFE.506 * classified as FTR_HIGHER_SAFE.
507 */507 */
508 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_SPECSEI_SHIFT, 4, 0),508 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
509 ARM64_FTR_END,509 ARM64_FTR_END,
510};510};
511 511
512static const struct arm64_ftr_bits ftr_id_isar4[] = {512static const struct arm64_ftr_bits ftr_id_isar4[] = {
513 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SWP_FRAC_SHIFT, 4, 0),513 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
514 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_PSR_M_SHIFT, 4, 0),514 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
515 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT, 4, 0),515 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
516 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_BARRIER_SHIFT, 4, 0),516 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
517 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SMC_SHIFT, 4, 0),517 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
518 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WRITEBACK_SHIFT, 4, 0),518 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
519 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WITHSHIFTS_SHIFT, 4, 0),519 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
520 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_UNPRIV_SHIFT, 4, 0),520 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
521 ARM64_FTR_END,521 ARM64_FTR_END,
522};522};
523 523
524static const struct arm64_ftr_bits ftr_id_mmfr5[] = {524static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
525 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_ETS_SHIFT, 4, 0),525 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
526 ARM64_FTR_END,526 ARM64_FTR_END,
527};527};
528 528
529static const struct arm64_ftr_bits ftr_id_isar6[] = {529static const struct arm64_ftr_bits ftr_id_isar6[] = {
530 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_I8MM_SHIFT, 4, 0),530 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
531 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_BF16_SHIFT, 4, 0),531 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
532 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SPECRES_SHIFT, 4, 0),532 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
533 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SB_SHIFT, 4, 0),533 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
534 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_FHM_SHIFT, 4, 0),534 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
535 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_DP_SHIFT, 4, 0),535 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
536 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_JSCVT_SHIFT, 4, 0),536 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
537 ARM64_FTR_END,537 ARM64_FTR_END,
538};538};
539 539
540static const struct arm64_ftr_bits ftr_id_pfr0[] = {540static const struct arm64_ftr_bits ftr_id_pfr0[] = {
541 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_DIT_SHIFT, 4, 0),541 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
542 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_CSV2_SHIFT, 4, 0),542 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
543 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE3_SHIFT, 4, 0),543 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
544 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE2_SHIFT, 4, 0),544 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
545 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE1_SHIFT, 4, 0),545 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
546 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE0_SHIFT, 4, 0),546 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
547 ARM64_FTR_END,547 ARM64_FTR_END,
548};548};
549 549
550static const struct arm64_ftr_bits ftr_id_pfr1[] = {550static const struct arm64_ftr_bits ftr_id_pfr1[] = {
551 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GIC_SHIFT, 4, 0),551 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
552 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRT_FRAC_SHIFT, 4, 0),552 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
553 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SEC_FRAC_SHIFT, 4, 0),553 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
554 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GENTIMER_SHIFT, 4, 0),554 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
555 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRTUALIZATION_SHIFT, 4, 0),555 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
556 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_MPROGMOD_SHIFT, 4, 0),556 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
557 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SECURITY_SHIFT, 4, 0),557 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
558 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_PROGMOD_SHIFT, 4, 0),558 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
559 ARM64_FTR_END,559 ARM64_FTR_END,
560};560};
561 561
562static const struct arm64_ftr_bits ftr_id_pfr2[] = {562static const struct arm64_ftr_bits ftr_id_pfr2[] = {
563 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),563 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
564 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),564 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
565 ARM64_FTR_END,565 ARM64_FTR_END,
566};566};
567 567
568static const struct arm64_ftr_bits ftr_id_dfr0[] = {568static const struct arm64_ftr_bits ftr_id_dfr0[] = {
569 /* [31:28] TraceFilt */569 /* [31:28] TraceFilt */
570 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_PERFMON_SHIFT, 4, 0),570 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
571 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MPROFDBG_SHIFT, 4, 0),571 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
572 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPTRC_SHIFT, 4, 0),572 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
573 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPTRC_SHIFT, 4, 0),573 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
574 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPDBG_SHIFT, 4, 0),574 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
575 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPSDBG_SHIFT, 4, 0),575 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
576 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPDBG_SHIFT, 4, 0),576 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
577 ARM64_FTR_END,577 ARM64_FTR_END,
578};578};
579 579
580static const struct arm64_ftr_bits ftr_id_dfr1[] = {580static const struct arm64_ftr_bits ftr_id_dfr1[] = {
581 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_MTPMU_SHIFT, 4, 0),581 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
582 ARM64_FTR_END,582 ARM64_FTR_END,
583};583};
584 584
@@ -1119,12 +1119,12 @@ static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, · 534 unmodified lines
1119 * EL1-dependent register fields to avoid spurious sanity check fails.1119 * EL1-dependent register fields to avoid spurious sanity check fails.
1120 */1120 */
1121 if (!id_aa64pfr0_32bit_el1(pfr0)) {1121 if (!id_aa64pfr0_32bit_el1(pfr0)) {
1122 relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_SMC_SHIFT);1122 relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
1123 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);1123 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
1124 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);1124 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
1125 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);1125 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
1126 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SECURITY_SHIFT);1126 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
1127 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_PROGMOD_SHIFT);1127 relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
1128 }1128 }
1129 1129
1130 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,1130 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
@@ -2074,8 +2074,10 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) · 943 unmodified lines
2074 * Clear the tags in the zero page. This needs to be done via the2074 * Clear the tags in the zero page. This needs to be done via the
2075 * linear map which has the Tagged attribute.2075 * linear map which has the Tagged attribute.
2076 */2076 */
2077 if (!test_and_set_bit(PG_mte_tagged, &ZERO_PAGE(0)->flags))2077 if (try_page_mte_tagging(ZERO_PAGE(0))) {
2078 mte_clear_page_tags(lm_alias(empty_zero_page));2078 mte_clear_page_tags(lm_alias(empty_zero_page));
2079 set_page_mte_tagged(ZERO_PAGE(0));
2080 }
2079 2081
2080 kasan_init_hw_tags_cpu();2082 kasan_init_hw_tags_cpu();
2081}2083}
@@ -2829,24 +2831,24 @@ static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope) · 747 unmodified lines
2829 else2831 else
2830 mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);2832 mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
2831 2833
2832 return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDSP_SHIFT) &&2834 return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
2833 cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDINT_SHIFT) &&2835 cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
2834 cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDLS_SHIFT);2836 cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
2835}2837}
2836#endif2838#endif
2837 2839
2838static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {2840static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
2839#ifdef CONFIG_COMPAT2841#ifdef CONFIG_COMPAT
2840 HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),2842 HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
2841 HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),2843 HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_EL1_SIMDFMAC_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
2842 /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */2844 /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
2843 HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),2845 HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
2844 HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),2846 HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_EL1_FPDP_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
2845 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),2847 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
2846 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),2848 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_AES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
2847 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),2849 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA1_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
2848 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),2850 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_SHA2_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
2849 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),2851 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_EL1_CRC32_SHIFT, 4, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
2850#endif2852#endif
2851 {},2853 {},
2852};2854};
arch/arm64/kernel/elfcore.c+1−1
@@ -47,7 +47,7 @@ static int mte_dump_tag_range(struct coredump_params *cprm, · 46 unmodified lines
47 * Pages mapped in user space as !pte_access_permitted() (e.g.47 * Pages mapped in user space as !pte_access_permitted() (e.g.
48 * PROT_EXEC only) may not have the PG_mte_tagged flag set.48 * PROT_EXEC only) may not have the PG_mte_tagged flag set.
49 */49 */
50 if (!test_bit(PG_mte_tagged, &page->flags)) {50 if (!page_mte_tagged(page)) {
51 put_page(page);51 put_page(page);
52 dump_skip(cprm, MTE_PAGE_TAG_STORAGE);52 dump_skip(cprm, MTE_PAGE_TAG_STORAGE);
53 continue;53 continue;
arch/arm64/kernel/hibernate.c+1−1
@@ -271,7 +271,7 @@ static int swsusp_mte_save_tags(void) · 270 unmodified lines
271 if (!page)271 if (!page)
272 continue;272 continue;
273 273
274 if (!test_bit(PG_mte_tagged, &page->flags))274 if (!page_mte_tagged(page))
275 continue;275 continue;
276 276
277 ret = save_tags(page, pfn);277 ret = save_tags(page, pfn);
arch/arm64/kernel/image-vars.h+0−15
@@ -71,12 +71,6 @@ KVM_NVHE_ALIAS(nvhe_hyp_panic_handler); · 70 unmodified lines
71/* Vectors installed by hyp-init on reset HVC. */71/* Vectors installed by hyp-init on reset HVC. */
72KVM_NVHE_ALIAS(__hyp_stub_vectors);72KVM_NVHE_ALIAS(__hyp_stub_vectors);
73 73
74/* Kernel symbol used by icache_is_vpipt(). */
75KVM_NVHE_ALIAS(__icache_flags);
76
77/* VMID bits set by the KVM VMID allocator */
78KVM_NVHE_ALIAS(kvm_arm_vmid_bits);
79
80/* Static keys which are set if a vGIC trap should be handled in hyp. */74/* Static keys which are set if a vGIC trap should be handled in hyp. */
81KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);75KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
82KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);76KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
@@ -92,9 +86,6 @@ KVM_NVHE_ALIAS(gic_nonsecure_priorities); · 9 unmodified lines
92KVM_NVHE_ALIAS(__start___kvm_ex_table);86KVM_NVHE_ALIAS(__start___kvm_ex_table);
93KVM_NVHE_ALIAS(__stop___kvm_ex_table);87KVM_NVHE_ALIAS(__stop___kvm_ex_table);
94 88
95/* Array containing bases of nVHE per-CPU memory regions. */
96KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base);
97
98/* PMU available static key */89/* PMU available static key */
99#ifdef CONFIG_HW_PERF_EVENTS90#ifdef CONFIG_HW_PERF_EVENTS
100KVM_NVHE_ALIAS(kvm_arm_pmu_available);91KVM_NVHE_ALIAS(kvm_arm_pmu_available);
@@ -111,12 +102,6 @@ KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy); · 10 unmodified lines
111KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);102KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);
112#endif103#endif
113 104
114/* Kernel memory sections */
115KVM_NVHE_ALIAS(__start_rodata);
116KVM_NVHE_ALIAS(__end_rodata);
117KVM_NVHE_ALIAS(__bss_start);
118KVM_NVHE_ALIAS(__bss_stop);
119
120/* Hyp memory sections */105/* Hyp memory sections */
121KVM_NVHE_ALIAS(__hyp_idmap_text_start);106KVM_NVHE_ALIAS(__hyp_idmap_text_start);
122KVM_NVHE_ALIAS(__hyp_idmap_text_end);107KVM_NVHE_ALIAS(__hyp_idmap_text_end);
arch/arm64/kernel/mte.c+10−11
@@ -41,19 +41,17 @@ static void mte_sync_page_tags(struct page *page, pte_t old_pte, · 40 unmodified lines
41 if (check_swap && is_swap_pte(old_pte)) {41 if (check_swap && is_swap_pte(old_pte)) {
42 swp_entry_t entry = pte_to_swp_entry(old_pte);42 swp_entry_t entry = pte_to_swp_entry(old_pte);
43 43
44 if (!non_swap_entry(entry) && mte_restore_tags(entry, page))44 if (!non_swap_entry(entry))
45 return;45 mte_restore_tags(entry, page);
46 }46 }
47 47
48 if (!pte_is_tagged)48 if (!pte_is_tagged)
49 return;49 return;
50 50
51 /*51 if (try_page_mte_tagging(page)) {
52 * Test PG_mte_tagged again in case it was racing with another
53 * set_pte_at().
54 */
55 if (!test_and_set_bit(PG_mte_tagged, &page->flags))
56 mte_clear_page_tags(page_address(page));52 mte_clear_page_tags(page_address(page));
53 set_page_mte_tagged(page);
54 }
57}55}
58 56
59void mte_sync_tags(pte_t old_pte, pte_t pte)57void mte_sync_tags(pte_t old_pte, pte_t pte)
@@ -69,9 +67,11 @@ void mte_sync_tags(pte_t old_pte, pte_t pte) · 9 unmodified lines
69 67
70 /* if PG_mte_tagged is set, tags have already been initialised */68 /* if PG_mte_tagged is set, tags have already been initialised */
71 for (i = 0; i < nr_pages; i++, page++) {69 for (i = 0; i < nr_pages; i++, page++) {
72 if (!test_bit(PG_mte_tagged, &page->flags))70 if (!page_mte_tagged(page)) {
73 mte_sync_page_tags(page, old_pte, check_swap,71 mte_sync_page_tags(page, old_pte, check_swap,
74 pte_is_tagged);72 pte_is_tagged);
73 set_page_mte_tagged(page);
74 }
75 }75 }
76 76
77 /* ensure the tags are visible before the PTE is set */77 /* ensure the tags are visible before the PTE is set */
@@ -96,8 +96,7 @@ int memcmp_pages(struct page *page1, struct page *page2) · 18 unmodified lines
96 * pages is tagged, set_pte_at() may zero or change the tags of the96 * pages is tagged, set_pte_at() may zero or change the tags of the
97 * other page via mte_sync_tags().97 * other page via mte_sync_tags().
98 */98 */
99 if (test_bit(PG_mte_tagged, &page1->flags) ||99 if (page_mte_tagged(page1) || page_mte_tagged(page2))
100 test_bit(PG_mte_tagged, &page2->flags))
101 return addr1 != addr2;100 return addr1 != addr2;
102 101
103 return ret;102 return ret;
@@ -454,7 +453,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, · 350 unmodified lines
454 put_page(page);453 put_page(page);
455 break;454 break;
456 }455 }
457 WARN_ON_ONCE(!test_bit(PG_mte_tagged, &page->flags));456 WARN_ON_ONCE(!page_mte_tagged(page));
458 457
459 /* limit access to the end of the page */458 /* limit access to the end of the page */
460 offset = offset_in_page(addr);459 offset = offset_in_page(addr);
arch/arm64/kvm/Kconfig+2−0
@@ -32,6 +32,8 @@ menuconfig KVM · 31 unmodified lines
32 select KVM_VFIO32 select KVM_VFIO
33 select HAVE_KVM_EVENTFD33 select HAVE_KVM_EVENTFD
34 select HAVE_KVM_IRQFD34 select HAVE_KVM_IRQFD
35 select HAVE_KVM_DIRTY_RING_ACQ_REL
36 select NEED_KVM_DIRTY_RING_WITH_BITMAP
35 select HAVE_KVM_MSI37 select HAVE_KVM_MSI
36 select HAVE_KVM_IRQCHIP38 select HAVE_KVM_IRQCHIP
37 select HAVE_KVM_IRQ_ROUTING39 select HAVE_KVM_IRQ_ROUTING
arch/arm64/kvm/arm.c+51−39
@@ -37,6 +37,7 @@ · 36 unmodified lines
37#include <asm/kvm_arm.h>37#include <asm/kvm_arm.h>
38#include <asm/kvm_asm.h>38#include <asm/kvm_asm.h>
39#include <asm/kvm_mmu.h>39#include <asm/kvm_mmu.h>
40#include <asm/kvm_pkvm.h>
40#include <asm/kvm_emulate.h>41#include <asm/kvm_emulate.h>
41#include <asm/sections.h>42#include <asm/sections.h>
42 43
@@ -50,7 +51,6 @@ DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); · 7 unmodified lines
50DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);51DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
51 52
52DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);53DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
53unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
54DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);54DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
55 55
56static bool vgic_present;56static bool vgic_present;
@@ -138,24 +138,24 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 81 unmodified lines
138{138{
139 int ret;139 int ret;
140 140
141 ret = kvm_arm_setup_stage2(kvm, type);141 ret = kvm_share_hyp(kvm, kvm + 1);
142 if (ret)
143 return ret;
144
145 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu);
146 if (ret)142 if (ret)
147 return ret;143 return ret;
148 144
149 ret = kvm_share_hyp(kvm, kvm + 1);145 ret = pkvm_init_host_vm(kvm);
150 if (ret)146 if (ret)
151 goto out_free_stage2_pgd;147 goto err_unshare_kvm;
152 148
153 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL)) {149 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL)) {
154 ret = -ENOMEM;150 ret = -ENOMEM;
155 goto out_free_stage2_pgd;151 goto err_unshare_kvm;
156 }152 }
157 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);153 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
158 154
155 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
156 if (ret)
157 goto err_free_cpumask;
158
159 kvm_vgic_early_init(kvm);159 kvm_vgic_early_init(kvm);
160 160
161 /* The maximum number of VCPUs is limited by the host's GIC model */161 /* The maximum number of VCPUs is limited by the host's GIC model */
@@ -164,9 +164,18 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 2 unmodified lines
164 set_default_spectre(kvm);164 set_default_spectre(kvm);
165 kvm_arm_init_hypercalls(kvm);165 kvm_arm_init_hypercalls(kvm);
166 166
167 return ret;167 /*
168out_free_stage2_pgd:168 * Initialise the default PMUver before there is a chance to
169 kvm_free_stage2_pgd(&kvm->arch.mmu);169 * create an actual PMU.
170 */
171 kvm->arch.dfr0_pmuver.imp = kvm_arm_pmu_get_pmuver_limit();
172
173 return 0;
174
175err_free_cpumask:
176 free_cpumask_var(kvm->arch.supported_cpus);
177err_unshare_kvm:
178 kvm_unshare_hyp(kvm, kvm + 1);
170 return ret;179 return ret;
171}180}
172 181
@@ -187,6 +196,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 14 unmodified lines
187 196
188 kvm_vgic_destroy(kvm);197 kvm_vgic_destroy(kvm);
189 198
199 if (is_protected_kvm_enabled())
200 pkvm_destroy_hyp_vm(kvm);
201
190 kvm_destroy_vcpus(kvm);202 kvm_destroy_vcpus(kvm);
191 203
192 kvm_unshare_hyp(kvm, kvm + 1);204 kvm_unshare_hyp(kvm, kvm + 1);
@@ -569,6 +581,12 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) · 376 unmodified lines
569 if (ret)581 if (ret)
570 return ret;582 return ret;
571 583
584 if (is_protected_kvm_enabled()) {
585 ret = pkvm_create_hyp_vm(kvm);
586 if (ret)
587 return ret;
588 }
589
572 if (!irqchip_in_kernel(kvm)) {590 if (!irqchip_in_kernel(kvm)) {
573 /*591 /*
574 * Tell the rest of the code that there are userspace irqchip592 * Tell the rest of the code that there are userspace irqchip
@@ -746,6 +764,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) · 171 unmodified lines
746 764
747 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))765 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
748 return kvm_vcpu_suspend(vcpu);766 return kvm_vcpu_suspend(vcpu);
767
768 if (kvm_dirty_ring_check_request(vcpu))
769 return 0;
749 }770 }
750 771
751 return 1;772 return 1;
@@ -1518,7 +1539,7 @@ static int kvm_init_vector_slots(void) · 766 unmodified lines
1518 return 0;1539 return 0;
1519}1540}
1520 1541
1521static void cpu_prepare_hyp_mode(int cpu)1542static void cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
1522{1543{
1523 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);1544 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
1524 unsigned long tcr;1545 unsigned long tcr;
@@ -1534,23 +1555,9 @@ static void cpu_prepare_hyp_mode(int cpu) · 9 unmodified lines
1534 1555
1535 params->mair_el2 = read_sysreg(mair_el1);1556 params->mair_el2 = read_sysreg(mair_el1);
1536 1557
1537 /*
1538 * The ID map may be configured to use an extended virtual address
1539 * range. This is only the case if system RAM is out of range for the
1540 * currently configured page size and VA_BITS, in which case we will
1541 * also need the extended virtual range for the HYP ID map, or we won't
1542 * be able to enable the EL2 MMU.
1543 *
1544 * However, at EL2, there is only one TTBR register, and we can't switch
1545 * between translation tables *and* update TCR_EL2.T0SZ at the same
1546 * time. Bottom line: we need to use the extended range with *both* our
1547 * translation tables.
1548 *
1549 * So use the same T0SZ value we use for the ID map.
1550 */
1551 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;1558 tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
1552 tcr &= ~TCR_T0SZ_MASK;1559 tcr &= ~TCR_T0SZ_MASK;
1553 tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;1560 tcr |= TCR_T0SZ(hyp_va_bits);
1554 params->tcr_el2 = tcr;1561 params->tcr_el2 = tcr;
1555 1562
1556 params->pgd_pa = kvm_mmu_get_httbr();1563 params->pgd_pa = kvm_mmu_get_httbr();
@@ -1844,13 +1851,13 @@ static void teardown_hyp_mode(void) · 287 unmodified lines
1844 free_hyp_pgds();1851 free_hyp_pgds();
1845 for_each_possible_cpu(cpu) {1852 for_each_possible_cpu(cpu) {
1846 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));1853 free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
1847 free_pages(kvm_arm_hyp_percpu_base[cpu], nvhe_percpu_order());1854 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
1848 }1855 }
1849}1856}
1850 1857
1851static int do_pkvm_init(u32 hyp_va_bits)1858static int do_pkvm_init(u32 hyp_va_bits)
1852{1859{
1853 void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);1860 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
1854 int ret;1861 int ret;
1855 1862
1856 preempt_disable();1863 preempt_disable();
@@ -1870,11 +1877,8 @@ static int do_pkvm_init(u32 hyp_va_bits) · 13 unmodified lines
1870 return ret;1877 return ret;
1871}1878}
1872 1879
1873static int kvm_hyp_init_protection(u32 hyp_va_bits)1880static void kvm_hyp_init_symbols(void)
1874{1881{
1875 void *addr = phys_to_virt(hyp_mem_base);
1876 int ret;
1877
1878 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);1882 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1879 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);1883 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
1880 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);1884 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
@@ -1883,6 +1887,14 @@ static int kvm_hyp_init_protection(u32 hyp_va_bits) · 2 unmodified lines
1883 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);1887 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
1884 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);1888 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
1885 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);1889 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
1890 kvm_nvhe_sym(__icache_flags) = __icache_flags;
1891 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
1892}
1893
1894static int kvm_hyp_init_protection(u32 hyp_va_bits)
1895{
1896 void *addr = phys_to_virt(hyp_mem_base);
1897 int ret;
1886 1898
1887 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);1899 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
1888 if (ret)1900 if (ret)
@@ -1950,7 +1962,7 @@ static int init_hyp_mode(void) · 61 unmodified lines
1950 1962
1951 page_addr = page_address(page);1963 page_addr = page_address(page);
1952 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());1964 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
1953 kvm_arm_hyp_percpu_base[cpu] = (unsigned long)page_addr;1965 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
1954 }1966 }
1955 1967
1956 /*1968 /*
@@ -2043,7 +2055,7 @@ static int init_hyp_mode(void) · 86 unmodified lines
2043 }2055 }
2044 2056
2045 for_each_possible_cpu(cpu) {2057 for_each_possible_cpu(cpu) {
2046 char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];2058 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2047 char *percpu_end = percpu_begin + nvhe_percpu_size();2059 char *percpu_end = percpu_begin + nvhe_percpu_size();
2048 2060
2049 /* Map Hyp percpu pages */2061 /* Map Hyp percpu pages */
@@ -2054,9 +2066,11 @@ static int init_hyp_mode(void) · 4 unmodified lines
2054 }2066 }
2055 2067
2056 /* Prepare the CPU initialization parameters */2068 /* Prepare the CPU initialization parameters */
2057 cpu_prepare_hyp_mode(cpu);2069 cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2058 }2070 }
2059 2071
2072 kvm_hyp_init_symbols();
2073
2060 if (is_protected_kvm_enabled()) {2074 if (is_protected_kvm_enabled()) {
2061 init_cpu_logical_map();2075 init_cpu_logical_map();
2062 2076
@@ -2064,9 +2078,7 @@ static int init_hyp_mode(void) · 1 unmodified line
2064 err = -ENODEV;2078 err = -ENODEV;
2065 goto out_err;2079 goto out_err;
2066 }2080 }
2067 }
2068 2081
2069 if (is_protected_kvm_enabled()) {
2070 err = kvm_hyp_init_protection(hyp_va_bits);2082 err = kvm_hyp_init_protection(hyp_va_bits);
2071 if (err) {2083 if (err) {
2072 kvm_err("Failed to init hyp memory protection\n");2084 kvm_err("Failed to init hyp memory protection\n");
arch/arm64/kvm/guest.c+11−7
@@ -1059,7 +1059,7 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 1058 unmodified lines
1059 maddr = page_address(page);1059 maddr = page_address(page);
1060 1060
1061 if (!write) {1061 if (!write) {
1062 if (test_bit(PG_mte_tagged, &page->flags))1062 if (page_mte_tagged(page))
1063 num_tags = mte_copy_tags_to_user(tags, maddr,1063 num_tags = mte_copy_tags_to_user(tags, maddr,
1064 MTE_GRANULES_PER_PAGE);1064 MTE_GRANULES_PER_PAGE);
1065 else1065 else
@@ -1068,15 +1068,19 @@ long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, · 2 unmodified lines
1068 clear_user(tags, MTE_GRANULES_PER_PAGE);1068 clear_user(tags, MTE_GRANULES_PER_PAGE);
1069 kvm_release_pfn_clean(pfn);1069 kvm_release_pfn_clean(pfn);
1070 } else {1070 } else {
1071 /*
1072 * Only locking to serialise with a concurrent
1073 * set_pte_at() in the VMM but still overriding the
1074 * tags, hence ignoring the return value.
1075 */
1076 try_page_mte_tagging(page);
1071 num_tags = mte_copy_tags_from_user(maddr, tags,1077 num_tags = mte_copy_tags_from_user(maddr, tags,
1072 MTE_GRANULES_PER_PAGE);1078 MTE_GRANULES_PER_PAGE);
1073 1079
1074 /*1080 /* uaccess failed, don't leave stale tags */
1075 * Set the flag after checking the write1081 if (num_tags != MTE_GRANULES_PER_PAGE)
1076 * completed fully1082 mte_clear_page_tags(page);
1077 */1083 set_page_mte_tagged(page);
1078 if (num_tags == MTE_GRANULES_PER_PAGE)
1079 set_bit(PG_mte_tagged, &page->flags);
1080 1084
1081 kvm_release_pfn_dirty(pfn);1085 kvm_release_pfn_dirty(pfn);
1082 }1086 }
arch/arm64/kvm/hyp/hyp-constants.c+3−0
@@ -2,9 +2,12 @@ · 1 unmodified line
2 2
3#include <linux/kbuild.h>3#include <linux/kbuild.h>
4#include <nvhe/memory.h>4#include <nvhe/memory.h>
5#include <nvhe/pkvm.h>
5 6
6int main(void)7int main(void)
7{8{
8 DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));9 DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));
10 DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm));
11 DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu));
9 return 0;12 return 0;
10}13}
arch/arm64/kvm/hyp/include/nvhe/mem_protect.h+21−4
@@ -8,8 +8,10 @@ · 7 unmodified lines
8#define __KVM_NVHE_MEM_PROTECT__8#define __KVM_NVHE_MEM_PROTECT__
9#include <linux/kvm_host.h>9#include <linux/kvm_host.h>
10#include <asm/kvm_hyp.h>10#include <asm/kvm_hyp.h>
11#include <asm/kvm_mmu.h>
11#include <asm/kvm_pgtable.h>12#include <asm/kvm_pgtable.h>
12#include <asm/virt.h>13#include <asm/virt.h>
14#include <nvhe/pkvm.h>
13#include <nvhe/spinlock.h>15#include <nvhe/spinlock.h>
14 16
15/*17/*
@@ -43,30 +45,45 @@ static inline enum pkvm_page_state pkvm_getstate(enum kvm_pgtable_prot prot) · 27 unmodified lines
43 return prot & PKVM_PAGE_STATE_PROT_MASK;45 return prot & PKVM_PAGE_STATE_PROT_MASK;
44}46}
45 47
46struct host_kvm {48struct host_mmu {
47 struct kvm_arch arch;49 struct kvm_arch arch;
48 struct kvm_pgtable pgt;50 struct kvm_pgtable pgt;
49 struct kvm_pgtable_mm_ops mm_ops;51 struct kvm_pgtable_mm_ops mm_ops;
50 hyp_spinlock_t lock;52 hyp_spinlock_t lock;
51};53};
52extern struct host_kvm host_kvm;54extern struct host_mmu host_mmu;
53 55
54extern const u8 pkvm_hyp_id;56/* This corresponds to page-table locking order */
57enum pkvm_component_id {
58 PKVM_ID_HOST,
59 PKVM_ID_HYP,
60};
61
62extern unsigned long hyp_nr_cpus;
55 63
56int __pkvm_prot_finalize(void);64int __pkvm_prot_finalize(void);
57int __pkvm_host_share_hyp(u64 pfn);65int __pkvm_host_share_hyp(u64 pfn);
58int __pkvm_host_unshare_hyp(u64 pfn);66int __pkvm_host_unshare_hyp(u64 pfn);
67int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages);
68int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages);
59 69
60bool addr_is_memory(phys_addr_t phys);70bool addr_is_memory(phys_addr_t phys);
61int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);71int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);
62int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id);72int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id);
63int kvm_host_prepare_stage2(void *pgt_pool_base);73int kvm_host_prepare_stage2(void *pgt_pool_base);
74int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd);
64void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt);75void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt);
65 76
77int hyp_pin_shared_mem(void *from, void *to);
78void hyp_unpin_shared_mem(void *from, void *to);
79void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc);
80int refill_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages,
81 struct kvm_hyp_memcache *host_mc);
82
66static __always_inline void __load_host_stage2(void)83static __always_inline void __load_host_stage2(void)
67{84{
68 if (static_branch_likely(&kvm_protected_mode_initialized))85 if (static_branch_likely(&kvm_protected_mode_initialized))
69 __load_stage2(&host_kvm.arch.mmu, &host_kvm.arch);86 __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
70 else87 else
71 write_sysreg(0, vttbr_el2);88 write_sysreg(0, vttbr_el2);
72}89}
arch/arm64/kvm/hyp/include/nvhe/memory.h+27−0
@@ -38,6 +38,10 @@ static inline phys_addr_t hyp_virt_to_phys(void *addr) · 37 unmodified lines
38#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))38#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))
39#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)39#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)
40 40
41/*
42 * Refcounting for 'struct hyp_page'.
43 * hyp_pool::lock must be held if atomic access to the refcount is required.
44 */
41static inline int hyp_page_count(void *addr)45static inline int hyp_page_count(void *addr)
42{46{
43 struct hyp_page *p = hyp_virt_to_page(addr);47 struct hyp_page *p = hyp_virt_to_page(addr);
@@ -45,4 +49,27 @@ static inline int hyp_page_count(void *addr) · 1 unmodified line
45 return p->refcount;49 return p->refcount;
46}50}
47 51
52static inline void hyp_page_ref_inc(struct hyp_page *p)
53{
54 BUG_ON(p->refcount == USHRT_MAX);
55 p->refcount++;
56}
57
58static inline void hyp_page_ref_dec(struct hyp_page *p)
59{
60 BUG_ON(!p->refcount);
61 p->refcount--;
62}
63
64static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
65{
66 hyp_page_ref_dec(p);
67 return (p->refcount == 0);
68}
69
70static inline void hyp_set_page_refcounted(struct hyp_page *p)
71{
72 BUG_ON(p->refcount);
73 p->refcount = 1;
74}
48#endif /* __KVM_HYP_MEMORY_H */75#endif /* __KVM_HYP_MEMORY_H */
arch/arm64/kvm/hyp/include/nvhe/mm.h+5−13
@@ -13,9 +13,13 @@ · 12 unmodified lines
13extern struct kvm_pgtable pkvm_pgtable;13extern struct kvm_pgtable pkvm_pgtable;
14extern hyp_spinlock_t pkvm_pgd_lock;14extern hyp_spinlock_t pkvm_pgd_lock;
15 15
16int hyp_create_pcpu_fixmap(void);
17void *hyp_fixmap_map(phys_addr_t phys);
18void hyp_fixmap_unmap(void);
19
16int hyp_create_idmap(u32 hyp_va_bits);20int hyp_create_idmap(u32 hyp_va_bits);
17int hyp_map_vectors(void);21int hyp_map_vectors(void);
18int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back);22int hyp_back_vmemmap(phys_addr_t back);
19int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);23int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
20int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);24int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
21int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);25int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);
@@ -24,16 +28,4 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, · 2 unmodified lines
24 unsigned long *haddr);28 unsigned long *haddr);
25int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);29int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);
26 30
27static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size,
28 unsigned long *start, unsigned long *end)
29{
30 unsigned long nr_pages = size >> PAGE_SHIFT;
31 struct hyp_page *p = hyp_phys_to_page(phys);
32
33 *start = (unsigned long)p;
34 *end = *start + nr_pages * sizeof(struct hyp_page);
35 *start = ALIGN_DOWN(*start, PAGE_SIZE);
36 *end = ALIGN(*end, PAGE_SIZE);
37}
38
39#endif /* __KVM_HYP_MM_H */31#endif /* __KVM_HYP_MM_H */
arch/arm64/kvm/hyp/include/nvhe/pkvm.h+68−0
@@ -0,0 +1,68 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2021 Google LLC
4 * Author: Fuad Tabba <tabba@google.com>
5 */
6
7#ifndef __ARM64_KVM_NVHE_PKVM_H__
8#define __ARM64_KVM_NVHE_PKVM_H__
9
10#include <asm/kvm_pkvm.h>
11
12#include <nvhe/gfp.h>
13#include <nvhe/spinlock.h>
14
15/*
16 * Holds the relevant data for maintaining the vcpu state completely at hyp.
17 */
18struct pkvm_hyp_vcpu {
19 struct kvm_vcpu vcpu;
20
21 /* Backpointer to the host's (untrusted) vCPU instance. */
22 struct kvm_vcpu *host_vcpu;
23};
24
25/*
26 * Holds the relevant data for running a protected vm.
27 */
28struct pkvm_hyp_vm {
29 struct kvm kvm;
30
31 /* Backpointer to the host's (untrusted) KVM instance. */
32 struct kvm *host_kvm;
33
34 /* The guest's stage-2 page-table managed by the hypervisor. */
35 struct kvm_pgtable pgt;
36 struct kvm_pgtable_mm_ops mm_ops;
37 struct hyp_pool pool;
38 hyp_spinlock_t lock;
39
40 /*
41 * The number of vcpus initialized and ready to run.
42 * Modifying this is protected by 'vm_table_lock'.
43 */
44 unsigned int nr_vcpus;
45
46 /* Array of the hyp vCPU structures for this VM. */
47 struct pkvm_hyp_vcpu *vcpus[];
48};
49
50static inline struct pkvm_hyp_vm *
51pkvm_hyp_vcpu_to_hyp_vm(struct pkvm_hyp_vcpu *hyp_vcpu)
52{
53 return container_of(hyp_vcpu->vcpu.kvm, struct pkvm_hyp_vm, kvm);
54}
55
56void pkvm_hyp_vm_table_init(void *tbl);
57
58int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
59 unsigned long pgd_hva);
60int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
61 unsigned long vcpu_hva);
62int __pkvm_teardown_vm(pkvm_handle_t handle);
63
64struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
65 unsigned int vcpu_idx);
66void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu);
67
68#endif /* __ARM64_KVM_NVHE_PKVM_H__ */
arch/arm64/kvm/hyp/include/nvhe/spinlock.h+9−1
@@ -28,9 +28,17 @@ typedef union hyp_spinlock { · 27 unmodified lines
28 };28 };
29} hyp_spinlock_t;29} hyp_spinlock_t;
30 30
31#define __HYP_SPIN_LOCK_INITIALIZER \
32 { .__val = 0 }
33
34#define __HYP_SPIN_LOCK_UNLOCKED \
35 ((hyp_spinlock_t) __HYP_SPIN_LOCK_INITIALIZER)
36
37#define DEFINE_HYP_SPINLOCK(x) hyp_spinlock_t x = __HYP_SPIN_LOCK_UNLOCKED
38
31#define hyp_spin_lock_init(l) \39#define hyp_spin_lock_init(l) \
32do { \40do { \
33 *(l) = (hyp_spinlock_t){ .__val = 0 }; \41 *(l) = __HYP_SPIN_LOCK_UNLOCKED; \
34} while (0)42} while (0)
35 43
36static inline void hyp_spin_lock(hyp_spinlock_t *lock)44static inline void hyp_spin_lock(hyp_spinlock_t *lock)
arch/arm64/kvm/hyp/nvhe/cache.S+11−0
@@ -12,3 +12,14 @@ SYM_FUNC_START(__pi_dcache_clean_inval_poc) · 11 unmodified lines
12 ret12 ret
13SYM_FUNC_END(__pi_dcache_clean_inval_poc)13SYM_FUNC_END(__pi_dcache_clean_inval_poc)
14SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)14SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)
15
16SYM_FUNC_START(__pi_icache_inval_pou)
17alternative_if ARM64_HAS_CACHE_DIC
18 isb
19 ret
20alternative_else_nop_endif
21
22 invalidate_icache_by_line x0, x1, x2, x3
23 ret
24SYM_FUNC_END(__pi_icache_inval_pou)
25SYM_FUNC_ALIAS(icache_inval_pou, __pi_icache_inval_pou)
arch/arm64/kvm/hyp/nvhe/hyp-main.c+108−2
@@ -15,17 +15,93 @@ · 14 unmodified lines
15 15
16#include <nvhe/mem_protect.h>16#include <nvhe/mem_protect.h>
17#include <nvhe/mm.h>17#include <nvhe/mm.h>
18#include <nvhe/pkvm.h>
18#include <nvhe/trap_handler.h>19#include <nvhe/trap_handler.h>
19 20
20DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);21DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
21 22
22void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);23void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
23 24
25static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
26{
27 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
28
29 hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
30
31 hyp_vcpu->vcpu.arch.sve_state = kern_hyp_va(host_vcpu->arch.sve_state);
32 hyp_vcpu->vcpu.arch.sve_max_vl = host_vcpu->arch.sve_max_vl;
33
34 hyp_vcpu->vcpu.arch.hw_mmu = host_vcpu->arch.hw_mmu;
35
36 hyp_vcpu->vcpu.arch.hcr_el2 = host_vcpu->arch.hcr_el2;
37 hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2;
38 hyp_vcpu->vcpu.arch.cptr_el2 = host_vcpu->arch.cptr_el2;
39
40 hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags;
41 hyp_vcpu->vcpu.arch.fp_state = host_vcpu->arch.fp_state;
42
43 hyp_vcpu->vcpu.arch.debug_ptr = kern_hyp_va(host_vcpu->arch.debug_ptr);
44 hyp_vcpu->vcpu.arch.host_fpsimd_state = host_vcpu->arch.host_fpsimd_state;
45
46 hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2;
47
48 hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3;
49}
50
51static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
52{
53 struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
54 struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
55 struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
56 unsigned int i;
57
58 host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
59
60 host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2;
61 host_vcpu->arch.cptr_el2 = hyp_vcpu->vcpu.arch.cptr_el2;
62
63 host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault;
64
65 host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags;
66 host_vcpu->arch.fp_state = hyp_vcpu->vcpu.arch.fp_state;
67
68 host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr;
69 for (i = 0; i < hyp_cpu_if->used_lrs; ++i)
70 host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
71}
72
24static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)73static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
25{74{
26 DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);75 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1);
76 int ret;
27 77
28 cpu_reg(host_ctxt, 1) = __kvm_vcpu_run(kern_hyp_va(vcpu));78 host_vcpu = kern_hyp_va(host_vcpu);
79
80 if (unlikely(is_protected_kvm_enabled())) {
81 struct pkvm_hyp_vcpu *hyp_vcpu;
82 struct kvm *host_kvm;
83
84 host_kvm = kern_hyp_va(host_vcpu->kvm);
85 hyp_vcpu = pkvm_load_hyp_vcpu(host_kvm->arch.pkvm.handle,
86 host_vcpu->vcpu_idx);
87 if (!hyp_vcpu) {
88 ret = -EINVAL;
89 goto out;
90 }
91
92 flush_hyp_vcpu(hyp_vcpu);
93
94 ret = __kvm_vcpu_run(&hyp_vcpu->vcpu);
95
96 sync_hyp_vcpu(hyp_vcpu);
97 pkvm_put_hyp_vcpu(hyp_vcpu);
98 } else {
99 /* The host is fully trusted, run its vCPU directly. */
100 ret = __kvm_vcpu_run(host_vcpu);
101 }
102
103out:
104 cpu_reg(host_ctxt, 1) = ret;
29}105}
30 106
31static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)107static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)
@@ -191,6 +267,33 @@ static void handle___pkvm_vcpu_init_traps(struct kvm_cpu_context *host_ctxt) · 159 unmodified lines
191 __pkvm_vcpu_init_traps(kern_hyp_va(vcpu));267 __pkvm_vcpu_init_traps(kern_hyp_va(vcpu));
192}268}
193 269
270static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt)
271{
272 DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1);
273 DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2);
274 DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3);
275
276 host_kvm = kern_hyp_va(host_kvm);
277 cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva);
278}
279
280static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt)
281{
282 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
283 DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2);
284 DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3);
285
286 host_vcpu = kern_hyp_va(host_vcpu);
287 cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva);
288}
289
290static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt)
291{
292 DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
293
294 cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle);
295}
296
194typedef void (*hcall_t)(struct kvm_cpu_context *);297typedef void (*hcall_t)(struct kvm_cpu_context *);
195 298
196#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x299#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -220,6 +323,9 @@ static const hcall_t host_hcall[] = { · 23 unmodified lines
220 HANDLE_FUNC(__vgic_v3_save_aprs),323 HANDLE_FUNC(__vgic_v3_save_aprs),
221 HANDLE_FUNC(__vgic_v3_restore_aprs),324 HANDLE_FUNC(__vgic_v3_restore_aprs),
222 HANDLE_FUNC(__pkvm_vcpu_init_traps),325 HANDLE_FUNC(__pkvm_vcpu_init_traps),
326 HANDLE_FUNC(__pkvm_init_vm),
327 HANDLE_FUNC(__pkvm_init_vcpu),
328 HANDLE_FUNC(__pkvm_teardown_vm),
223};329};
224 330
225static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)331static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
arch/arm64/kvm/hyp/nvhe/hyp-smp.c+2−0
@@ -23,6 +23,8 @@ u64 cpu_logical_map(unsigned int cpu) · 22 unmodified lines
23 return hyp_cpu_logical_map[cpu];23 return hyp_cpu_logical_map[cpu];
24}24}
25 25
26unsigned long __ro_after_init kvm_arm_hyp_percpu_base[NR_CPUS];
27
26unsigned long __hyp_per_cpu_offset(unsigned int cpu)28unsigned long __hyp_per_cpu_offset(unsigned int cpu)
27{29{
28 unsigned long *cpu_base_array;30 unsigned long *cpu_base_array;
arch/arm64/kvm/hyp/nvhe/mem_protect.c+476−45
@@ -21,21 +21,33 @@ · 20 unmodified lines
21 21
22#define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP)22#define KVM_HOST_S2_FLAGS (KVM_PGTABLE_S2_NOFWB | KVM_PGTABLE_S2_IDMAP)
23 23
24extern unsigned long hyp_nr_cpus;24struct host_mmu host_mmu;
25struct host_kvm host_kvm;
26 25
27static struct hyp_pool host_s2_pool;26static struct hyp_pool host_s2_pool;
28 27
29const u8 pkvm_hyp_id = 1;28static DEFINE_PER_CPU(struct pkvm_hyp_vm *, __current_vm);
29#define current_vm (*this_cpu_ptr(&__current_vm))
30
31static void guest_lock_component(struct pkvm_hyp_vm *vm)
32{
33 hyp_spin_lock(&vm->lock);
34 current_vm = vm;
35}
36
37static void guest_unlock_component(struct pkvm_hyp_vm *vm)
38{
39 current_vm = NULL;
40 hyp_spin_unlock(&vm->lock);
41}
30 42
31static void host_lock_component(void)43static void host_lock_component(void)
32{44{
33 hyp_spin_lock(&host_kvm.lock);45 hyp_spin_lock(&host_mmu.lock);
34}46}
35 47
36static void host_unlock_component(void)48static void host_unlock_component(void)
37{49{
38 hyp_spin_unlock(&host_kvm.lock);50 hyp_spin_unlock(&host_mmu.lock);
39}51}
40 52
41static void hyp_lock_component(void)53static void hyp_lock_component(void)
@@ -79,6 +91,11 @@ static void host_s2_put_page(void *addr) · 37 unmodified lines
79 hyp_put_page(&host_s2_pool, addr);91 hyp_put_page(&host_s2_pool, addr);
80}92}
81 93
94static void host_s2_free_removed_table(void *addr, u32 level)
95{
96 kvm_pgtable_stage2_free_removed(&host_mmu.mm_ops, addr, level);
97}
98
82static int prepare_s2_pool(void *pgt_pool_base)99static int prepare_s2_pool(void *pgt_pool_base)
83{100{
84 unsigned long nr_pages, pfn;101 unsigned long nr_pages, pfn;
@@ -90,9 +107,10 @@ static int prepare_s2_pool(void *pgt_pool_base) · 5 unmodified lines
90 if (ret)107 if (ret)
91 return ret;108 return ret;
92 109
93 host_kvm.mm_ops = (struct kvm_pgtable_mm_ops) {110 host_mmu.mm_ops = (struct kvm_pgtable_mm_ops) {
94 .zalloc_pages_exact = host_s2_zalloc_pages_exact,111 .zalloc_pages_exact = host_s2_zalloc_pages_exact,
95 .zalloc_page = host_s2_zalloc_page,112 .zalloc_page = host_s2_zalloc_page,
113 .free_removed_table = host_s2_free_removed_table,
96 .phys_to_virt = hyp_phys_to_virt,114 .phys_to_virt = hyp_phys_to_virt,
97 .virt_to_phys = hyp_virt_to_phys,115 .virt_to_phys = hyp_virt_to_phys,
98 .page_count = hyp_page_count,116 .page_count = hyp_page_count,
@@ -111,7 +129,7 @@ static void prepare_host_vtcr(void) · 12 unmodified lines
111 parange = kvm_get_parange(id_aa64mmfr0_el1_sys_val);129 parange = kvm_get_parange(id_aa64mmfr0_el1_sys_val);
112 phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange);130 phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange);
113 131
114 host_kvm.arch.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val,132 host_mmu.arch.vtcr = kvm_get_vtcr(id_aa64mmfr0_el1_sys_val,
115 id_aa64mmfr1_el1_sys_val, phys_shift);133 id_aa64mmfr1_el1_sys_val, phys_shift);
116}134}
117 135
@@ -119,45 +137,170 @@ static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot pr · 1 unmodified line
119 137
120int kvm_host_prepare_stage2(void *pgt_pool_base)138int kvm_host_prepare_stage2(void *pgt_pool_base)
121{139{
122 struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;140 struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
123 int ret;141 int ret;
124 142
125 prepare_host_vtcr();143 prepare_host_vtcr();
126 hyp_spin_lock_init(&host_kvm.lock);144 hyp_spin_lock_init(&host_mmu.lock);
127 mmu->arch = &host_kvm.arch;145 mmu->arch = &host_mmu.arch;
128 146
129 ret = prepare_s2_pool(pgt_pool_base);147 ret = prepare_s2_pool(pgt_pool_base);
130 if (ret)148 if (ret)
131 return ret;149 return ret;
132 150
133 ret = __kvm_pgtable_stage2_init(&host_kvm.pgt, mmu,151 ret = __kvm_pgtable_stage2_init(&host_mmu.pgt, mmu,
134 &host_kvm.mm_ops, KVM_HOST_S2_FLAGS,152 &host_mmu.mm_ops, KVM_HOST_S2_FLAGS,
135 host_stage2_force_pte_cb);153 host_stage2_force_pte_cb);
136 if (ret)154 if (ret)
137 return ret;155 return ret;
138 156
139 mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);157 mmu->pgd_phys = __hyp_pa(host_mmu.pgt.pgd);
140 mmu->pgt = &host_kvm.pgt;158 mmu->pgt = &host_mmu.pgt;
141 atomic64_set(&mmu->vmid.id, 0);159 atomic64_set(&mmu->vmid.id, 0);
142 160
143 return 0;161 return 0;
144}162}
145 163
164static bool guest_stage2_force_pte_cb(u64 addr, u64 end,
165 enum kvm_pgtable_prot prot)
166{
167 return true;
168}
169
170static void *guest_s2_zalloc_pages_exact(size_t size)
171{
172 void *addr = hyp_alloc_pages(&current_vm->pool, get_order(size));
173
174 WARN_ON(size != (PAGE_SIZE << get_order(size)));
175 hyp_split_page(hyp_virt_to_page(addr));
176
177 return addr;
178}
179
180static void guest_s2_free_pages_exact(void *addr, unsigned long size)
181{
182 u8 order = get_order(size);
183 unsigned int i;
184
185 for (i = 0; i < (1 << order); i++)
186 hyp_put_page(&current_vm->pool, addr + (i * PAGE_SIZE));
187}
188
189static void *guest_s2_zalloc_page(void *mc)
190{
191 struct hyp_page *p;
192 void *addr;
193
194 addr = hyp_alloc_pages(&current_vm->pool, 0);
195 if (addr)
196 return addr;
197
198 addr = pop_hyp_memcache(mc, hyp_phys_to_virt);
199 if (!addr)
200 return addr;
201
202 memset(addr, 0, PAGE_SIZE);
203 p = hyp_virt_to_page(addr);
204 memset(p, 0, sizeof(*p));
205 p->refcount = 1;
206
207 return addr;
208}
209
210static void guest_s2_get_page(void *addr)
211{
212 hyp_get_page(&current_vm->pool, addr);
213}
214
215static void guest_s2_put_page(void *addr)
216{
217 hyp_put_page(&current_vm->pool, addr);
218}
219
220static void clean_dcache_guest_page(void *va, size_t size)
221{
222 __clean_dcache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
223 hyp_fixmap_unmap();
224}
225
226static void invalidate_icache_guest_page(void *va, size_t size)
227{
228 __invalidate_icache_guest_page(hyp_fixmap_map(__hyp_pa(va)), size);
229 hyp_fixmap_unmap();
230}
231
232int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd)
233{
234 struct kvm_s2_mmu *mmu = &vm->kvm.arch.mmu;
235 unsigned long nr_pages;
236 int ret;
237
238 nr_pages = kvm_pgtable_stage2_pgd_size(vm->kvm.arch.vtcr) >> PAGE_SHIFT;
239 ret = hyp_pool_init(&vm->pool, hyp_virt_to_pfn(pgd), nr_pages, 0);
240 if (ret)
241 return ret;
242
243 hyp_spin_lock_init(&vm->lock);
244 vm->mm_ops = (struct kvm_pgtable_mm_ops) {
245 .zalloc_pages_exact = guest_s2_zalloc_pages_exact,
246 .free_pages_exact = guest_s2_free_pages_exact,
247 .zalloc_page = guest_s2_zalloc_page,
248 .phys_to_virt = hyp_phys_to_virt,
249 .virt_to_phys = hyp_virt_to_phys,
250 .page_count = hyp_page_count,
251 .get_page = guest_s2_get_page,
252 .put_page = guest_s2_put_page,
253 .dcache_clean_inval_poc = clean_dcache_guest_page,
254 .icache_inval_pou = invalidate_icache_guest_page,
255 };
256
257 guest_lock_component(vm);
258 ret = __kvm_pgtable_stage2_init(mmu->pgt, mmu, &vm->mm_ops, 0,
259 guest_stage2_force_pte_cb);
260 guest_unlock_component(vm);
261 if (ret)
262 return ret;
263
264 vm->kvm.arch.mmu.pgd_phys = __hyp_pa(vm->pgt.pgd);
265
266 return 0;
267}
268
269void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc)
270{
271 void *addr;
272
273 /* Dump all pgtable pages in the hyp_pool */
274 guest_lock_component(vm);
275 kvm_pgtable_stage2_destroy(&vm->pgt);
276 vm->kvm.arch.mmu.pgd_phys = 0ULL;
277 guest_unlock_component(vm);
278
279 /* Drain the hyp_pool into the memcache */
280 addr = hyp_alloc_pages(&vm->pool, 0);
281 while (addr) {
282 memset(hyp_virt_to_page(addr), 0, sizeof(struct hyp_page));
283 push_hyp_memcache(mc, addr, hyp_virt_to_phys);
284 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1));
285 addr = hyp_alloc_pages(&vm->pool, 0);
286 }
287}
288
146int __pkvm_prot_finalize(void)289int __pkvm_prot_finalize(void)
147{290{
148 struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;291 struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu;
149 struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);292 struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
150 293
151 if (params->hcr_el2 & HCR_VM)294 if (params->hcr_el2 & HCR_VM)
152 return -EPERM;295 return -EPERM;
153 296
154 params->vttbr = kvm_get_vttbr(mmu);297 params->vttbr = kvm_get_vttbr(mmu);
155 params->vtcr = host_kvm.arch.vtcr;298 params->vtcr = host_mmu.arch.vtcr;
156 params->hcr_el2 |= HCR_VM;299 params->hcr_el2 |= HCR_VM;
157 kvm_flush_dcache_to_poc(params, sizeof(*params));300 kvm_flush_dcache_to_poc(params, sizeof(*params));
158 301
159 write_sysreg(params->hcr_el2, hcr_el2);302 write_sysreg(params->hcr_el2, hcr_el2);
160 __load_stage2(&host_kvm.arch.mmu, &host_kvm.arch);303 __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
161 304
162 /*305 /*
163 * Make sure to have an ISB before the TLB maintenance below but only306 * Make sure to have an ISB before the TLB maintenance below but only
@@ -175,7 +318,7 @@ int __pkvm_prot_finalize(void) · 11 unmodified lines
175 318
176static int host_stage2_unmap_dev_all(void)319static int host_stage2_unmap_dev_all(void)
177{320{
178 struct kvm_pgtable *pgt = &host_kvm.pgt;321 struct kvm_pgtable *pgt = &host_mmu.pgt;
179 struct memblock_region *reg;322 struct memblock_region *reg;
180 u64 addr = 0;323 u64 addr = 0;
181 int i, ret;324 int i, ret;
@@ -195,7 +338,7 @@ struct kvm_mem_range { · 13 unmodified lines
195 u64 end;338 u64 end;
196};339};
197 340
198static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)341static struct memblock_region *find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)
199{342{
200 int cur, left = 0, right = hyp_memblock_nr;343 int cur, left = 0, right = hyp_memblock_nr;
201 struct memblock_region *reg;344 struct memblock_region *reg;
@@ -218,18 +361,28 @@ static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range) · 16 unmodified lines
218 } else {361 } else {
219 range->start = reg->base;362 range->start = reg->base;
220 range->end = end;363 range->end = end;
221 return true;364 return reg;
222 }365 }
223 }366 }
224 367
225 return false;368 return NULL;
226}369}
227 370
228bool addr_is_memory(phys_addr_t phys)371bool addr_is_memory(phys_addr_t phys)
229{372{
230 struct kvm_mem_range range;373 struct kvm_mem_range range;
231 374
232 return find_mem_range(phys, &range);375 return !!find_mem_range(phys, &range);
376}
377
378static bool addr_is_allowed_memory(phys_addr_t phys)
379{
380 struct memblock_region *reg;
381 struct kvm_mem_range range;
382
383 reg = find_mem_range(phys, &range);
384
385 return reg && !(reg->flags & MEMBLOCK_NOMAP);
233}386}
234 387
235static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range)388static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range)
@@ -250,8 +403,8 @@ static bool range_is_memory(u64 start, u64 end) · 14 unmodified lines
250static inline int __host_stage2_idmap(u64 start, u64 end,403static inline int __host_stage2_idmap(u64 start, u64 end,
251 enum kvm_pgtable_prot prot)404 enum kvm_pgtable_prot prot)
252{405{
253 return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start,406 return kvm_pgtable_stage2_map(&host_mmu.pgt, start, end - start, start,
254 prot, &host_s2_pool);407 prot, &host_s2_pool, 0);
255}408}
256 409
257/*410/*
@@ -263,7 +416,7 @@ static inline int __host_stage2_idmap(u64 start, u64 end, · 5 unmodified lines
263#define host_stage2_try(fn, ...) \416#define host_stage2_try(fn, ...) \
264 ({ \417 ({ \
265 int __ret; \418 int __ret; \
266 hyp_assert_lock_held(&host_kvm.lock); \419 hyp_assert_lock_held(&host_mmu.lock); \
267 __ret = fn(__VA_ARGS__); \420 __ret = fn(__VA_ARGS__); \
268 if (__ret == -ENOMEM) { \421 if (__ret == -ENOMEM) { \
269 __ret = host_stage2_unmap_dev_all(); \422 __ret = host_stage2_unmap_dev_all(); \
@@ -286,8 +439,8 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) · 16 unmodified lines
286 u32 level;439 u32 level;
287 int ret;440 int ret;
288 441
289 hyp_assert_lock_held(&host_kvm.lock);442 hyp_assert_lock_held(&host_mmu.lock);
290 ret = kvm_pgtable_get_leaf(&host_kvm.pgt, addr, &pte, &level);443 ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
291 if (ret)444 if (ret)
292 return ret;445 return ret;
293 446
@@ -319,7 +472,7 @@ int host_stage2_idmap_locked(phys_addr_t addr, u64 size, · 25 unmodified lines
319 472
320int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)473int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
321{474{
322 return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_kvm.pgt,475 return host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
323 addr, size, &host_s2_pool, owner_id);476 addr, size, &host_s2_pool, owner_id);
324}477}
325 478
@@ -348,7 +501,7 @@ static bool host_stage2_force_pte_cb(u64 addr, u64 end, enum kvm_pgtable_prot pr · 22 unmodified lines
348static int host_stage2_idmap(u64 addr)501static int host_stage2_idmap(u64 addr)
349{502{
350 struct kvm_mem_range range;503 struct kvm_mem_range range;
351 bool is_memory = find_mem_range(addr, &range);504 bool is_memory = !!find_mem_range(addr, &range);
352 enum kvm_pgtable_prot prot;505 enum kvm_pgtable_prot prot;
353 int ret;506 int ret;
354 507
@@ -380,12 +533,6 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) · 25 unmodified lines
380 BUG_ON(ret && ret != -EAGAIN);533 BUG_ON(ret && ret != -EAGAIN);
381}534}
382 535
383/* This corresponds to locking order */
384enum pkvm_component_id {
385 PKVM_ID_HOST,
386 PKVM_ID_HYP,
387};
388
389struct pkvm_mem_transition {536struct pkvm_mem_transition {
390 u64 nr_pages;537 u64 nr_pages;
391 538
@@ -399,6 +546,9 @@ struct pkvm_mem_transition { · 7 unmodified lines
399 /* Address in the completer's address space */546 /* Address in the completer's address space */
400 u64 completer_addr;547 u64 completer_addr;
401 } host;548 } host;
549 struct {
550 u64 completer_addr;
551 } hyp;
402 };552 };
403 } initiator;553 } initiator;
404 554
@@ -412,23 +562,24 @@ struct pkvm_mem_share { · 7 unmodified lines
412 const enum kvm_pgtable_prot completer_prot;562 const enum kvm_pgtable_prot completer_prot;
413};563};
414 564
565struct pkvm_mem_donation {
566 const struct pkvm_mem_transition tx;
567};
568
415struct check_walk_data {569struct check_walk_data {
416 enum pkvm_page_state desired;570 enum pkvm_page_state desired;
417 enum pkvm_page_state (*get_page_state)(kvm_pte_t pte);571 enum pkvm_page_state (*get_page_state)(kvm_pte_t pte);
418};572};
419 573
420static int __check_page_state_visitor(u64 addr, u64 end, u32 level,574static int __check_page_state_visitor(const struct kvm_pgtable_visit_ctx *ctx,
421 kvm_pte_t *ptep,575 enum kvm_pgtable_walk_flags visit)
422 enum kvm_pgtable_walk_flags flag,
423 void * const arg)
424{576{
425 struct check_walk_data *d = arg;577 struct check_walk_data *d = ctx->arg;
426 kvm_pte_t pte = *ptep;
427 578
428 if (kvm_pte_valid(pte) && !addr_is_memory(kvm_pte_to_phys(pte)))579 if (kvm_pte_valid(ctx->old) && !addr_is_allowed_memory(kvm_pte_to_phys(ctx->old)))
429 return -EINVAL;580 return -EINVAL;
430 581
431 return d->get_page_state(pte) == d->desired ? 0 : -EPERM;582 return d->get_page_state(ctx->old) == d->desired ? 0 : -EPERM;
432}583}
433 584
434static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,585static int check_page_state_range(struct kvm_pgtable *pgt, u64 addr, u64 size,
@@ -459,8 +610,8 @@ static int __host_check_page_state_range(u64 addr, u64 size, · 24 unmodified lines
459 .get_page_state = host_get_page_state,610 .get_page_state = host_get_page_state,
460 };611 };
461 612
462 hyp_assert_lock_held(&host_kvm.lock);613 hyp_assert_lock_held(&host_mmu.lock);
463 return check_page_state_range(&host_kvm.pgt, addr, size, &d);614 return check_page_state_range(&host_mmu.pgt, addr, size, &d);
464}615}
465 616
466static int __host_set_page_state_range(u64 addr, u64 size,617static int __host_set_page_state_range(u64 addr, u64 size,
@@ -511,6 +662,46 @@ static int host_initiate_unshare(u64 *completer_addr, · 44 unmodified lines
511 return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);662 return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
512}663}
513 664
665static int host_initiate_donation(u64 *completer_addr,
666 const struct pkvm_mem_transition *tx)
667{
668 u8 owner_id = tx->completer.id;
669 u64 size = tx->nr_pages * PAGE_SIZE;
670
671 *completer_addr = tx->initiator.host.completer_addr;
672 return host_stage2_set_owner_locked(tx->initiator.addr, size, owner_id);
673}
674
675static bool __host_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
676{
677 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
678 tx->initiator.id != PKVM_ID_HYP);
679}
680
681static int __host_ack_transition(u64 addr, const struct pkvm_mem_transition *tx,
682 enum pkvm_page_state state)
683{
684 u64 size = tx->nr_pages * PAGE_SIZE;
685
686 if (__host_ack_skip_pgtable_check(tx))
687 return 0;
688
689 return __host_check_page_state_range(addr, size, state);
690}
691
692static int host_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
693{
694 return __host_ack_transition(addr, tx, PKVM_NOPAGE);
695}
696
697static int host_complete_donation(u64 addr, const struct pkvm_mem_transition *tx)
698{
699 u64 size = tx->nr_pages * PAGE_SIZE;
700 u8 host_id = tx->completer.id;
701
702 return host_stage2_set_owner_locked(addr, size, host_id);
703}
704
514static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte)705static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte)
515{706{
516 if (!kvm_pte_valid(pte))707 if (!kvm_pte_valid(pte))
@@ -531,6 +722,27 @@ static int __hyp_check_page_state_range(u64 addr, u64 size, · 14 unmodified lines
531 return check_page_state_range(&pkvm_pgtable, addr, size, &d);722 return check_page_state_range(&pkvm_pgtable, addr, size, &d);
532}723}
533 724
725static int hyp_request_donation(u64 *completer_addr,
726 const struct pkvm_mem_transition *tx)
727{
728 u64 size = tx->nr_pages * PAGE_SIZE;
729 u64 addr = tx->initiator.addr;
730
731 *completer_addr = tx->initiator.hyp.completer_addr;
732 return __hyp_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
733}
734
735static int hyp_initiate_donation(u64 *completer_addr,
736 const struct pkvm_mem_transition *tx)
737{
738 u64 size = tx->nr_pages * PAGE_SIZE;
739 int ret;
740
741 *completer_addr = tx->initiator.hyp.completer_addr;
742 ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, tx->initiator.addr, size);
743 return (ret != size) ? -EFAULT : 0;
744}
745
534static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)746static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
535{747{
536 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||748 return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
@@ -555,6 +767,9 @@ static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 18 unmodified lines
555{767{
556 u64 size = tx->nr_pages * PAGE_SIZE;768 u64 size = tx->nr_pages * PAGE_SIZE;
557 769
770 if (tx->initiator.id == PKVM_ID_HOST && hyp_page_count((void *)addr))
771 return -EBUSY;
772
558 if (__hyp_ack_skip_pgtable_check(tx))773 if (__hyp_ack_skip_pgtable_check(tx))
559 return 0;774 return 0;
560 775
@@ -562,6 +777,16 @@ static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 1 unmodified line
562 PKVM_PAGE_SHARED_BORROWED);777 PKVM_PAGE_SHARED_BORROWED);
563}778}
564 779
780static int hyp_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
781{
782 u64 size = tx->nr_pages * PAGE_SIZE;
783
784 if (__hyp_ack_skip_pgtable_check(tx))
785 return 0;
786
787 return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
788}
789
565static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,790static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
566 enum kvm_pgtable_prot perms)791 enum kvm_pgtable_prot perms)
567{792{
@@ -580,6 +805,15 @@ static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx) · 12 unmodified lines
580 return (ret != size) ? -EFAULT : 0;805 return (ret != size) ? -EFAULT : 0;
581}806}
582 807
808static int hyp_complete_donation(u64 addr,
809 const struct pkvm_mem_transition *tx)
810{
811 void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
812 enum kvm_pgtable_prot prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_OWNED);
813
814 return pkvm_create_mappings_locked(start, end, prot);
815}
816
583static int check_share(struct pkvm_mem_share *share)817static int check_share(struct pkvm_mem_share *share)
584{818{
585 const struct pkvm_mem_transition *tx = &share->tx;819 const struct pkvm_mem_transition *tx = &share->tx;
@@ -732,6 +966,94 @@ static int do_unshare(struct pkvm_mem_share *share) · 146 unmodified lines
732 return WARN_ON(__do_unshare(share));966 return WARN_ON(__do_unshare(share));
733}967}
734 968
969static int check_donation(struct pkvm_mem_donation *donation)
970{
971 const struct pkvm_mem_transition *tx = &donation->tx;
972 u64 completer_addr;
973 int ret;
974
975 switch (tx->initiator.id) {
976 case PKVM_ID_HOST:
977 ret = host_request_owned_transition(&completer_addr, tx);
978 break;
979 case PKVM_ID_HYP:
980 ret = hyp_request_donation(&completer_addr, tx);
981 break;
982 default:
983 ret = -EINVAL;
984 }
985
986 if (ret)
987 return ret;
988
989 switch (tx->completer.id) {
990 case PKVM_ID_HOST:
991 ret = host_ack_donation(completer_addr, tx);
992 break;
993 case PKVM_ID_HYP:
994 ret = hyp_ack_donation(completer_addr, tx);
995 break;
996 default:
997 ret = -EINVAL;
998 }
999
1000 return ret;
1001}
1002
1003static int __do_donate(struct pkvm_mem_donation *donation)
1004{
1005 const struct pkvm_mem_transition *tx = &donation->tx;
1006 u64 completer_addr;
1007 int ret;
1008
1009 switch (tx->initiator.id) {
1010 case PKVM_ID_HOST:
1011 ret = host_initiate_donation(&completer_addr, tx);
1012 break;
1013 case PKVM_ID_HYP:
1014 ret = hyp_initiate_donation(&completer_addr, tx);
1015 break;
1016 default:
1017 ret = -EINVAL;
1018 }
1019
1020 if (ret)
1021 return ret;
1022
1023 switch (tx->completer.id) {
1024 case PKVM_ID_HOST:
1025 ret = host_complete_donation(completer_addr, tx);
1026 break;
1027 case PKVM_ID_HYP:
1028 ret = hyp_complete_donation(completer_addr, tx);
1029 break;
1030 default:
1031 ret = -EINVAL;
1032 }
1033
1034 return ret;
1035}
1036
1037/*
1038 * do_donate():
1039 *
1040 * The page owner transfers ownership to another component, losing access
1041 * as a consequence.
1042 *
1043 * Initiator: OWNED => NOPAGE
1044 * Completer: NOPAGE => OWNED
1045 */
1046static int do_donate(struct pkvm_mem_donation *donation)
1047{
1048 int ret;
1049
1050 ret = check_donation(donation);
1051 if (ret)
1052 return ret;
1053
1054 return WARN_ON(__do_donate(donation));
1055}
1056
735int __pkvm_host_share_hyp(u64 pfn)1057int __pkvm_host_share_hyp(u64 pfn)
736{1058{
737 int ret;1059 int ret;
@@ -797,3 +1119,112 @@ int __pkvm_host_unshare_hyp(u64 pfn) · 59 unmodified lines
797 1119
798 return ret;1120 return ret;
799}1121}
1122
1123int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
1124{
1125 int ret;
1126 u64 host_addr = hyp_pfn_to_phys(pfn);
1127 u64 hyp_addr = (u64)__hyp_va(host_addr);
1128 struct pkvm_mem_donation donation = {
1129 .tx = {
1130 .nr_pages = nr_pages,
1131 .initiator = {
1132 .id = PKVM_ID_HOST,
1133 .addr = host_addr,
1134 .host = {
1135 .completer_addr = hyp_addr,
1136 },
1137 },
1138 .completer = {
1139 .id = PKVM_ID_HYP,
1140 },
1141 },
1142 };
1143
1144 host_lock_component();
1145 hyp_lock_component();
1146
1147 ret = do_donate(&donation);
1148
1149 hyp_unlock_component();
1150 host_unlock_component();
1151
1152 return ret;
1153}
1154
1155int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages)
1156{
1157 int ret;
1158 u64 host_addr = hyp_pfn_to_phys(pfn);
1159 u64 hyp_addr = (u64)__hyp_va(host_addr);
1160 struct pkvm_mem_donation donation = {
1161 .tx = {
1162 .nr_pages = nr_pages,
1163 .initiator = {
1164 .id = PKVM_ID_HYP,
1165 .addr = hyp_addr,
1166 .hyp = {
1167 .completer_addr = host_addr,
1168 },
1169 },
1170 .completer = {
1171 .id = PKVM_ID_HOST,
1172 },
1173 },
1174 };
1175
1176 host_lock_component();
1177 hyp_lock_component();
1178
1179 ret = do_donate(&donation);
1180
1181 hyp_unlock_component();
1182 host_unlock_component();
1183
1184 return ret;
1185}
1186
1187int hyp_pin_shared_mem(void *from, void *to)
1188{
1189 u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1190 u64 end = PAGE_ALIGN((u64)to);
1191 u64 size = end - start;
1192 int ret;
1193
1194 host_lock_component();
1195 hyp_lock_component();
1196
1197 ret = __host_check_page_state_range(__hyp_pa(start), size,
1198 PKVM_PAGE_SHARED_OWNED);
1199 if (ret)
1200 goto unlock;
1201
1202 ret = __hyp_check_page_state_range(start, size,
1203 PKVM_PAGE_SHARED_BORROWED);
1204 if (ret)
1205 goto unlock;
1206
1207 for (cur = start; cur < end; cur += PAGE_SIZE)
1208 hyp_page_ref_inc(hyp_virt_to_page(cur));
1209
1210unlock:
1211 hyp_unlock_component();
1212 host_unlock_component();
1213
1214 return ret;
1215}
1216
1217void hyp_unpin_shared_mem(void *from, void *to)
1218{
1219 u64 cur, start = ALIGN_DOWN((u64)from, PAGE_SIZE);
1220 u64 end = PAGE_ALIGN((u64)to);
1221
1222 host_lock_component();
1223 hyp_lock_component();
1224
1225 for (cur = start; cur < end; cur += PAGE_SIZE)
1226 hyp_page_ref_dec(hyp_virt_to_page(cur));
1227
1228 hyp_unlock_component();
1229 host_unlock_component();
1230}
arch/arm64/kvm/hyp/nvhe/mm.c+163−4
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <nvhe/early_alloc.h>14#include <nvhe/early_alloc.h>
15#include <nvhe/gfp.h>15#include <nvhe/gfp.h>
16#include <nvhe/memory.h>16#include <nvhe/memory.h>
17#include <nvhe/mem_protect.h>
17#include <nvhe/mm.h>18#include <nvhe/mm.h>
18#include <nvhe/spinlock.h>19#include <nvhe/spinlock.h>
19 20
@@ -25,6 +26,12 @@ unsigned int hyp_memblock_nr; · 5 unmodified lines
25 26
26static u64 __io_map_base;27static u64 __io_map_base;
27 28
29struct hyp_fixmap_slot {
30 u64 addr;
31 kvm_pte_t *ptep;
32};
33static DEFINE_PER_CPU(struct hyp_fixmap_slot, fixmap_slots);
34
28static int __pkvm_create_mappings(unsigned long start, unsigned long size,35static int __pkvm_create_mappings(unsigned long start, unsigned long size,
29 unsigned long phys, enum kvm_pgtable_prot prot)36 unsigned long phys, enum kvm_pgtable_prot prot)
30{37{
@@ -129,13 +136,36 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) · 98 unmodified lines
129 return ret;136 return ret;
130}137}
131 138
132int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)139int hyp_back_vmemmap(phys_addr_t back)
133{140{
134 unsigned long start, end;141 unsigned long i, start, size, end = 0;
142 int ret;
135 143
136 hyp_vmemmap_range(phys, size, &start, &end);144 for (i = 0; i < hyp_memblock_nr; i++) {
145 start = hyp_memory[i].base;
146 start = ALIGN_DOWN((u64)hyp_phys_to_page(start), PAGE_SIZE);
147 /*
148 * The begining of the hyp_vmemmap region for the current
149 * memblock may already be backed by the page backing the end
150 * the previous region, so avoid mapping it twice.
151 */
152 start = max(start, end);
153
154 end = hyp_memory[i].base + hyp_memory[i].size;
155 end = PAGE_ALIGN((u64)hyp_phys_to_page(end));
156 if (start >= end)
157 continue;
158
159 size = end - start;
160 ret = __pkvm_create_mappings(start, size, back, PAGE_HYP);
161 if (ret)
162 return ret;
163
164 memset(hyp_phys_to_virt(back), 0, size);
165 back += size;
166 }
137 167
138 return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);168 return 0;
139}169}
140 170
141static void *__hyp_bp_vect_base;171static void *__hyp_bp_vect_base;
@@ -189,6 +219,102 @@ int hyp_map_vectors(void) · 47 unmodified lines
189 return 0;219 return 0;
190}220}
191 221
222void *hyp_fixmap_map(phys_addr_t phys)
223{
224 struct hyp_fixmap_slot *slot = this_cpu_ptr(&fixmap_slots);
225 kvm_pte_t pte, *ptep = slot->ptep;
226
227 pte = *ptep;
228 pte &= ~kvm_phys_to_pte(KVM_PHYS_INVALID);
229 pte |= kvm_phys_to_pte(phys) | KVM_PTE_VALID;
230 WRITE_ONCE(*ptep, pte);
231 dsb(ishst);
232
233 return (void *)slot->addr;
234}
235
236static void fixmap_clear_slot(struct hyp_fixmap_slot *slot)
237{
238 kvm_pte_t *ptep = slot->ptep;
239 u64 addr = slot->addr;
240
241 WRITE_ONCE(*ptep, *ptep & ~KVM_PTE_VALID);
242
243 /*
244 * Irritatingly, the architecture requires that we use inner-shareable
245 * broadcast TLB invalidation here in case another CPU speculates
246 * through our fixmap and decides to create an "amalagamation of the
247 * values held in the TLB" due to the apparent lack of a
248 * break-before-make sequence.
249 *
250 * https://lore.kernel.org/kvm/20221017115209.2099-1-will@kernel.org/T/#mf10dfbaf1eaef9274c581b81c53758918c1d0f03
251 */
252 dsb(ishst);
253 __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), (KVM_PGTABLE_MAX_LEVELS - 1));
254 dsb(ish);
255 isb();
256}
257
258void hyp_fixmap_unmap(void)
259{
260 fixmap_clear_slot(this_cpu_ptr(&fixmap_slots));
261}
262
263static int __create_fixmap_slot_cb(const struct kvm_pgtable_visit_ctx *ctx,
264 enum kvm_pgtable_walk_flags visit)
265{
266 struct hyp_fixmap_slot *slot = per_cpu_ptr(&fixmap_slots, (u64)ctx->arg);
267
268 if (!kvm_pte_valid(ctx->old) || ctx->level != KVM_PGTABLE_MAX_LEVELS - 1)
269 return -EINVAL;
270
271 slot->addr = ctx->addr;
272 slot->ptep = ctx->ptep;
273
274 /*
275 * Clear the PTE, but keep the page-table page refcount elevated to
276 * prevent it from ever being freed. This lets us manipulate the PTEs
277 * by hand safely without ever needing to allocate memory.
278 */
279 fixmap_clear_slot(slot);
280
281 return 0;
282}
283
284static int create_fixmap_slot(u64 addr, u64 cpu)
285{
286 struct kvm_pgtable_walker walker = {
287 .cb = __create_fixmap_slot_cb,
288 .flags = KVM_PGTABLE_WALK_LEAF,
289 .arg = (void *)cpu,
290 };
291
292 return kvm_pgtable_walk(&pkvm_pgtable, addr, PAGE_SIZE, &walker);
293}
294
295int hyp_create_pcpu_fixmap(void)
296{
297 unsigned long addr, i;
298 int ret;
299
300 for (i = 0; i < hyp_nr_cpus; i++) {
301 ret = pkvm_alloc_private_va_range(PAGE_SIZE, &addr);
302 if (ret)
303 return ret;
304
305 ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, PAGE_SIZE,
306 __hyp_pa(__hyp_bss_start), PAGE_HYP);
307 if (ret)
308 return ret;
309
310 ret = create_fixmap_slot(addr, i);
311 if (ret)
312 return ret;
313 }
314
315 return 0;
316}
317
192int hyp_create_idmap(u32 hyp_va_bits)318int hyp_create_idmap(u32 hyp_va_bits)
193{319{
194 unsigned long start, end;320 unsigned long start, end;
@@ -213,3 +339,36 @@ int hyp_create_idmap(u32 hyp_va_bits) · 18 unmodified lines
213 339
214 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);340 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
215}341}
342
343static void *admit_host_page(void *arg)
344{
345 struct kvm_hyp_memcache *host_mc = arg;
346
347 if (!host_mc->nr_pages)
348 return NULL;
349
350 /*
351 * The host still owns the pages in its memcache, so we need to go
352 * through a full host-to-hyp donation cycle to change it. Fortunately,
353 * __pkvm_host_donate_hyp() takes care of races for us, so if it
354 * succeeds we're good to go.
355 */
356 if (__pkvm_host_donate_hyp(hyp_phys_to_pfn(host_mc->head), 1))
357 return NULL;
358
359 return pop_hyp_memcache(host_mc, hyp_phys_to_virt);
360}
361
362/* Refill our local memcache by poping pages from the one provided by the host. */
363int refill_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages,
364 struct kvm_hyp_memcache *host_mc)
365{
366 struct kvm_hyp_memcache tmp = *host_mc;
367 int ret;
368
369 ret = __topup_hyp_memcache(mc, min_pages, admit_host_page,
370 hyp_virt_to_phys, &tmp);
371 *host_mc = tmp;
372
373 return ret;
374}
arch/arm64/kvm/hyp/nvhe/page_alloc.c+7−22
@@ -93,11 +93,16 @@ static inline struct hyp_page *node_to_page(struct list_head *node) · 92 unmodified lines
93static void __hyp_attach_page(struct hyp_pool *pool,93static void __hyp_attach_page(struct hyp_pool *pool,
94 struct hyp_page *p)94 struct hyp_page *p)
95{95{
96 phys_addr_t phys = hyp_page_to_phys(p);
96 unsigned short order = p->order;97 unsigned short order = p->order;
97 struct hyp_page *buddy;98 struct hyp_page *buddy;
98 99
99 memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);100 memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
100 101
102 /* Skip coalescing for 'external' pages being freed into the pool. */
103 if (phys < pool->range_start || phys >= pool->range_end)
104 goto insert;
105
101 /*106 /*
102 * Only the first struct hyp_page of a high-order page (otherwise known107 * Only the first struct hyp_page of a high-order page (otherwise known
103 * as the 'head') should have p->order set. The non-head pages should108 * as the 'head') should have p->order set. The non-head pages should
@@ -116,6 +121,7 @@ static void __hyp_attach_page(struct hyp_pool *pool, · 12 unmodified lines
116 p = min(p, buddy);121 p = min(p, buddy);
117 }122 }
118 123
124insert:
119 /* Mark the new head, and insert it */125 /* Mark the new head, and insert it */
120 p->order = order;126 p->order = order;
121 page_add_to_list(p, &pool->free_area[order]);127 page_add_to_list(p, &pool->free_area[order]);
@@ -144,25 +150,6 @@ static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, · 22 unmodified lines
144 return p;150 return p;
145}151}
146 152
147static inline void hyp_page_ref_inc(struct hyp_page *p)
148{
149 BUG_ON(p->refcount == USHRT_MAX);
150 p->refcount++;
151}
152
153static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
154{
155 BUG_ON(!p->refcount);
156 p->refcount--;
157 return (p->refcount == 0);
158}
159
160static inline void hyp_set_page_refcounted(struct hyp_page *p)
161{
162 BUG_ON(p->refcount);
163 p->refcount = 1;
164}
165
166static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p)153static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p)
167{154{
168 if (hyp_page_ref_dec_and_test(p))155 if (hyp_page_ref_dec_and_test(p))
@@ -249,10 +236,8 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, · 80 unmodified lines
249 236
250 /* Init the vmemmap portion */237 /* Init the vmemmap portion */
251 p = hyp_phys_to_page(phys);238 p = hyp_phys_to_page(phys);
252 for (i = 0; i < nr_pages; i++) {239 for (i = 0; i < nr_pages; i++)
253 p[i].order = 0;
254 hyp_set_page_refcounted(&p[i]);240 hyp_set_page_refcounted(&p[i]);
255 }
256 241
257 /* Attach the unused pages to the buddy tree */242 /* Attach the unused pages to the buddy tree */
258 for (i = reserved_pages; i < nr_pages; i++)243 for (i = reserved_pages; i < nr_pages; i++)
arch/arm64/kvm/hyp/nvhe/pkvm.c+436−0
@@ -7,8 +7,17 @@ · 6 unmodified lines
7#include <linux/kvm_host.h>7#include <linux/kvm_host.h>
8#include <linux/mm.h>8#include <linux/mm.h>
9#include <nvhe/fixed_config.h>9#include <nvhe/fixed_config.h>
10#include <nvhe/mem_protect.h>
11#include <nvhe/memory.h>
12#include <nvhe/pkvm.h>
10#include <nvhe/trap_handler.h>13#include <nvhe/trap_handler.h>
11 14
15/* Used by icache_is_vpipt(). */
16unsigned long __icache_flags;
17
18/* Used by kvm_get_vttbr(). */
19unsigned int kvm_arm_vmid_bits;
20
12/*21/*
13 * Set trap register values based on features in ID_AA64PFR0.22 * Set trap register values based on features in ID_AA64PFR0.
14 */23 */
@@ -183,3 +192,430 @@ void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu) · 168 unmodified lines
183 pvm_init_traps_aa64mmfr0(vcpu);192 pvm_init_traps_aa64mmfr0(vcpu);
184 pvm_init_traps_aa64mmfr1(vcpu);193 pvm_init_traps_aa64mmfr1(vcpu);
185}194}
195
196/*
197 * Start the VM table handle at the offset defined instead of at 0.
198 * Mainly for sanity checking and debugging.
199 */
200#define HANDLE_OFFSET 0x1000
201
202static unsigned int vm_handle_to_idx(pkvm_handle_t handle)
203{
204 return handle - HANDLE_OFFSET;
205}
206
207static pkvm_handle_t idx_to_vm_handle(unsigned int idx)
208{
209 return idx + HANDLE_OFFSET;
210}
211
212/*
213 * Spinlock for protecting state related to the VM table. Protects writes
214 * to 'vm_table' and 'nr_table_entries' as well as reads and writes to
215 * 'last_hyp_vcpu_lookup'.
216 */
217static DEFINE_HYP_SPINLOCK(vm_table_lock);
218
219/*
220 * The table of VM entries for protected VMs in hyp.
221 * Allocated at hyp initialization and setup.
222 */
223static struct pkvm_hyp_vm **vm_table;
224
225void pkvm_hyp_vm_table_init(void *tbl)
226{
227 WARN_ON(vm_table);
228 vm_table = tbl;
229}
230
231/*
232 * Return the hyp vm structure corresponding to the handle.
233 */
234static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle)
235{
236 unsigned int idx = vm_handle_to_idx(handle);
237
238 if (unlikely(idx >= KVM_MAX_PVMS))
239 return NULL;
240
241 return vm_table[idx];
242}
243
244struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
245 unsigned int vcpu_idx)
246{
247 struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
248 struct pkvm_hyp_vm *hyp_vm;
249
250 hyp_spin_lock(&vm_table_lock);
251 hyp_vm = get_vm_by_handle(handle);
252 if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx)
253 goto unlock;
254
255 hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
256 hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
257unlock:
258 hyp_spin_unlock(&vm_table_lock);
259 return hyp_vcpu;
260}
261
262void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
263{
264 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
265
266 hyp_spin_lock(&vm_table_lock);
267 hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
268 hyp_spin_unlock(&vm_table_lock);
269}
270
271static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
272{
273 if (host_vcpu)
274 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1);
275}
276
277static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[],
278 unsigned int nr_vcpus)
279{
280 int i;
281
282 for (i = 0; i < nr_vcpus; i++)
283 unpin_host_vcpu(hyp_vcpus[i]->host_vcpu);
284}
285
286static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
287 unsigned int nr_vcpus)
288{
289 hyp_vm->host_kvm = host_kvm;
290 hyp_vm->kvm.created_vcpus = nr_vcpus;
291 hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr;
292}
293
294static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
295 struct pkvm_hyp_vm *hyp_vm,
296 struct kvm_vcpu *host_vcpu,
297 unsigned int vcpu_idx)
298{
299 int ret = 0;
300
301 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
302 return -EBUSY;
303
304 if (host_vcpu->vcpu_idx != vcpu_idx) {
305 ret = -EINVAL;
306 goto done;
307 }
308
309 hyp_vcpu->host_vcpu = host_vcpu;
310
311 hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
312 hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id);
313 hyp_vcpu->vcpu.vcpu_idx = vcpu_idx;
314
315 hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
316 hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
317done:
318 if (ret)
319 unpin_host_vcpu(host_vcpu);
320 return ret;
321}
322
323static int find_free_vm_table_entry(struct kvm *host_kvm)
324{
325 int i;
326
327 for (i = 0; i < KVM_MAX_PVMS; ++i) {
328 if (!vm_table[i])
329 return i;
330 }
331
332 return -ENOMEM;
333}
334
335/*
336 * Allocate a VM table entry and insert a pointer to the new vm.
337 *
338 * Return a unique handle to the protected VM on success,
339 * negative error code on failure.
340 */
341static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm,
342 struct pkvm_hyp_vm *hyp_vm)
343{
344 struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu;
345 int idx;
346
347 hyp_assert_lock_held(&vm_table_lock);
348
349 /*
350 * Initializing protected state might have failed, yet a malicious
351 * host could trigger this function. Thus, ensure that 'vm_table'
352 * exists.
353 */
354 if (unlikely(!vm_table))
355 return -EINVAL;
356
357 idx = find_free_vm_table_entry(host_kvm);
358 if (idx < 0)
359 return idx;
360
361 hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx);
362
363 /* VMID 0 is reserved for the host */
364 atomic64_set(&mmu->vmid.id, idx + 1);
365
366 mmu->arch = &hyp_vm->kvm.arch;
367 mmu->pgt = &hyp_vm->pgt;
368
369 vm_table[idx] = hyp_vm;
370 return hyp_vm->kvm.arch.pkvm.handle;
371}
372
373/*
374 * Deallocate and remove the VM table entry corresponding to the handle.
375 */
376static void remove_vm_table_entry(pkvm_handle_t handle)
377{
378 hyp_assert_lock_held(&vm_table_lock);
379 vm_table[vm_handle_to_idx(handle)] = NULL;
380}
381
382static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
383{
384 return size_add(sizeof(struct pkvm_hyp_vm),
385 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
386}
387
388static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
389{
390 void *va = (void *)kern_hyp_va(host_va);
391
392 if (!PAGE_ALIGNED(va))
393 return NULL;
394
395 if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va),
396 PAGE_ALIGN(size) >> PAGE_SHIFT))
397 return NULL;
398
399 return va;
400}
401
402static void *map_donated_memory(unsigned long host_va, size_t size)
403{
404 void *va = map_donated_memory_noclear(host_va, size);
405
406 if (va)
407 memset(va, 0, size);
408
409 return va;
410}
411
412static void __unmap_donated_memory(void *va, size_t size)
413{
414 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va),
415 PAGE_ALIGN(size) >> PAGE_SHIFT));
416}
417
418static void unmap_donated_memory(void *va, size_t size)
419{
420 if (!va)
421 return;
422
423 memset(va, 0, size);
424 __unmap_donated_memory(va, size);
425}
426
427static void unmap_donated_memory_noclear(void *va, size_t size)
428{
429 if (!va)
430 return;
431
432 __unmap_donated_memory(va, size);
433}
434
435/*
436 * Initialize the hypervisor copy of the protected VM state using the
437 * memory donated by the host.
438 *
439 * Unmaps the donated memory from the host at stage 2.
440 *
441 * host_kvm: A pointer to the host's struct kvm.
442 * vm_hva: The host va of the area being donated for the VM state.
443 * Must be page aligned.
444 * pgd_hva: The host va of the area being donated for the stage-2 PGD for
445 * the VM. Must be page aligned. Its size is implied by the VM's
446 * VTCR.
447 *
448 * Return a unique handle to the protected VM on success,
449 * negative error code on failure.
450 */
451int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
452 unsigned long pgd_hva)
453{
454 struct pkvm_hyp_vm *hyp_vm = NULL;
455 size_t vm_size, pgd_size;
456 unsigned int nr_vcpus;
457 void *pgd = NULL;
458 int ret;
459
460 ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1);
461 if (ret)
462 return ret;
463
464 nr_vcpus = READ_ONCE(host_kvm->created_vcpus);
465 if (nr_vcpus < 1) {
466 ret = -EINVAL;
467 goto err_unpin_kvm;
468 }
469
470 vm_size = pkvm_get_hyp_vm_size(nr_vcpus);
471 pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr);
472
473 ret = -ENOMEM;
474
475 hyp_vm = map_donated_memory(vm_hva, vm_size);
476 if (!hyp_vm)
477 goto err_remove_mappings;
478
479 pgd = map_donated_memory_noclear(pgd_hva, pgd_size);
480 if (!pgd)
481 goto err_remove_mappings;
482
483 init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus);
484
485 hyp_spin_lock(&vm_table_lock);
486 ret = insert_vm_table_entry(host_kvm, hyp_vm);
487 if (ret < 0)
488 goto err_unlock;
489
490 ret = kvm_guest_prepare_stage2(hyp_vm, pgd);
491 if (ret)
492 goto err_remove_vm_table_entry;
493 hyp_spin_unlock(&vm_table_lock);
494
495 return hyp_vm->kvm.arch.pkvm.handle;
496
497err_remove_vm_table_entry:
498 remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle);
499err_unlock:
500 hyp_spin_unlock(&vm_table_lock);
501err_remove_mappings:
502 unmap_donated_memory(hyp_vm, vm_size);
503 unmap_donated_memory(pgd, pgd_size);
504err_unpin_kvm:
505 hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
506 return ret;
507}
508
509/*
510 * Initialize the hypervisor copy of the protected vCPU state using the
511 * memory donated by the host.
512 *
513 * handle: The handle for the protected vm.
514 * host_vcpu: A pointer to the corresponding host vcpu.
515 * vcpu_hva: The host va of the area being donated for the vcpu state.
516 * Must be page aligned. The size of the area must be equal to
517 * the page-aligned size of 'struct pkvm_hyp_vcpu'.
518 * Return 0 on success, negative error code on failure.
519 */
520int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
521 unsigned long vcpu_hva)
522{
523 struct pkvm_hyp_vcpu *hyp_vcpu;
524 struct pkvm_hyp_vm *hyp_vm;
525 unsigned int idx;
526 int ret;
527
528 hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
529 if (!hyp_vcpu)
530 return -ENOMEM;
531
532 hyp_spin_lock(&vm_table_lock);
533
534 hyp_vm = get_vm_by_handle(handle);
535 if (!hyp_vm) {
536 ret = -ENOENT;
537 goto unlock;
538 }
539
540 idx = hyp_vm->nr_vcpus;
541 if (idx >= hyp_vm->kvm.created_vcpus) {
542 ret = -EINVAL;
543 goto unlock;
544 }
545
546 ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx);
547 if (ret)
548 goto unlock;
549
550 hyp_vm->vcpus[idx] = hyp_vcpu;
551 hyp_vm->nr_vcpus++;
552unlock:
553 hyp_spin_unlock(&vm_table_lock);
554
555 if (ret)
556 unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
557
558 return ret;
559}
560
561static void
562teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size)
563{
564 size = PAGE_ALIGN(size);
565 memset(addr, 0, size);
566
567 for (void *start = addr; start < addr + size; start += PAGE_SIZE)
568 push_hyp_memcache(mc, start, hyp_virt_to_phys);
569
570 unmap_donated_memory_noclear(addr, size);
571}
572
573int __pkvm_teardown_vm(pkvm_handle_t handle)
574{
575 struct kvm_hyp_memcache *mc;
576 struct pkvm_hyp_vm *hyp_vm;
577 struct kvm *host_kvm;
578 unsigned int idx;
579 size_t vm_size;
580 int err;
581
582 hyp_spin_lock(&vm_table_lock);
583 hyp_vm = get_vm_by_handle(handle);
584 if (!hyp_vm) {
585 err = -ENOENT;
586 goto err_unlock;
587 }
588
589 if (WARN_ON(hyp_page_count(hyp_vm))) {
590 err = -EBUSY;
591 goto err_unlock;
592 }
593
594 host_kvm = hyp_vm->host_kvm;
595
596 /* Ensure the VMID is clean before it can be reallocated */
597 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu);
598 remove_vm_table_entry(handle);
599 hyp_spin_unlock(&vm_table_lock);
600
601 /* Reclaim guest pages (including page-table pages) */
602 mc = &host_kvm->arch.pkvm.teardown_mc;
603 reclaim_guest_pages(hyp_vm, mc);
604 unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus);
605
606 /* Push the metadata pages to the teardown memcache */
607 for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) {
608 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
609
610 teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu));
611 }
612
613 vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
614 teardown_donated_memory(mc, hyp_vm, vm_size);
615 hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
616 return 0;
617
618err_unlock:
619 hyp_spin_unlock(&vm_table_lock);
620 return err;
621}
arch/arm64/kvm/hyp/nvhe/setup.c+61−37
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <nvhe/memory.h>16#include <nvhe/memory.h>
17#include <nvhe/mem_protect.h>17#include <nvhe/mem_protect.h>
18#include <nvhe/mm.h>18#include <nvhe/mm.h>
19#include <nvhe/pkvm.h>
19#include <nvhe/trap_handler.h>20#include <nvhe/trap_handler.h>
20 21
21unsigned long hyp_nr_cpus;22unsigned long hyp_nr_cpus;
@@ -24,6 +25,7 @@ unsigned long hyp_nr_cpus; · 2 unmodified lines
24 (unsigned long)__per_cpu_start)25 (unsigned long)__per_cpu_start)
25 26
26static void *vmemmap_base;27static void *vmemmap_base;
28static void *vm_table_base;
27static void *hyp_pgt_base;29static void *hyp_pgt_base;
28static void *host_s2_pgt_base;30static void *host_s2_pgt_base;
29static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;31static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
@@ -31,16 +33,20 @@ static struct hyp_pool hpool; · 1 unmodified line
31 33
32static int divide_memory_pool(void *virt, unsigned long size)34static int divide_memory_pool(void *virt, unsigned long size)
33{35{
34 unsigned long vstart, vend, nr_pages;36 unsigned long nr_pages;
35 37
36 hyp_early_alloc_init(virt, size);38 hyp_early_alloc_init(virt, size);
37 39
38 hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend);40 nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page));
39 nr_pages = (vend - vstart) >> PAGE_SHIFT;
40 vmemmap_base = hyp_early_alloc_contig(nr_pages);41 vmemmap_base = hyp_early_alloc_contig(nr_pages);
41 if (!vmemmap_base)42 if (!vmemmap_base)
42 return -ENOMEM;43 return -ENOMEM;
43 44
45 nr_pages = hyp_vm_table_pages();
46 vm_table_base = hyp_early_alloc_contig(nr_pages);
47 if (!vm_table_base)
48 return -ENOMEM;
49
44 nr_pages = hyp_s1_pgtable_pages();50 nr_pages = hyp_s1_pgtable_pages();
45 hyp_pgt_base = hyp_early_alloc_contig(nr_pages);51 hyp_pgt_base = hyp_early_alloc_contig(nr_pages);
46 if (!hyp_pgt_base)52 if (!hyp_pgt_base)
@@ -78,7 +84,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, · 31 unmodified lines
78 if (ret)84 if (ret)
79 return ret;85 return ret;
80 86
81 ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base));87 ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base));
82 if (ret)88 if (ret)
83 return ret;89 return ret;
84 90
@@ -138,20 +144,17 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, · 53 unmodified lines
138 }144 }
139 145
140 /*146 /*
141 * Map the host's .bss and .rodata sections RO in the hypervisor, but147 * Map the host sections RO in the hypervisor, but transfer the
142 * transfer the ownership from the host to the hypervisor itself to148 * ownership from the host to the hypervisor itself to make sure they
143 * make sure it can't be donated or shared with another entity.149 * can't be donated or shared with another entity.
144 *150 *
145 * The ownership transition requires matching changes in the host151 * The ownership transition requires matching changes in the host
146 * stage-2. This will be done later (see finalize_host_mappings()) once152 * stage-2. This will be done later (see finalize_host_mappings()) once
147 * the hyp_vmemmap is addressable.153 * the hyp_vmemmap is addressable.
148 */154 */
149 prot = pkvm_mkstate(PAGE_HYP_RO, PKVM_PAGE_SHARED_OWNED);155 prot = pkvm_mkstate(PAGE_HYP_RO, PKVM_PAGE_SHARED_OWNED);
150 ret = pkvm_create_mappings(__start_rodata, __end_rodata, prot);156 ret = pkvm_create_mappings(&kvm_vgic_global_state,
151 if (ret)157 &kvm_vgic_global_state + 1, prot);
152 return ret;
153
154 ret = pkvm_create_mappings(__hyp_bss_end, __bss_stop, prot);
155 if (ret)158 if (ret)
156 return ret;159 return ret;
157 160
@@ -186,33 +189,20 @@ static void hpool_put_page(void *addr) · 28 unmodified lines
186 hyp_put_page(&hpool, addr);189 hyp_put_page(&hpool, addr);
187}190}
188 191
189static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level,192static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
190 kvm_pte_t *ptep,193 enum kvm_pgtable_walk_flags visit)
191 enum kvm_pgtable_walk_flags flag,
192 void * const arg)
193{194{
194 struct kvm_pgtable_mm_ops *mm_ops = arg;
195 enum kvm_pgtable_prot prot;195 enum kvm_pgtable_prot prot;
196 enum pkvm_page_state state;196 enum pkvm_page_state state;
197 kvm_pte_t pte = *ptep;
198 phys_addr_t phys;197 phys_addr_t phys;
199 198
200 if (!kvm_pte_valid(pte))199 if (!kvm_pte_valid(ctx->old))
201 return 0;200 return 0;
202 201
203 /*202 if (ctx->level != (KVM_PGTABLE_MAX_LEVELS - 1))
204 * Fix-up the refcount for the page-table pages as the early allocator
205 * was unable to access the hyp_vmemmap and so the buddy allocator has
206 * initialised the refcount to '1'.
207 */
208 mm_ops->get_page(ptep);
209 if (flag != KVM_PGTABLE_WALK_LEAF)
210 return 0;
211
212 if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
213 return -EINVAL;203 return -EINVAL;
214 204
215 phys = kvm_pte_to_phys(pte);205 phys = kvm_pte_to_phys(ctx->old);
216 if (!addr_is_memory(phys))206 if (!addr_is_memory(phys))
217 return -EINVAL;207 return -EINVAL;
218 208
@@ -220,10 +210,10 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 1 unmodified line
220 * Adjust the host stage-2 mappings to match the ownership attributes210 * Adjust the host stage-2 mappings to match the ownership attributes
221 * configured in the hypervisor stage-1.211 * configured in the hypervisor stage-1.
222 */212 */
223 state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(pte));213 state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(ctx->old));
224 switch (state) {214 switch (state) {
225 case PKVM_PAGE_OWNED:215 case PKVM_PAGE_OWNED:
226 return host_stage2_set_owner_locked(phys, PAGE_SIZE, pkvm_hyp_id);216 return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP);
227 case PKVM_PAGE_SHARED_OWNED:217 case PKVM_PAGE_SHARED_OWNED:
228 prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_BORROWED);218 prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_BORROWED);
229 break;219 break;
@@ -237,12 +227,25 @@ static int finalize_host_mappings_walker(u64 addr, u64 end, u32 level, · 7 unmodified lines
237 return host_stage2_idmap_locked(phys, PAGE_SIZE, prot);227 return host_stage2_idmap_locked(phys, PAGE_SIZE, prot);
238}228}
239 229
240static int finalize_host_mappings(void)230static int fix_hyp_pgtable_refcnt_walker(const struct kvm_pgtable_visit_ctx *ctx,
231 enum kvm_pgtable_walk_flags visit)
232{
233 /*
234 * Fix-up the refcount for the page-table pages as the early allocator
235 * was unable to access the hyp_vmemmap and so the buddy allocator has
236 * initialised the refcount to '1'.
237 */
238 if (kvm_pte_valid(ctx->old))
239 ctx->mm_ops->get_page(ctx->ptep);
240
241 return 0;
242}
243
244static int fix_host_ownership(void)
241{245{
242 struct kvm_pgtable_walker walker = {246 struct kvm_pgtable_walker walker = {
243 .cb = finalize_host_mappings_walker,247 .cb = fix_host_ownership_walker,
244 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,248 .flags = KVM_PGTABLE_WALK_LEAF,
245 .arg = pkvm_pgtable.mm_ops,
246 };249 };
247 int i, ret;250 int i, ret;
248 251
@@ -258,6 +261,18 @@ static int finalize_host_mappings(void) · 9 unmodified lines
258 return 0;261 return 0;
259}262}
260 263
264static int fix_hyp_pgtable_refcnt(void)
265{
266 struct kvm_pgtable_walker walker = {
267 .cb = fix_hyp_pgtable_refcnt_walker,
268 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
269 .arg = pkvm_pgtable.mm_ops,
270 };
271
272 return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits),
273 &walker);
274}
275
261void __noreturn __pkvm_init_finalise(void)276void __noreturn __pkvm_init_finalise(void)
262{277{
263 struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data);278 struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data);
@@ -287,10 +302,19 @@ void __noreturn __pkvm_init_finalise(void) · 23 unmodified lines
287 };302 };
288 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;303 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
289 304
290 ret = finalize_host_mappings();305 ret = fix_host_ownership();
306 if (ret)
307 goto out;
308
309 ret = fix_hyp_pgtable_refcnt();
310 if (ret)
311 goto out;
312
313 ret = hyp_create_pcpu_fixmap();
291 if (ret)314 if (ret)
292 goto out;315 goto out;
293 316
317 pkvm_hyp_vm_table_init(vm_table_base);
294out:318out:
295 /*319 /*
296 * We tail-called to here from handle___pkvm_init() and will not return,320 * We tail-called to here from handle___pkvm_init() and will not return,
arch/arm64/kvm/hyp/pgtable.c+343−309
@@ -49,35 +49,38 @@ · 48 unmodified lines
49#define KVM_INVALID_PTE_OWNER_MASK GENMASK(9, 2)49#define KVM_INVALID_PTE_OWNER_MASK GENMASK(9, 2)
50#define KVM_MAX_OWNER_ID 150#define KVM_MAX_OWNER_ID 1
51 51
52/*
53 * Used to indicate a pte for which a 'break-before-make' sequence is in
54 * progress.
55 */
56#define KVM_INVALID_PTE_LOCKED BIT(10)
57
52struct kvm_pgtable_walk_data {58struct kvm_pgtable_walk_data {
53 struct kvm_pgtable *pgt;
54 struct kvm_pgtable_walker *walker;59 struct kvm_pgtable_walker *walker;
55 60
56 u64 addr;61 u64 addr;
57 u64 end;62 u64 end;
58};63};
59 64
60#define KVM_PHYS_INVALID (-1ULL)
61
62static bool kvm_phys_is_valid(u64 phys)65static bool kvm_phys_is_valid(u64 phys)
63{66{
64 return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX));67 return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX));
65}68}
66 69
67static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)70static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys)
68{71{
69 u64 granule = kvm_granule_size(level);72 u64 granule = kvm_granule_size(ctx->level);
70 73
71 if (!kvm_level_supports_block_mapping(level))74 if (!kvm_level_supports_block_mapping(ctx->level))
72 return false;75 return false;
73 76
74 if (granule > (end - addr))77 if (granule > (ctx->end - ctx->addr))
75 return false;78 return false;
76 79
77 if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule))80 if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule))
78 return false;81 return false;
79 82
80 return IS_ALIGNED(addr, granule);83 return IS_ALIGNED(ctx->addr, granule);
81}84}
82 85
83static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)86static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)
@@ -88,7 +91,7 @@ static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) · 4 unmodified lines
88 return (data->addr >> shift) & mask;91 return (data->addr >> shift) & mask;
89}92}
90 93
91static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)94static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)
92{95{
93 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */96 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */
94 u64 mask = BIT(pgt->ia_bits) - 1;97 u64 mask = BIT(pgt->ia_bits) - 1;
@@ -96,11 +99,6 @@ static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) · 1 unmodified line
96 return (addr & mask) >> shift;99 return (addr & mask) >> shift;
97}100}
98 101
99static u32 kvm_pgd_page_idx(struct kvm_pgtable_walk_data *data)
100{
101 return __kvm_pgd_page_idx(data->pgt, data->addr);
102}
103
104static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level)102static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level)
105{103{
106 struct kvm_pgtable pgt = {104 struct kvm_pgtable pgt = {
@@ -108,7 +106,7 @@ static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) · 1 unmodified line
108 .start_level = start_level,106 .start_level = start_level,
109 };107 };
110 108
111 return __kvm_pgd_page_idx(&pgt, -1ULL) + 1;109 return kvm_pgd_page_idx(&pgt, -1ULL) + 1;
112}110}
113 111
114static bool kvm_pte_table(kvm_pte_t pte, u32 level)112static bool kvm_pte_table(kvm_pte_t pte, u32 level)
@@ -122,16 +120,6 @@ static bool kvm_pte_table(kvm_pte_t pte, u32 level) · 7 unmodified lines
122 return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE;120 return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE;
123}121}
124 122
125static kvm_pte_t kvm_phys_to_pte(u64 pa)
126{
127 kvm_pte_t pte = pa & KVM_PTE_ADDR_MASK;
128
129 if (PAGE_SHIFT == 16)
130 pte |= FIELD_PREP(KVM_PTE_ADDR_51_48, pa >> 48);
131
132 return pte;
133}
134
135static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)123static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
136{124{
137 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));125 return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
@@ -142,16 +130,13 @@ static void kvm_clear_pte(kvm_pte_t *ptep) · 4 unmodified lines
142 WRITE_ONCE(*ptep, 0);130 WRITE_ONCE(*ptep, 0);
143}131}
144 132
145static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,133static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
146 struct kvm_pgtable_mm_ops *mm_ops)
147{134{
148 kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));135 kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
149 136
150 pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);137 pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);
151 pte |= KVM_PTE_VALID;138 pte |= KVM_PTE_VALID;
152 139 return pte;
153 WARN_ON(kvm_pte_valid(old));
154 smp_store_release(ptep, pte);
155}140}
156 141
157static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)142static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
@@ -172,36 +157,47 @@ static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id) · 14 unmodified lines
172 return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);157 return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
173}158}
174 159
175static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,160static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data,
176 u32 level, kvm_pte_t *ptep,161 const struct kvm_pgtable_visit_ctx *ctx,
177 enum kvm_pgtable_walk_flags flag)162 enum kvm_pgtable_walk_flags visit)
178{163{
179 struct kvm_pgtable_walker *walker = data->walker;164 struct kvm_pgtable_walker *walker = data->walker;
180 return walker->cb(addr, data->end, level, ptep, flag, walker->arg);165
166 /* Ensure the appropriate lock is held (e.g. RCU lock for stage-2 MMU) */
167 WARN_ON_ONCE(kvm_pgtable_walk_shared(ctx) && !kvm_pgtable_walk_lock_held());
168 return walker->cb(ctx, visit);
181}169}
182 170
183static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,171static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
184 kvm_pte_t *pgtable, u32 level);172 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level);
185 173
186static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,174static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
187 kvm_pte_t *ptep, u32 level)175 struct kvm_pgtable_mm_ops *mm_ops,
176 kvm_pteref_t pteref, u32 level)
188{177{
189 int ret = 0;
190 u64 addr = data->addr;
191 kvm_pte_t *childp, pte = *ptep;
192 bool table = kvm_pte_table(pte, level);
193 enum kvm_pgtable_walk_flags flags = data->walker->flags;178 enum kvm_pgtable_walk_flags flags = data->walker->flags;
179 kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);
180 struct kvm_pgtable_visit_ctx ctx = {
181 .ptep = ptep,
182 .old = READ_ONCE(*ptep),
183 .arg = data->walker->arg,
184 .mm_ops = mm_ops,
185 .addr = data->addr,
186 .end = data->end,
187 .level = level,
188 .flags = flags,
189 };
190 int ret = 0;
191 kvm_pteref_t childp;
192 bool table = kvm_pte_table(ctx.old, level);
194 193
195 if (table && (flags & KVM_PGTABLE_WALK_TABLE_PRE)) {194 if (table && (ctx.flags & KVM_PGTABLE_WALK_TABLE_PRE))
196 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,195 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_PRE);
197 KVM_PGTABLE_WALK_TABLE_PRE);
198 }
199 196
200 if (!table && (flags & KVM_PGTABLE_WALK_LEAF)) {197 if (!table && (ctx.flags & KVM_PGTABLE_WALK_LEAF)) {
201 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,198 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_LEAF);
202 KVM_PGTABLE_WALK_LEAF);199 ctx.old = READ_ONCE(*ptep);
203 pte = *ptep;200 table = kvm_pte_table(ctx.old, level);
204 table = kvm_pte_table(pte, level);
205 }201 }
206 202
207 if (ret)203 if (ret)
@@ -213,22 +209,20 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, · 5 unmodified lines
213 goto out;209 goto out;
214 }210 }
215 211
216 childp = kvm_pte_follow(pte, data->pgt->mm_ops);212 childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
217 ret = __kvm_pgtable_walk(data, childp, level + 1);213 ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
218 if (ret)214 if (ret)
219 goto out;215 goto out;
220 216
221 if (flags & KVM_PGTABLE_WALK_TABLE_POST) {217 if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
222 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep,218 ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST);
223 KVM_PGTABLE_WALK_TABLE_POST);
224 }
225 219
226out:220out:
227 return ret;221 return ret;
228}222}
229 223
230static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,224static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
231 kvm_pte_t *pgtable, u32 level)225 struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, u32 level)
232{226{
233 u32 idx;227 u32 idx;
234 int ret = 0;228 int ret = 0;
@@ -237,12 +231,12 @@ static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, · 2 unmodified lines
237 return -EINVAL;231 return -EINVAL;
238 232
239 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {233 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {
240 kvm_pte_t *ptep = &pgtable[idx];234 kvm_pteref_t pteref = &pgtable[idx];
241 235
242 if (data->addr >= data->end)236 if (data->addr >= data->end)
243 break;237 break;
244 238
245 ret = __kvm_pgtable_visit(data, ptep, level);239 ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
246 if (ret)240 if (ret)
247 break;241 break;
248 }242 }
@@ -250,11 +244,10 @@ static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, · 1 unmodified line
250 return ret;244 return ret;
251}245}
252 246
253static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data)247static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)
254{248{
255 u32 idx;249 u32 idx;
256 int ret = 0;250 int ret = 0;
257 struct kvm_pgtable *pgt = data->pgt;
258 u64 limit = BIT(pgt->ia_bits);251 u64 limit = BIT(pgt->ia_bits);
259 252
260 if (data->addr > limit || data->end > limit)253 if (data->addr > limit || data->end > limit)
@@ -263,10 +256,10 @@ static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data) · 2 unmodified lines
263 if (!pgt->pgd)256 if (!pgt->pgd)
264 return -EINVAL;257 return -EINVAL;
265 258
266 for (idx = kvm_pgd_page_idx(data); data->addr < data->end; ++idx) {259 for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {
267 kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE];260 kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];
268 261
269 ret = __kvm_pgtable_walk(data, ptep, pgt->start_level);262 ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
270 if (ret)263 if (ret)
271 break;264 break;
272 }265 }
@@ -278,13 +271,20 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, · 5 unmodified lines
278 struct kvm_pgtable_walker *walker)271 struct kvm_pgtable_walker *walker)
279{272{
280 struct kvm_pgtable_walk_data walk_data = {273 struct kvm_pgtable_walk_data walk_data = {
281 .pgt = pgt,
282 .addr = ALIGN_DOWN(addr, PAGE_SIZE),274 .addr = ALIGN_DOWN(addr, PAGE_SIZE),
283 .end = PAGE_ALIGN(walk_data.addr + size),275 .end = PAGE_ALIGN(walk_data.addr + size),
284 .walker = walker,276 .walker = walker,
285 };277 };
278 int r;
286 279
287 return _kvm_pgtable_walk(&walk_data);280 r = kvm_pgtable_walk_begin(walker);
281 if (r)
282 return r;
283
284 r = _kvm_pgtable_walk(pgt, &walk_data);
285 kvm_pgtable_walk_end(walker);
286
287 return r;
288}288}
289 289
290struct leaf_walk_data {290struct leaf_walk_data {
@@ -292,13 +292,13 @@ struct leaf_walk_data { · 1 unmodified line
292 u32 level;292 u32 level;
293};293};
294 294
295static int leaf_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,295static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
296 enum kvm_pgtable_walk_flags flag, void * const arg)296 enum kvm_pgtable_walk_flags visit)
297{297{
298 struct leaf_walk_data *data = arg;298 struct leaf_walk_data *data = ctx->arg;
299 299
300 data->pte = *ptep;300 data->pte = ctx->old;
301 data->level = level;301 data->level = ctx->level;
302 302
303 return 0;303 return 0;
304}304}
@@ -329,7 +329,6 @@ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, · 24 unmodified lines
329struct hyp_map_data {329struct hyp_map_data {
330 u64 phys;330 u64 phys;
331 kvm_pte_t attr;331 kvm_pte_t attr;
332 struct kvm_pgtable_mm_ops *mm_ops;
333};332};
334 333
335static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)334static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
@@ -383,47 +382,49 @@ enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte) · 47 unmodified lines
383 return prot;382 return prot;
384}383}
385 384
386static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,385static bool hyp_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
387 kvm_pte_t *ptep, struct hyp_map_data *data)386 struct hyp_map_data *data)
388{387{
389 kvm_pte_t new, old = *ptep;388 kvm_pte_t new;
390 u64 granule = kvm_granule_size(level), phys = data->phys;389 u64 granule = kvm_granule_size(ctx->level), phys = data->phys;
391 390
392 if (!kvm_block_mapping_supported(addr, end, phys, level))391 if (!kvm_block_mapping_supported(ctx, phys))
393 return false;392 return false;
394 393
395 data->phys += granule;394 data->phys += granule;
396 new = kvm_init_valid_leaf_pte(phys, data->attr, level);395 new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
397 if (old == new)396 if (ctx->old == new)
398 return true;397 return true;
399 if (!kvm_pte_valid(old))398 if (!kvm_pte_valid(ctx->old))
400 data->mm_ops->get_page(ptep);399 ctx->mm_ops->get_page(ctx->ptep);
401 else if (WARN_ON((old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))400 else if (WARN_ON((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
402 return false;401 return false;
403 402
404 smp_store_release(ptep, new);403 smp_store_release(ctx->ptep, new);
405 return true;404 return true;
406}405}
407 406
408static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,407static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
409 enum kvm_pgtable_walk_flags flag, void * const arg)408 enum kvm_pgtable_walk_flags visit)
410{409{
411 kvm_pte_t *childp;410 kvm_pte_t *childp, new;
412 struct hyp_map_data *data = arg;411 struct hyp_map_data *data = ctx->arg;
413 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;412 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
414 413
415 if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))414 if (hyp_map_walker_try_leaf(ctx, data))
416 return 0;415 return 0;
417 416
418 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))417 if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1))
419 return -EINVAL;418 return -EINVAL;
420 419
421 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);420 childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
422 if (!childp)421 if (!childp)
423 return -ENOMEM;422 return -ENOMEM;
424 423
425 kvm_set_table_pte(ptep, childp, mm_ops);424 new = kvm_init_table_pte(childp, mm_ops);
426 mm_ops->get_page(ptep);425 mm_ops->get_page(ctx->ptep);
426 smp_store_release(ctx->ptep, new);
427
427 return 0;428 return 0;
428}429}
429 430
@@ -433,7 +434,6 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, · 3 unmodified lines
433 int ret;434 int ret;
434 struct hyp_map_data map_data = {435 struct hyp_map_data map_data = {
435 .phys = ALIGN_DOWN(phys, PAGE_SIZE),436 .phys = ALIGN_DOWN(phys, PAGE_SIZE),
436 .mm_ops = pgt->mm_ops,
437 };437 };
438 struct kvm_pgtable_walker walker = {438 struct kvm_pgtable_walker walker = {
439 .cb = hyp_map_walker,439 .cb = hyp_map_walker,
@@ -451,44 +451,39 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, · 11 unmodified lines
451 return ret;451 return ret;
452}452}
453 453
454struct hyp_unmap_data {454static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
455 u64 unmapped;455 enum kvm_pgtable_walk_flags visit)
456 struct kvm_pgtable_mm_ops *mm_ops;
457};
458
459static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
460 enum kvm_pgtable_walk_flags flag, void * const arg)
461{456{
462 kvm_pte_t pte = *ptep, *childp = NULL;457 kvm_pte_t *childp = NULL;
463 u64 granule = kvm_granule_size(level);458 u64 granule = kvm_granule_size(ctx->level);
464 struct hyp_unmap_data *data = arg;459 u64 *unmapped = ctx->arg;
465 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;460 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
466 461
467 if (!kvm_pte_valid(pte))462 if (!kvm_pte_valid(ctx->old))
468 return -EINVAL;463 return -EINVAL;
469 464
470 if (kvm_pte_table(pte, level)) {465 if (kvm_pte_table(ctx->old, ctx->level)) {
471 childp = kvm_pte_follow(pte, mm_ops);466 childp = kvm_pte_follow(ctx->old, mm_ops);
472 467
473 if (mm_ops->page_count(childp) != 1)468 if (mm_ops->page_count(childp) != 1)
474 return 0;469 return 0;
475 470
476 kvm_clear_pte(ptep);471 kvm_clear_pte(ctx->ptep);
477 dsb(ishst);472 dsb(ishst);
478 __tlbi_level(vae2is, __TLBI_VADDR(addr, 0), level);473 __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
479 } else {474 } else {
480 if (end - addr < granule)475 if (ctx->end - ctx->addr < granule)
481 return -EINVAL;476 return -EINVAL;
482 477
483 kvm_clear_pte(ptep);478 kvm_clear_pte(ctx->ptep);
484 dsb(ishst);479 dsb(ishst);
485 __tlbi_level(vale2is, __TLBI_VADDR(addr, 0), level);480 __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
486 data->unmapped += granule;481 *unmapped += granule;
487 }482 }
488 483
489 dsb(ish);484 dsb(ish);
490 isb();485 isb();
491 mm_ops->put_page(ptep);486 mm_ops->put_page(ctx->ptep);
492 487
493 if (childp)488 if (childp)
494 mm_ops->put_page(childp);489 mm_ops->put_page(childp);
@@ -498,12 +493,10 @@ static int hyp_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 3 unmodified lines
498 493
499u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)494u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
500{495{
501 struct hyp_unmap_data unmap_data = {496 u64 unmapped = 0;
502 .mm_ops = pgt->mm_ops,
503 };
504 struct kvm_pgtable_walker walker = {497 struct kvm_pgtable_walker walker = {
505 .cb = hyp_unmap_walker,498 .cb = hyp_unmap_walker,
506 .arg = &unmap_data,499 .arg = &unmapped,
507 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,500 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
508 };501 };
509 502
@@ -511,7 +504,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) · 1 unmodified line
511 return 0;504 return 0;
512 505
513 kvm_pgtable_walk(pgt, addr, size, &walker);506 kvm_pgtable_walk(pgt, addr, size, &walker);
514 return unmap_data.unmapped;507 return unmapped;
515}508}
516 509
517int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,510int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
@@ -519,7 +512,7 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, · 1 unmodified line
519{512{
520 u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits);513 u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits);
521 514
522 pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL);515 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
523 if (!pgt->pgd)516 if (!pgt->pgd)
524 return -ENOMEM;517 return -ENOMEM;
525 518
@@ -532,19 +525,18 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits, · 6 unmodified lines
532 return 0;525 return 0;
533}526}
534 527
535static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,528static int hyp_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
536 enum kvm_pgtable_walk_flags flag, void * const arg)529 enum kvm_pgtable_walk_flags visit)
537{530{
538 struct kvm_pgtable_mm_ops *mm_ops = arg;531 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
539 kvm_pte_t pte = *ptep;
540 532
541 if (!kvm_pte_valid(pte))533 if (!kvm_pte_valid(ctx->old))
542 return 0;534 return 0;
543 535
544 mm_ops->put_page(ptep);536 mm_ops->put_page(ctx->ptep);
545 537
546 if (kvm_pte_table(pte, level))538 if (kvm_pte_table(ctx->old, ctx->level))
547 mm_ops->put_page(kvm_pte_follow(pte, mm_ops));539 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
548 540
549 return 0;541 return 0;
550}542}
@@ -554,11 +546,10 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) · 3 unmodified lines
554 struct kvm_pgtable_walker walker = {546 struct kvm_pgtable_walker walker = {
555 .cb = hyp_free_walker,547 .cb = hyp_free_walker,
556 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,548 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
557 .arg = pgt->mm_ops,
558 };549 };
559 550
560 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));551 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
561 pgt->mm_ops->put_page(pgt->pgd);552 pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
562 pgt->pgd = NULL;553 pgt->pgd = NULL;
563}554}
564 555
@@ -573,8 +564,6 @@ struct stage2_map_data { · 8 unmodified lines
573 struct kvm_s2_mmu *mmu;564 struct kvm_s2_mmu *mmu;
574 void *memcache;565 void *memcache;
575 566
576 struct kvm_pgtable_mm_ops *mm_ops;
577
578 /* Force mappings to page granularity */567 /* Force mappings to page granularity */
579 bool force_pte;568 bool force_pte;
580};569};
@@ -682,19 +671,92 @@ static bool stage2_pte_is_counted(kvm_pte_t pte) · 101 unmodified lines
682 return !!pte;671 return !!pte;
683}672}
684 673
685static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,674static bool stage2_pte_is_locked(kvm_pte_t pte)
686 u32 level, struct kvm_pgtable_mm_ops *mm_ops)675{
676 return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED);
677}
678
679static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
680{
681 if (!kvm_pgtable_walk_shared(ctx)) {
682 WRITE_ONCE(*ctx->ptep, new);
683 return true;
684 }
685
686 return cmpxchg(ctx->ptep, ctx->old, new) == ctx->old;
687}
688
689/**
690 * stage2_try_break_pte() - Invalidates a pte according to the
691 * 'break-before-make' requirements of the
692 * architecture.
693 *
694 * @ctx: context of the visited pte.
695 * @mmu: stage-2 mmu
696 *
697 * Returns: true if the pte was successfully broken.
698 *
699 * If the removed pte was valid, performs the necessary serialization and TLB
700 * invalidation for the old value. For counted ptes, drops the reference count
701 * on the containing table page.
702 */
703static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
704 struct kvm_s2_mmu *mmu)
705{
706 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
707
708 if (stage2_pte_is_locked(ctx->old)) {
709 /*
710 * Should never occur if this walker has exclusive access to the
711 * page tables.
712 */
713 WARN_ON(!kvm_pgtable_walk_shared(ctx));
714 return false;
715 }
716
717 if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED))
718 return false;
719
720 /*
721 * Perform the appropriate TLB invalidation based on the evicted pte
722 * value (if any).
723 */
724 if (kvm_pte_table(ctx->old, ctx->level))
725 kvm_call_hyp(__kvm_tlb_flush_vmid, mmu);
726 else if (kvm_pte_valid(ctx->old))
727 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level);
728
729 if (stage2_pte_is_counted(ctx->old))
730 mm_ops->put_page(ctx->ptep);
731
732 return true;
733}
734
735static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
736{
737 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
738
739 WARN_ON(!stage2_pte_is_locked(*ctx->ptep));
740
741 if (stage2_pte_is_counted(new))
742 mm_ops->get_page(ctx->ptep);
743
744 smp_store_release(ctx->ptep, new);
745}
746
747static void stage2_put_pte(const struct kvm_pgtable_visit_ctx *ctx, struct kvm_s2_mmu *mmu,
748 struct kvm_pgtable_mm_ops *mm_ops)
687{749{
688 /*750 /*
689 * Clear the existing PTE, and perform break-before-make with751 * Clear the existing PTE, and perform break-before-make with
690 * TLB maintenance if it was valid.752 * TLB maintenance if it was valid.
691 */753 */
692 if (kvm_pte_valid(*ptep)) {754 if (kvm_pte_valid(ctx->old)) {
693 kvm_clear_pte(ptep);755 kvm_clear_pte(ctx->ptep);
694 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);756 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr, ctx->level);
695 }757 }
696 758
697 mm_ops->put_page(ptep);759 mm_ops->put_page(ctx->ptep);
698}760}
699 761
700static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)762static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)
@@ -708,44 +770,42 @@ static bool stage2_pte_executable(kvm_pte_t pte) · 7 unmodified lines
708 return !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN);770 return !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN);
709}771}
710 772
711static bool stage2_leaf_mapping_allowed(u64 addr, u64 end, u32 level,773static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx,
712 struct stage2_map_data *data)774 struct stage2_map_data *data)
713{775{
714 if (data->force_pte && (level < (KVM_PGTABLE_MAX_LEVELS - 1)))776 if (data->force_pte && (ctx->level < (KVM_PGTABLE_MAX_LEVELS - 1)))
715 return false;777 return false;
716 778
717 return kvm_block_mapping_supported(addr, end, data->phys, level);779 return kvm_block_mapping_supported(ctx, data->phys);
718}780}
719 781
720static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,782static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
721 kvm_pte_t *ptep,
722 struct stage2_map_data *data)783 struct stage2_map_data *data)
723{784{
724 kvm_pte_t new, old = *ptep;785 kvm_pte_t new;
725 u64 granule = kvm_granule_size(level), phys = data->phys;786 u64 granule = kvm_granule_size(ctx->level), phys = data->phys;
726 struct kvm_pgtable *pgt = data->mmu->pgt;787 struct kvm_pgtable *pgt = data->mmu->pgt;
727 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;788 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
728 789
729 if (!stage2_leaf_mapping_allowed(addr, end, level, data))790 if (!stage2_leaf_mapping_allowed(ctx, data))
730 return -E2BIG;791 return -E2BIG;
731 792
732 if (kvm_phys_is_valid(phys))793 if (kvm_phys_is_valid(phys))
733 new = kvm_init_valid_leaf_pte(phys, data->attr, level);794 new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
734 else795 else
735 new = kvm_init_invalid_leaf_owner(data->owner_id);796 new = kvm_init_invalid_leaf_owner(data->owner_id);
736 797
737 if (stage2_pte_is_counted(old)) {798 /*
738 /*799 * Skip updating the PTE if we are trying to recreate the exact
739 * Skip updating the PTE if we are trying to recreate the exact800 * same mapping or only change the access permissions. Instead,
740 * same mapping or only change the access permissions. Instead,801 * the vCPU will exit one more time from guest if still needed
741 * the vCPU will exit one more time from guest if still needed802 * and then go through the path of relaxing permissions.
742 * and then go through the path of relaxing permissions.803 */
743 */804 if (!stage2_pte_needs_update(ctx->old, new))
744 if (!stage2_pte_needs_update(old, new))805 return -EAGAIN;
745 return -EAGAIN;
746 806
747 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);807 if (!stage2_try_break_pte(ctx, data->mmu))
748 }808 return -EAGAIN;
749 809
750 /* Perform CMOs before installation of the guest stage-2 PTE */810 /* Perform CMOs before installation of the guest stage-2 PTE */
751 if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new))811 if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new))
@@ -755,56 +815,43 @@ static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, · 3 unmodified lines
755 if (mm_ops->icache_inval_pou && stage2_pte_executable(new))815 if (mm_ops->icache_inval_pou && stage2_pte_executable(new))
756 mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);816 mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);
757 817
758 smp_store_release(ptep, new);818 stage2_make_pte(ctx, new);
759 if (stage2_pte_is_counted(new))819
760 mm_ops->get_page(ptep);
761 if (kvm_phys_is_valid(phys))820 if (kvm_phys_is_valid(phys))
762 data->phys += granule;821 data->phys += granule;
763 return 0;822 return 0;
764}823}
765 824
766static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,825static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx,
767 kvm_pte_t *ptep,
768 struct stage2_map_data *data)826 struct stage2_map_data *data)
769{827{
770 if (data->anchor)828 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
771 return 0;829 kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
830 int ret;
772 831
773 if (!stage2_leaf_mapping_allowed(addr, end, level, data))832 if (!stage2_leaf_mapping_allowed(ctx, data))
774 return 0;833 return 0;
775 834
776 data->childp = kvm_pte_follow(*ptep, data->mm_ops);835 ret = stage2_map_walker_try_leaf(ctx, data);
777 kvm_clear_pte(ptep);836 if (ret)
837 return ret;
778 838
779 /*839 mm_ops->free_removed_table(childp, ctx->level);
780 * Invalidate the whole stage-2, as we may have numerous leaf
781 * entries below us which would otherwise need invalidating
782 * individually.
783 */
784 kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu);
785 data->anchor = ptep;
786 return 0;840 return 0;
787}841}
788 842
789static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,843static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx,
790 struct stage2_map_data *data)844 struct stage2_map_data *data)
791{845{
792 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;846 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
793 kvm_pte_t *childp, pte = *ptep;847 kvm_pte_t *childp, new;
794 int ret;848 int ret;
795 849
796 if (data->anchor) {850 ret = stage2_map_walker_try_leaf(ctx, data);
797 if (stage2_pte_is_counted(pte))
798 mm_ops->put_page(ptep);
799
800 return 0;
801 }
802
803 ret = stage2_map_walker_try_leaf(addr, end, level, ptep, data);
804 if (ret != -E2BIG)851 if (ret != -E2BIG)
805 return ret;852 return ret;
806 853
807 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))854 if (WARN_ON(ctx->level == KVM_PGTABLE_MAX_LEVELS - 1))
808 return -EINVAL;855 return -EINVAL;
809 856
810 if (!data->memcache)857 if (!data->memcache)
@@ -814,99 +861,62 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 3 unmodified lines
814 if (!childp)861 if (!childp)
815 return -ENOMEM;862 return -ENOMEM;
816 863
864 if (!stage2_try_break_pte(ctx, data->mmu)) {
865 mm_ops->put_page(childp);
866 return -EAGAIN;
867 }
868
817 /*869 /*
818 * If we've run into an existing block mapping then replace it with870 * If we've run into an existing block mapping then replace it with
819 * a table. Accesses beyond 'end' that fall within the new table871 * a table. Accesses beyond 'end' that fall within the new table
820 * will be mapped lazily.872 * will be mapped lazily.
821 */873 */
822 if (stage2_pte_is_counted(pte))874 new = kvm_init_table_pte(childp, mm_ops);
823 stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);875 stage2_make_pte(ctx, new);
824
825 kvm_set_table_pte(ptep, childp, mm_ops);
826 mm_ops->get_page(ptep);
827 876
828 return 0;877 return 0;
829}878}
830 879
831static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level,
832 kvm_pte_t *ptep,
833 struct stage2_map_data *data)
834{
835 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
836 kvm_pte_t *childp;
837 int ret = 0;
838
839 if (!data->anchor)
840 return 0;
841
842 if (data->anchor == ptep) {
843 childp = data->childp;
844 data->anchor = NULL;
845 data->childp = NULL;
846 ret = stage2_map_walk_leaf(addr, end, level, ptep, data);
847 } else {
848 childp = kvm_pte_follow(*ptep, mm_ops);
849 }
850
851 mm_ops->put_page(childp);
852 mm_ops->put_page(ptep);
853
854 return ret;
855}
856
857/*880/*
858 * This is a little fiddly, as we use all three of the walk flags. The idea881 * The TABLE_PRE callback runs for table entries on the way down, looking
859 * is that the TABLE_PRE callback runs for table entries on the way down,882 * for table entries which we could conceivably replace with a block entry
860 * looking for table entries which we could conceivably replace with a883 * for this mapping. If it finds one it replaces the entry and calls
861 * block entry for this mapping. If it finds one, then it sets the 'anchor'884 * kvm_pgtable_mm_ops::free_removed_table() to tear down the detached table.
862 * field in 'struct stage2_map_data' to point at the table entry, before
863 * clearing the entry to zero and descending into the now detached table.
864 *885 *
865 * The behaviour of the LEAF callback then depends on whether or not the886 * Otherwise, the LEAF callback performs the mapping at the existing leaves
866 * anchor has been set. If not, then we're not using a block mapping higher887 * instead.
867 * up the table and we perform the mapping at the existing leaves instead.
868 * If, on the other hand, the anchor _is_ set, then we drop references to
869 * all valid leaves so that the pages beneath the anchor can be freed.
870 *
871 * Finally, the TABLE_POST callback does nothing if the anchor has not
872 * been set, but otherwise frees the page-table pages while walking back up
873 * the page-table, installing the block entry when it revisits the anchor
874 * pointer and clearing the anchor to NULL.
875 */888 */
876static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,889static int stage2_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
877 enum kvm_pgtable_walk_flags flag, void * const arg)890 enum kvm_pgtable_walk_flags visit)
878{891{
879 struct stage2_map_data *data = arg;892 struct stage2_map_data *data = ctx->arg;
880 893
881 switch (flag) {894 switch (visit) {
882 case KVM_PGTABLE_WALK_TABLE_PRE:895 case KVM_PGTABLE_WALK_TABLE_PRE:
883 return stage2_map_walk_table_pre(addr, end, level, ptep, data);896 return stage2_map_walk_table_pre(ctx, data);
884 case KVM_PGTABLE_WALK_LEAF:897 case KVM_PGTABLE_WALK_LEAF:
885 return stage2_map_walk_leaf(addr, end, level, ptep, data);898 return stage2_map_walk_leaf(ctx, data);
886 case KVM_PGTABLE_WALK_TABLE_POST:899 default:
887 return stage2_map_walk_table_post(addr, end, level, ptep, data);900 return -EINVAL;
888 }901 }
889
890 return -EINVAL;
891}902}
892 903
893int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,904int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
894 u64 phys, enum kvm_pgtable_prot prot,905 u64 phys, enum kvm_pgtable_prot prot,
895 void *mc)906 void *mc, enum kvm_pgtable_walk_flags flags)
896{907{
897 int ret;908 int ret;
898 struct stage2_map_data map_data = {909 struct stage2_map_data map_data = {
899 .phys = ALIGN_DOWN(phys, PAGE_SIZE),910 .phys = ALIGN_DOWN(phys, PAGE_SIZE),
900 .mmu = pgt->mmu,911 .mmu = pgt->mmu,
901 .memcache = mc,912 .memcache = mc,
902 .mm_ops = pgt->mm_ops,
903 .force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),913 .force_pte = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),
904 };914 };
905 struct kvm_pgtable_walker walker = {915 struct kvm_pgtable_walker walker = {
906 .cb = stage2_map_walker,916 .cb = stage2_map_walker,
907 .flags = KVM_PGTABLE_WALK_TABLE_PRE |917 .flags = flags |
908 KVM_PGTABLE_WALK_LEAF |918 KVM_PGTABLE_WALK_TABLE_PRE |
909 KVM_PGTABLE_WALK_TABLE_POST,919 KVM_PGTABLE_WALK_LEAF,
910 .arg = &map_data,920 .arg = &map_data,
911 };921 };
912 922
@@ -930,15 +940,13 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, · 17 unmodified lines
930 .phys = KVM_PHYS_INVALID,940 .phys = KVM_PHYS_INVALID,
931 .mmu = pgt->mmu,941 .mmu = pgt->mmu,
932 .memcache = mc,942 .memcache = mc,
933 .mm_ops = pgt->mm_ops,
934 .owner_id = owner_id,943 .owner_id = owner_id,
935 .force_pte = true,944 .force_pte = true,
936 };945 };
937 struct kvm_pgtable_walker walker = {946 struct kvm_pgtable_walker walker = {
938 .cb = stage2_map_walker,947 .cb = stage2_map_walker,
939 .flags = KVM_PGTABLE_WALK_TABLE_PRE |948 .flags = KVM_PGTABLE_WALK_TABLE_PRE |
940 KVM_PGTABLE_WALK_LEAF |949 KVM_PGTABLE_WALK_LEAF,
941 KVM_PGTABLE_WALK_TABLE_POST,
942 .arg = &map_data,950 .arg = &map_data,
943 };951 };
944 952
@@ -949,30 +957,29 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, · 4 unmodified lines
949 return ret;957 return ret;
950}958}
951 959
952static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,960static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
953 enum kvm_pgtable_walk_flags flag,961 enum kvm_pgtable_walk_flags visit)
954 void * const arg)
955{962{
956 struct kvm_pgtable *pgt = arg;963 struct kvm_pgtable *pgt = ctx->arg;
957 struct kvm_s2_mmu *mmu = pgt->mmu;964 struct kvm_s2_mmu *mmu = pgt->mmu;
958 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;965 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
959 kvm_pte_t pte = *ptep, *childp = NULL;966 kvm_pte_t *childp = NULL;
960 bool need_flush = false;967 bool need_flush = false;
961 968
962 if (!kvm_pte_valid(pte)) {969 if (!kvm_pte_valid(ctx->old)) {
963 if (stage2_pte_is_counted(pte)) {970 if (stage2_pte_is_counted(ctx->old)) {
964 kvm_clear_pte(ptep);971 kvm_clear_pte(ctx->ptep);
965 mm_ops->put_page(ptep);972 mm_ops->put_page(ctx->ptep);
966 }973 }
967 return 0;974 return 0;
968 }975 }
969 976
970 if (kvm_pte_table(pte, level)) {977 if (kvm_pte_table(ctx->old, ctx->level)) {
971 childp = kvm_pte_follow(pte, mm_ops);978 childp = kvm_pte_follow(ctx->old, mm_ops);
972 979
973 if (mm_ops->page_count(childp) != 1)980 if (mm_ops->page_count(childp) != 1)
974 return 0;981 return 0;
975 } else if (stage2_pte_cacheable(pgt, pte)) {982 } else if (stage2_pte_cacheable(pgt, ctx->old)) {
976 need_flush = !stage2_has_fwb(pgt);983 need_flush = !stage2_has_fwb(pgt);
977 }984 }
978 985
@@ -981,11 +988,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 2 unmodified lines
981 * block entry and rely on the remaining portions being faulted988 * block entry and rely on the remaining portions being faulted
982 * back lazily.989 * back lazily.
983 */990 */
984 stage2_put_pte(ptep, mmu, addr, level, mm_ops);991 stage2_put_pte(ctx, mmu, mm_ops);
985 992
986 if (need_flush && mm_ops->dcache_clean_inval_poc)993 if (need_flush && mm_ops->dcache_clean_inval_poc)
987 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),994 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
988 kvm_granule_size(level));995 kvm_granule_size(ctx->level));
989 996
990 if (childp)997 if (childp)
991 mm_ops->put_page(childp);998 mm_ops->put_page(childp);
@@ -1009,21 +1016,19 @@ struct stage2_attr_data { · 17 unmodified lines
1009 kvm_pte_t attr_clr;1016 kvm_pte_t attr_clr;
1010 kvm_pte_t pte;1017 kvm_pte_t pte;
1011 u32 level;1018 u32 level;
1012 struct kvm_pgtable_mm_ops *mm_ops;
1013};1019};
1014 1020
1015static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,1021static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx,
1016 enum kvm_pgtable_walk_flags flag,1022 enum kvm_pgtable_walk_flags visit)
1017 void * const arg)
1018{1023{
1019 kvm_pte_t pte = *ptep;1024 kvm_pte_t pte = ctx->old;
1020 struct stage2_attr_data *data = arg;1025 struct stage2_attr_data *data = ctx->arg;
1021 struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;1026 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1022 1027
1023 if (!kvm_pte_valid(pte))1028 if (!kvm_pte_valid(ctx->old))
1024 return 0;1029 return 0;
1025 1030
1026 data->level = level;1031 data->level = ctx->level;
1027 data->pte = pte;1032 data->pte = pte;
1028 pte &= ~data->attr_clr;1033 pte &= ~data->attr_clr;
1029 pte |= data->attr_set;1034 pte |= data->attr_set;
@@ -1039,10 +1044,12 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 9 unmodified lines
1039 * stage-2 PTE if we are going to add executable permission.1044 * stage-2 PTE if we are going to add executable permission.
1040 */1045 */
1041 if (mm_ops->icache_inval_pou &&1046 if (mm_ops->icache_inval_pou &&
1042 stage2_pte_executable(pte) && !stage2_pte_executable(*ptep))1047 stage2_pte_executable(pte) && !stage2_pte_executable(ctx->old))
1043 mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),1048 mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
1044 kvm_granule_size(level));1049 kvm_granule_size(ctx->level));
1045 WRITE_ONCE(*ptep, pte);1050
1051 if (!stage2_try_set_pte(ctx, pte))
1052 return -EAGAIN;
1046 }1053 }
1047 1054
1048 return 0;1055 return 0;
@@ -1051,19 +1058,18 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 2 unmodified lines
1051static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,1058static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,
1052 u64 size, kvm_pte_t attr_set,1059 u64 size, kvm_pte_t attr_set,
1053 kvm_pte_t attr_clr, kvm_pte_t *orig_pte,1060 kvm_pte_t attr_clr, kvm_pte_t *orig_pte,
1054 u32 *level)1061 u32 *level, enum kvm_pgtable_walk_flags flags)
1055{1062{
1056 int ret;1063 int ret;
1057 kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;1064 kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;
1058 struct stage2_attr_data data = {1065 struct stage2_attr_data data = {
1059 .attr_set = attr_set & attr_mask,1066 .attr_set = attr_set & attr_mask,
1060 .attr_clr = attr_clr & attr_mask,1067 .attr_clr = attr_clr & attr_mask,
1061 .mm_ops = pgt->mm_ops,
1062 };1068 };
1063 struct kvm_pgtable_walker walker = {1069 struct kvm_pgtable_walker walker = {
1064 .cb = stage2_attr_walker,1070 .cb = stage2_attr_walker,
1065 .arg = &data,1071 .arg = &data,
1066 .flags = KVM_PGTABLE_WALK_LEAF,1072 .flags = flags | KVM_PGTABLE_WALK_LEAF,
1067 };1073 };
1068 1074
1069 ret = kvm_pgtable_walk(pgt, addr, size, &walker);1075 ret = kvm_pgtable_walk(pgt, addr, size, &walker);
@@ -1082,14 +1088,14 @@ int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) · 12 unmodified lines
1082{1088{
1083 return stage2_update_leaf_attrs(pgt, addr, size, 0,1089 return stage2_update_leaf_attrs(pgt, addr, size, 0,
1084 KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,1090 KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
1085 NULL, NULL);1091 NULL, NULL, 0);
1086}1092}
1087 1093
1088kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)1094kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr)
1089{1095{
1090 kvm_pte_t pte = 0;1096 kvm_pte_t pte = 0;
1091 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,1097 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,
1092 &pte, NULL);1098 &pte, NULL, 0);
1093 dsb(ishst);1099 dsb(ishst);
1094 return pte;1100 return pte;
1095}1101}
@@ -1098,7 +1104,7 @@ kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) · 2 unmodified lines
1098{1104{
1099 kvm_pte_t pte = 0;1105 kvm_pte_t pte = 0;
1100 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF,1106 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF,
1101 &pte, NULL);1107 &pte, NULL, 0);
1102 /*1108 /*
1103 * "But where's the TLBI?!", you scream.1109 * "But where's the TLBI?!", you scream.
1104 * "Over in the core code", I sigh.1110 * "Over in the core code", I sigh.
@@ -1111,7 +1117,7 @@ kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) · 6 unmodified lines
1111bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr)1117bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr)
1112{1118{
1113 kvm_pte_t pte = 0;1119 kvm_pte_t pte = 0;
1114 stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL);1120 stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL, 0);
1115 return pte & KVM_PTE_LEAF_ATTR_LO_S2_AF;1121 return pte & KVM_PTE_LEAF_ATTR_LO_S2_AF;
1116}1122}
1117 1123
@@ -1134,26 +1140,25 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, · 16 unmodified lines
1134 if (prot & KVM_PGTABLE_PROT_X)1140 if (prot & KVM_PGTABLE_PROT_X)
1135 clr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;1141 clr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;
1136 1142
1137 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level);1143 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level,
1144 KVM_PGTABLE_WALK_SHARED);
1138 if (!ret)1145 if (!ret)
1139 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);1146 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level);
1140 return ret;1147 return ret;
1141}1148}
1142 1149
1143static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,1150static int stage2_flush_walker(const struct kvm_pgtable_visit_ctx *ctx,
1144 enum kvm_pgtable_walk_flags flag,1151 enum kvm_pgtable_walk_flags visit)
1145 void * const arg)
1146{1152{
1147 struct kvm_pgtable *pgt = arg;1153 struct kvm_pgtable *pgt = ctx->arg;
1148 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;1154 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1149 kvm_pte_t pte = *ptep;
1150 1155
1151 if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))1156 if (!kvm_pte_valid(ctx->old) || !stage2_pte_cacheable(pgt, ctx->old))
1152 return 0;1157 return 0;
1153 1158
1154 if (mm_ops->dcache_clean_inval_poc)1159 if (mm_ops->dcache_clean_inval_poc)
1155 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(pte, mm_ops),1160 mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1156 kvm_granule_size(level));1161 kvm_granule_size(ctx->level));
1157 return 0;1162 return 0;
1158}1163}
1159 1164
@@ -1184,7 +1189,7 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 24 unmodified lines
1184 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;1189 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
1185 1190
1186 pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;1191 pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1187 pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz);1192 pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
1188 if (!pgt->pgd)1193 if (!pgt->pgd)
1189 return -ENOMEM;1194 return -ENOMEM;
1190 1195
@@ -1200,20 +1205,27 @@ int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, · 9 unmodified lines
1200 return 0;1205 return 0;
1201}1206}
1202 1207
1203static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,1208size_t kvm_pgtable_stage2_pgd_size(u64 vtcr)
1204 enum kvm_pgtable_walk_flags flag,1209{
1205 void * const arg)1210 u32 ia_bits = VTCR_EL2_IPA(vtcr);
1211 u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
1212 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
1213
1214 return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1215}
1216
1217static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
1218 enum kvm_pgtable_walk_flags visit)
1206{1219{
1207 struct kvm_pgtable_mm_ops *mm_ops = arg;1220 struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1208 kvm_pte_t pte = *ptep;
1209 1221
1210 if (!stage2_pte_is_counted(pte))1222 if (!stage2_pte_is_counted(ctx->old))
1211 return 0;1223 return 0;
1212 1224
1213 mm_ops->put_page(ptep);1225 mm_ops->put_page(ctx->ptep);
1214 1226
1215 if (kvm_pte_table(pte, level))1227 if (kvm_pte_table(ctx->old, ctx->level))
1216 mm_ops->put_page(kvm_pte_follow(pte, mm_ops));1228 mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
1217 1229
1218 return 0;1230 return 0;
1219}1231}
@@ -1225,11 +1237,33 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt) · 5 unmodified lines
1225 .cb = stage2_free_walker,1237 .cb = stage2_free_walker,
1226 .flags = KVM_PGTABLE_WALK_LEAF |1238 .flags = KVM_PGTABLE_WALK_LEAF |
1227 KVM_PGTABLE_WALK_TABLE_POST,1239 KVM_PGTABLE_WALK_TABLE_POST,
1228 .arg = pgt->mm_ops,
1229 };1240 };
1230 1241
1231 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));1242 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
1232 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;1243 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
1233 pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz);1244 pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);
1234 pgt->pgd = NULL;1245 pgt->pgd = NULL;
1235}1246}
1247
1248void kvm_pgtable_stage2_free_removed(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, u32 level)
1249{
1250 kvm_pteref_t ptep = (kvm_pteref_t)pgtable;
1251 struct kvm_pgtable_walker walker = {
1252 .cb = stage2_free_walker,
1253 .flags = KVM_PGTABLE_WALK_LEAF |
1254 KVM_PGTABLE_WALK_TABLE_POST,
1255 };
1256 struct kvm_pgtable_walk_data data = {
1257 .walker = &walker,
1258
1259 /*
1260 * At this point the IPA really doesn't matter, as the page
1261 * table being traversed has already been removed from the stage
1262 * 2. Set an appropriate range to cover the entire page table.
1263 */
1264 .addr = 0,
1265 .end = kvm_granule_size(level),
1266 };
1267
1268 WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));
1269}
arch/arm64/kvm/hyp/vhe/Makefile+1−1
@@ -1,6 +1,6 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2#2#
3# Makefile for Kernel-based Virtual Machine module, HYP/nVHE part3# Makefile for Kernel-based Virtual Machine module, HYP/VHE part
4#4#
5 5
6asflags-y := -D__KVM_VHE_HYPERVISOR__6asflags-y := -D__KVM_VHE_HYPERVISOR__
arch/arm64/kvm/mmu.c+132−61
@@ -128,6 +128,25 @@ static void kvm_s2_free_pages_exact(void *virt, size_t size) · 127 unmodified lines
128 free_pages_exact(virt, size);128 free_pages_exact(virt, size);
129}129}
130 130
131static struct kvm_pgtable_mm_ops kvm_s2_mm_ops;
132
133static void stage2_free_removed_table_rcu_cb(struct rcu_head *head)
134{
135 struct page *page = container_of(head, struct page, rcu_head);
136 void *pgtable = page_to_virt(page);
137 u32 level = page_private(page);
138
139 kvm_pgtable_stage2_free_removed(&kvm_s2_mm_ops, pgtable, level);
140}
141
142static void stage2_free_removed_table(void *addr, u32 level)
143{
144 struct page *page = virt_to_page(addr);
145
146 set_page_private(page, (unsigned long)level);
147 call_rcu(&page->rcu_head, stage2_free_removed_table_rcu_cb);
148}
149
131static void kvm_host_get_page(void *addr)150static void kvm_host_get_page(void *addr)
132{151{
133 get_page(virt_to_page(addr));152 get_page(virt_to_page(addr));
@@ -640,8 +659,8 @@ static struct kvm_pgtable_mm_ops kvm_user_mm_ops = { · 506 unmodified lines
640static int get_user_mapping_size(struct kvm *kvm, u64 addr)659static int get_user_mapping_size(struct kvm *kvm, u64 addr)
641{660{
642 struct kvm_pgtable pgt = {661 struct kvm_pgtable pgt = {
643 .pgd = (kvm_pte_t *)kvm->mm->pgd,662 .pgd = (kvm_pteref_t)kvm->mm->pgd,
644 .ia_bits = VA_BITS,663 .ia_bits = vabits_actual,
645 .start_level = (KVM_PGTABLE_MAX_LEVELS -664 .start_level = (KVM_PGTABLE_MAX_LEVELS -
646 CONFIG_PGTABLE_LEVELS),665 CONFIG_PGTABLE_LEVELS),
647 .mm_ops = &kvm_user_mm_ops,666 .mm_ops = &kvm_user_mm_ops,
@@ -662,6 +681,7 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { · 14 unmodified lines
662 .zalloc_page = stage2_memcache_zalloc_page,681 .zalloc_page = stage2_memcache_zalloc_page,
663 .zalloc_pages_exact = kvm_s2_zalloc_pages_exact,682 .zalloc_pages_exact = kvm_s2_zalloc_pages_exact,
664 .free_pages_exact = kvm_s2_free_pages_exact,683 .free_pages_exact = kvm_s2_free_pages_exact,
684 .free_removed_table = stage2_free_removed_table,
665 .get_page = kvm_host_get_page,685 .get_page = kvm_host_get_page,
666 .put_page = kvm_s2_put_page,686 .put_page = kvm_s2_put_page,
667 .page_count = kvm_host_page_count,687 .page_count = kvm_host_page_count,
@@ -675,15 +695,42 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { · 7 unmodified lines
675 * kvm_init_stage2_mmu - Initialise a S2 MMU structure695 * kvm_init_stage2_mmu - Initialise a S2 MMU structure
676 * @kvm: The pointer to the KVM structure696 * @kvm: The pointer to the KVM structure
677 * @mmu: The pointer to the s2 MMU structure697 * @mmu: The pointer to the s2 MMU structure
698 * @type: The machine type of the virtual machine
678 *699 *
679 * Allocates only the stage-2 HW PGD level table(s).700 * Allocates only the stage-2 HW PGD level table(s).
680 * Note we don't need locking here as this is only called when the VM is701 * Note we don't need locking here as this is only called when the VM is
681 * created, which can only be done once.702 * created, which can only be done once.
682 */703 */
683int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)704int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long type)
684{705{
706 u32 kvm_ipa_limit = get_kvm_ipa_limit();
685 int cpu, err;707 int cpu, err;
686 struct kvm_pgtable *pgt;708 struct kvm_pgtable *pgt;
709 u64 mmfr0, mmfr1;
710 u32 phys_shift;
711
712 if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
713 return -EINVAL;
714
715 phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
716 if (is_protected_kvm_enabled()) {
717 phys_shift = kvm_ipa_limit;
718 } else if (phys_shift) {
719 if (phys_shift > kvm_ipa_limit ||
720 phys_shift < ARM64_MIN_PARANGE_BITS)
721 return -EINVAL;
722 } else {
723 phys_shift = KVM_PHYS_SHIFT;
724 if (phys_shift > kvm_ipa_limit) {
725 pr_warn_once("%s using unsupported default IPA limit, upgrade your VMM\n",
726 current->comm);
727 return -EINVAL;
728 }
729 }
730
731 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
732 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
733 kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
687 734
688 if (mmu->pgt != NULL) {735 if (mmu->pgt != NULL) {
689 kvm_err("kvm_arch already initialized?\n");736 kvm_err("kvm_arch already initialized?\n");
@@ -807,6 +854,32 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) · 117 unmodified lines
807 }854 }
808}855}
809 856
857static void hyp_mc_free_fn(void *addr, void *unused)
858{
859 free_page((unsigned long)addr);
860}
861
862static void *hyp_mc_alloc_fn(void *unused)
863{
864 return (void *)__get_free_page(GFP_KERNEL_ACCOUNT);
865}
866
867void free_hyp_memcache(struct kvm_hyp_memcache *mc)
868{
869 if (is_protected_kvm_enabled())
870 __free_hyp_memcache(mc, hyp_mc_free_fn,
871 kvm_host_va, NULL);
872}
873
874int topup_hyp_memcache(struct kvm_hyp_memcache *mc, unsigned long min_pages)
875{
876 if (!is_protected_kvm_enabled())
877 return 0;
878
879 return __topup_hyp_memcache(mc, min_pages, hyp_mc_alloc_fn,
880 kvm_host_pa, NULL);
881}
882
810/**883/**
811 * kvm_phys_addr_ioremap - map a device range to guest IPA884 * kvm_phys_addr_ioremap - map a device range to guest IPA
812 *885 *
@@ -841,7 +914,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, · 28 unmodified lines
841 914
842 write_lock(&kvm->mmu_lock);915 write_lock(&kvm->mmu_lock);
843 ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,916 ret = kvm_pgtable_stage2_map(pgt, addr, PAGE_SIZE, pa, prot,
844 &cache);917 &cache, 0);
845 write_unlock(&kvm->mmu_lock);918 write_unlock(&kvm->mmu_lock);
846 if (ret)919 if (ret)
847 break;920 break;
@@ -1091,32 +1164,26 @@ static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva) · 243 unmodified lines
1091 * - mmap_lock protects between a VM faulting a page in and the VMM performing1164 * - mmap_lock protects between a VM faulting a page in and the VMM performing
1092 * an mprotect() to add VM_MTE1165 * an mprotect() to add VM_MTE
1093 */1166 */
1094static int sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,1167static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,
1095 unsigned long size)1168 unsigned long size)
1096{1169{
1097 unsigned long i, nr_pages = size >> PAGE_SHIFT;1170 unsigned long i, nr_pages = size >> PAGE_SHIFT;
1098 struct page *page;1171 struct page *page = pfn_to_page(pfn);
1099 1172
1100 if (!kvm_has_mte(kvm))1173 if (!kvm_has_mte(kvm))
1101 return 0;1174 return;
1102
1103 /*
1104 * pfn_to_online_page() is used to reject ZONE_DEVICE pages
1105 * that may not support tags.
1106 */
1107 page = pfn_to_online_page(pfn);
1108
1109 if (!page)
1110 return -EFAULT;
1111 1175
1112 for (i = 0; i < nr_pages; i++, page++) {1176 for (i = 0; i < nr_pages; i++, page++) {
1113 if (!test_bit(PG_mte_tagged, &page->flags)) {1177 if (try_page_mte_tagging(page)) {
1114 mte_clear_page_tags(page_address(page));1178 mte_clear_page_tags(page_address(page));
1115 set_bit(PG_mte_tagged, &page->flags);1179 set_page_mte_tagged(page);
1116 }1180 }
1117 }1181 }
1182}
1118 1183
1119 return 0;1184static bool kvm_vma_mte_allowed(struct vm_area_struct *vma)
1185{
1186 return vma->vm_flags & VM_MTE_ALLOWED;
1120}1187}
1121 1188
1122static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,1189static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
@@ -1127,7 +1194,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 4 unmodified lines
1127 bool write_fault, writable, force_pte = false;1194 bool write_fault, writable, force_pte = false;
1128 bool exec_fault;1195 bool exec_fault;
1129 bool device = false;1196 bool device = false;
1130 bool shared;
1131 unsigned long mmu_seq;1197 unsigned long mmu_seq;
1132 struct kvm *kvm = vcpu->kvm;1198 struct kvm *kvm = vcpu->kvm;
1133 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;1199 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
@@ -1136,7 +1202,6 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 2 unmodified lines
1136 gfn_t gfn;1202 gfn_t gfn;
1137 kvm_pfn_t pfn;1203 kvm_pfn_t pfn;
1138 bool logging_active = memslot_is_logging(memslot);1204 bool logging_active = memslot_is_logging(memslot);
1139 bool use_read_lock = false;
1140 unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);1205 unsigned long fault_level = kvm_vcpu_trap_get_fault_level(vcpu);
1141 unsigned long vma_pagesize, fault_granule;1206 unsigned long vma_pagesize, fault_granule;
1142 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;1207 enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
@@ -1171,14 +1236,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 28 unmodified lines
1171 if (logging_active) {1236 if (logging_active) {
1172 force_pte = true;1237 force_pte = true;
1173 vma_shift = PAGE_SHIFT;1238 vma_shift = PAGE_SHIFT;
1174 use_read_lock = (fault_status == FSC_PERM && write_fault &&
1175 fault_granule == PAGE_SIZE);
1176 } else {1239 } else {
1177 vma_shift = get_vma_page_shift(vma, hva);1240 vma_shift = get_vma_page_shift(vma, hva);
1178 }1241 }
1179 1242
1180 shared = (vma->vm_flags & VM_SHARED);
1181
1182 switch (vma_shift) {1243 switch (vma_shift) {
1183#ifndef __PAGETABLE_PMD_FOLDED1244#ifndef __PAGETABLE_PMD_FOLDED
1184 case PUD_SHIFT:1245 case PUD_SHIFT:
@@ -1271,15 +1332,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 86 unmodified lines
1271 if (exec_fault && device)1332 if (exec_fault && device)
1272 return -ENOEXEC;1333 return -ENOEXEC;
1273 1334
1274 /*1335 read_lock(&kvm->mmu_lock);
1275 * To reduce MMU contentions and enhance concurrency during dirty
1276 * logging dirty logging, only acquire read lock for permission
1277 * relaxation.
1278 */
1279 if (use_read_lock)
1280 read_lock(&kvm->mmu_lock);
1281 else
1282 write_lock(&kvm->mmu_lock);
1283 pgt = vcpu->arch.hw_mmu->pgt;1336 pgt = vcpu->arch.hw_mmu->pgt;
1284 if (mmu_invalidate_retry(kvm, mmu_seq))1337 if (mmu_invalidate_retry(kvm, mmu_seq))
1285 goto out_unlock;1338 goto out_unlock;
@@ -1298,13 +1351,13 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 12 unmodified lines
1298 }1351 }
1299 1352
1300 if (fault_status != FSC_PERM && !device && kvm_has_mte(kvm)) {1353 if (fault_status != FSC_PERM && !device && kvm_has_mte(kvm)) {
1301 /* Check the VMM hasn't introduced a new VM_SHARED VMA */1354 /* Check the VMM hasn't introduced a new disallowed VMA */
1302 if (!shared)1355 if (kvm_vma_mte_allowed(vma)) {
1303 ret = sanitise_mte_tags(kvm, pfn, vma_pagesize);1356 sanitise_mte_tags(kvm, pfn, vma_pagesize);
1304 else1357 } else {
1305 ret = -EFAULT;1358 ret = -EFAULT;
1306 if (ret)
1307 goto out_unlock;1359 goto out_unlock;
1360 }
1308 }1361 }
1309 1362
1310 if (writable)1363 if (writable)
@@ -1323,15 +1376,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 12 unmodified lines
1323 * permissions only if vma_pagesize equals fault_granule. Otherwise,1376 * permissions only if vma_pagesize equals fault_granule. Otherwise,
1324 * kvm_pgtable_stage2_map() should be called to change block size.1377 * kvm_pgtable_stage2_map() should be called to change block size.
1325 */1378 */
1326 if (fault_status == FSC_PERM && vma_pagesize == fault_granule) {1379 if (fault_status == FSC_PERM && vma_pagesize == fault_granule)
1327 ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);1380 ret = kvm_pgtable_stage2_relax_perms(pgt, fault_ipa, prot);
1328 } else {1381 else
1329 WARN_ONCE(use_read_lock, "Attempted stage-2 map outside of write lock\n");
1330
1331 ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,1382 ret = kvm_pgtable_stage2_map(pgt, fault_ipa, vma_pagesize,
1332 __pfn_to_phys(pfn), prot,1383 __pfn_to_phys(pfn), prot,
1333 memcache);1384 memcache, KVM_PGTABLE_WALK_SHARED);
1334 }
1335 1385
1336 /* Mark the page dirty only if the fault is handled successfully */1386 /* Mark the page dirty only if the fault is handled successfully */
1337 if (writable && !ret) {1387 if (writable && !ret) {
@@ -1340,10 +1390,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 2 unmodified lines
1340 }1390 }
1341 1391
1342out_unlock:1392out_unlock:
1343 if (use_read_lock)1393 read_unlock(&kvm->mmu_lock);
1344 read_unlock(&kvm->mmu_lock);
1345 else
1346 write_unlock(&kvm->mmu_lock);
1347 kvm_set_pfn_accessed(pfn);1394 kvm_set_pfn_accessed(pfn);
1348 kvm_release_pfn_clean(pfn);1395 kvm_release_pfn_clean(pfn);
1349 return ret != -EAGAIN ? ret : 0;1396 return ret != -EAGAIN ? ret : 0;
@@ -1526,15 +1573,18 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) · 176 unmodified lines
1526bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)1573bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1527{1574{
1528 kvm_pfn_t pfn = pte_pfn(range->pte);1575 kvm_pfn_t pfn = pte_pfn(range->pte);
1529 int ret;
1530 1576
1531 if (!kvm->arch.mmu.pgt)1577 if (!kvm->arch.mmu.pgt)
1532 return false;1578 return false;
1533 1579
1534 WARN_ON(range->end - range->start != 1);1580 WARN_ON(range->end - range->start != 1);
1535 1581
1536 ret = sanitise_mte_tags(kvm, pfn, PAGE_SIZE);1582 /*
1537 if (ret)1583 * If the page isn't tagged, defer to user_mem_abort() for sanitising
1584 * the MTE tags. The S2 pte should have been unmapped by
1585 * mmu_notifier_invalidate_range_end().
1586 */
1587 if (kvm_has_mte(kvm) && !page_mte_tagged(pfn_to_page(pfn)))
1538 return false;1588 return false;
1539 1589
1540 /*1590 /*
@@ -1549,7 +1599,7 @@ bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) · 8 unmodified lines
1549 */1599 */
1550 kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT,1600 kvm_pgtable_stage2_map(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT,
1551 PAGE_SIZE, __pfn_to_phys(pfn),1601 PAGE_SIZE, __pfn_to_phys(pfn),
1552 KVM_PGTABLE_PROT_R, NULL);1602 KVM_PGTABLE_PROT_R, NULL, 0);
1553 1603
1554 return false;1604 return false;
1555}1605}
@@ -1618,6 +1668,8 @@ static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = { · 62 unmodified lines
1618int kvm_mmu_init(u32 *hyp_va_bits)1668int kvm_mmu_init(u32 *hyp_va_bits)
1619{1669{
1620 int err;1670 int err;
1671 u32 idmap_bits;
1672 u32 kernel_bits;
1621 1673
1622 hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);1674 hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
1623 hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);1675 hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
@@ -1631,7 +1683,31 @@ int kvm_mmu_init(u32 *hyp_va_bits) · 7 unmodified lines
1631 */1683 */
1632 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);1684 BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1633 1685
1634 *hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);1686 /*
1687 * The ID map may be configured to use an extended virtual address
1688 * range. This is only the case if system RAM is out of range for the
1689 * currently configured page size and VA_BITS_MIN, in which case we will
1690 * also need the extended virtual range for the HYP ID map, or we won't
1691 * be able to enable the EL2 MMU.
1692 *
1693 * However, in some cases the ID map may be configured for fewer than
1694 * the number of VA bits used by the regular kernel stage 1. This
1695 * happens when VA_BITS=52 and the kernel image is placed in PA space
1696 * below 48 bits.
1697 *
1698 * At EL2, there is only one TTBR register, and we can't switch between
1699 * translation tables *and* update TCR_EL2.T0SZ at the same time. Bottom
1700 * line: we need to use the extended range with *both* our translation
1701 * tables.
1702 *
1703 * So use the maximum of the idmap VA bits and the regular kernel stage
1704 * 1 VA bits to assure that the hypervisor can both ID map its code page
1705 * and map any kernel memory.
1706 */
1707 idmap_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
1708 kernel_bits = vabits_actual;
1709 *hyp_va_bits = max(idmap_bits, kernel_bits);
1710
1635 kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);1711 kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);
1636 kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);1712 kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
1637 kvm_debug("HYP VA range: %lx:%lx\n",1713 kvm_debug("HYP VA range: %lx:%lx\n",
@@ -1740,12 +1816,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, · 102 unmodified lines
1740 if (!vma)1816 if (!vma)
1741 break;1817 break;
1742 1818
1743 /*1819 if (kvm_has_mte(kvm) && !kvm_vma_mte_allowed(vma)) {
1744 * VM_SHARED mappings are not allowed with MTE to avoid races
1745 * when updating the PG_mte_tagged page flag, see
1746 * sanitise_mte_tags for more details.
1747 */
1748 if (kvm_has_mte(kvm) && vma->vm_flags & VM_SHARED) {
1749 ret = -EINVAL;1820 ret = -EINVAL;
1750 break;1821 break;
1751 }1822 }
arch/arm64/kvm/pkvm.c+122−16
@@ -6,6 +6,7 @@ · 5 unmodified lines
6 6
7#include <linux/kvm_host.h>7#include <linux/kvm_host.h>
8#include <linux/memblock.h>8#include <linux/memblock.h>
9#include <linux/mutex.h>
9#include <linux/sort.h>10#include <linux/sort.h>
10 11
11#include <asm/kvm_pkvm.h>12#include <asm/kvm_pkvm.h>
@@ -53,7 +54,7 @@ static int __init register_memblock_regions(void) · 41 unmodified lines
53 54
54void __init kvm_hyp_reserve(void)55void __init kvm_hyp_reserve(void)
55{56{
56 u64 nr_pages, prev, hyp_mem_pages = 0;57 u64 hyp_mem_pages = 0;
57 int ret;58 int ret;
58 59
59 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())60 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
@@ -71,21 +72,8 @@ void __init kvm_hyp_reserve(void) · 11 unmodified lines
71 72
72 hyp_mem_pages += hyp_s1_pgtable_pages();73 hyp_mem_pages += hyp_s1_pgtable_pages();
73 hyp_mem_pages += host_s2_pgtable_pages();74 hyp_mem_pages += host_s2_pgtable_pages();
74 75 hyp_mem_pages += hyp_vm_table_pages();
75 /*76 hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
76 * The hyp_vmemmap needs to be backed by pages, but these pages
77 * themselves need to be present in the vmemmap, so compute the number
78 * of pages needed by looking for a fixed point.
79 */
80 nr_pages = 0;
81 do {
82 prev = nr_pages;
83 nr_pages = hyp_mem_pages + prev;
84 nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE,
85 PAGE_SIZE);
86 nr_pages += __hyp_pgtable_max_pages(nr_pages);
87 } while (nr_pages != prev);
88 hyp_mem_pages += nr_pages;
89 77
90 /*78 /*
91 * Try to allocate a PMD-aligned region to reduce TLB pressure once79 * Try to allocate a PMD-aligned region to reduce TLB pressure once
@@ -107,3 +95,121 @@ void __init kvm_hyp_reserve(void) · 15 unmodified lines
107 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,95 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
108 hyp_mem_base);96 hyp_mem_base);
109}97}
98
99/*
100 * Allocates and donates memory for hypervisor VM structs at EL2.
101 *
102 * Allocates space for the VM state, which includes the hyp vm as well as
103 * the hyp vcpus.
104 *
105 * Stores an opaque handler in the kvm struct for future reference.
106 *
107 * Return 0 on success, negative error code on failure.
108 */
109static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
110{
111 size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz;
112 struct kvm_vcpu *host_vcpu;
113 pkvm_handle_t handle;
114 void *pgd, *hyp_vm;
115 unsigned long idx;
116 int ret;
117
118 if (host_kvm->created_vcpus < 1)
119 return -EINVAL;
120
121 pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.vtcr);
122
123 /*
124 * The PGD pages will be reclaimed using a hyp_memcache which implies
125 * page granularity. So, use alloc_pages_exact() to get individual
126 * refcounts.
127 */
128 pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
129 if (!pgd)
130 return -ENOMEM;
131
132 /* Allocate memory to donate to hyp for vm and vcpu pointers. */
133 hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
134 size_mul(sizeof(void *),
135 host_kvm->created_vcpus)));
136 hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
137 if (!hyp_vm) {
138 ret = -ENOMEM;
139 goto free_pgd;
140 }
141
142 /* Donate the VM memory to hyp and let hyp initialize it. */
143 ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd);
144 if (ret < 0)
145 goto free_vm;
146
147 handle = ret;
148
149 host_kvm->arch.pkvm.handle = handle;
150
151 /* Donate memory for the vcpus at hyp and initialize it. */
152 hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
153 kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
154 void *hyp_vcpu;
155
156 /* Indexing of the vcpus to be sequential starting at 0. */
157 if (WARN_ON(host_vcpu->vcpu_idx != idx)) {
158 ret = -EINVAL;
159 goto destroy_vm;
160 }
161
162 hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
163 if (!hyp_vcpu) {
164 ret = -ENOMEM;
165 goto destroy_vm;
166 }
167
168 ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu,
169 hyp_vcpu);
170 if (ret) {
171 free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
172 goto destroy_vm;
173 }
174 }
175
176 return 0;
177
178destroy_vm:
179 pkvm_destroy_hyp_vm(host_kvm);
180 return ret;
181free_vm:
182 free_pages_exact(hyp_vm, hyp_vm_sz);
183free_pgd:
184 free_pages_exact(pgd, pgd_sz);
185 return ret;
186}
187
188int pkvm_create_hyp_vm(struct kvm *host_kvm)
189{
190 int ret = 0;
191
192 mutex_lock(&host_kvm->lock);
193 if (!host_kvm->arch.pkvm.handle)
194 ret = __pkvm_create_hyp_vm(host_kvm);
195 mutex_unlock(&host_kvm->lock);
196
197 return ret;
198}
199
200void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
201{
202 if (host_kvm->arch.pkvm.handle) {
203 WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm,
204 host_kvm->arch.pkvm.handle));
205 }
206
207 host_kvm->arch.pkvm.handle = 0;
208 free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc);
209}
210
211int pkvm_init_host_vm(struct kvm *host_kvm)
212{
213 mutex_init(&host_kvm->lock);
214 return 0;
215}
arch/arm64/kvm/pmu-emul.c+196−286
@@ -15,16 +15,25 @@ · 14 unmodified lines
15#include <kvm/arm_pmu.h>15#include <kvm/arm_pmu.h>
16#include <kvm/arm_vgic.h>16#include <kvm/arm_vgic.h>
17 17
18#define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0)
19
18DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);20DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
19 21
20static LIST_HEAD(arm_pmus);22static LIST_HEAD(arm_pmus);
21static DEFINE_MUTEX(arm_pmus_lock);23static DEFINE_MUTEX(arm_pmus_lock);
22 24
23static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx);25static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc);
24static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx);26static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc);
25static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc);27
28static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc)
29{
30 return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]);
31}
26 32
27#define PERF_ATTR_CFG1_KVM_PMU_CHAINED 0x133static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx)
34{
35 return &vcpu->arch.pmu.pmc[cnt_idx];
36}
28 37
29static u32 kvm_pmu_event_mask(struct kvm *kvm)38static u32 kvm_pmu_event_mask(struct kvm *kvm)
30{39{
@@ -47,113 +56,46 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) · 16 unmodified lines
47}56}
48 57
49/**58/**
50 * kvm_pmu_idx_is_64bit - determine if select_idx is a 64bit counter59 * kvm_pmc_is_64bit - determine if counter is 64bit
51 * @vcpu: The vcpu pointer60 * @pmc: counter context
52 * @select_idx: The counter index
53 */61 */
54static bool kvm_pmu_idx_is_64bit(struct kvm_vcpu *vcpu, u64 select_idx)62static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc)
55{63{
56 return (select_idx == ARMV8_PMU_CYCLE_IDX &&64 return (pmc->idx == ARMV8_PMU_CYCLE_IDX ||
57 __vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_LC);65 kvm_pmu_is_3p5(kvm_pmc_to_vcpu(pmc)));
58}66}
59 67
60static struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)68static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc)
61{69{
62 struct kvm_pmu *pmu;70 u64 val = __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), PMCR_EL0);
63 struct kvm_vcpu_arch *vcpu_arch;
64 71
65 pmc -= pmc->idx;72 return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) ||
66 pmu = container_of(pmc, struct kvm_pmu, pmc[0]);73 (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC));
67 vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
68 return container_of(vcpu_arch, struct kvm_vcpu, arch);
69}74}
70 75
71/**76static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc)
72 * kvm_pmu_pmc_is_chained - determine if the pmc is chained
73 * @pmc: The PMU counter pointer
74 */
75static bool kvm_pmu_pmc_is_chained(struct kvm_pmc *pmc)
76{77{
77 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);78 return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX &&
78 79 !kvm_pmc_has_64bit_overflow(pmc));
79 return test_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
80}80}
81 81
82/**82static u32 counter_index_to_reg(u64 idx)
83 * kvm_pmu_idx_is_high_counter - determine if select_idx is a high/low counter
84 * @select_idx: The counter index
85 */
86static bool kvm_pmu_idx_is_high_counter(u64 select_idx)
87{
88 return select_idx & 0x1;
89}
90
91/**
92 * kvm_pmu_get_canonical_pmc - obtain the canonical pmc
93 * @pmc: The PMU counter pointer
94 *
95 * When a pair of PMCs are chained together we use the low counter (canonical)
96 * to hold the underlying perf event.
97 */
98static struct kvm_pmc *kvm_pmu_get_canonical_pmc(struct kvm_pmc *pmc)
99{
100 if (kvm_pmu_pmc_is_chained(pmc) &&
101 kvm_pmu_idx_is_high_counter(pmc->idx))
102 return pmc - 1;
103
104 return pmc;
105}
106static struct kvm_pmc *kvm_pmu_get_alternate_pmc(struct kvm_pmc *pmc)
107{83{
108 if (kvm_pmu_idx_is_high_counter(pmc->idx))84 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx;
109 return pmc - 1;
110 else
111 return pmc + 1;
112}85}
113 86
114/**87static u32 counter_index_to_evtreg(u64 idx)
115 * kvm_pmu_idx_has_chain_evtype - determine if the event type is chain
116 * @vcpu: The vcpu pointer
117 * @select_idx: The counter index
118 */
119static bool kvm_pmu_idx_has_chain_evtype(struct kvm_vcpu *vcpu, u64 select_idx)
120{88{
121 u64 eventsel, reg;89 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx;
122
123 select_idx |= 0x1;
124
125 if (select_idx == ARMV8_PMU_CYCLE_IDX)
126 return false;
127
128 reg = PMEVTYPER0_EL0 + select_idx;
129 eventsel = __vcpu_sys_reg(vcpu, reg) & kvm_pmu_event_mask(vcpu->kvm);
130
131 return eventsel == ARMV8_PMUV3_PERFCTR_CHAIN;
132}90}
133 91
134/**92static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc)
135 * kvm_pmu_get_pair_counter_value - get PMU counter value
136 * @vcpu: The vcpu pointer
137 * @pmc: The PMU counter pointer
138 */
139static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu,
140 struct kvm_pmc *pmc)
141{93{
142 u64 counter, counter_high, reg, enabled, running;94 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
143 95 u64 counter, reg, enabled, running;
144 if (kvm_pmu_pmc_is_chained(pmc)) {
145 pmc = kvm_pmu_get_canonical_pmc(pmc);
146 reg = PMEVCNTR0_EL0 + pmc->idx;
147 96
148 counter = __vcpu_sys_reg(vcpu, reg);97 reg = counter_index_to_reg(pmc->idx);
149 counter_high = __vcpu_sys_reg(vcpu, reg + 1);98 counter = __vcpu_sys_reg(vcpu, reg);
150
151 counter = lower_32_bits(counter) | (counter_high << 32);
152 } else {
153 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
154 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
155 counter = __vcpu_sys_reg(vcpu, reg);
156 }
157 99
158 /*100 /*
159 * The real counter value is equal to the value of counter register plus101 * The real counter value is equal to the value of counter register plus
@@ -163,6 +105,9 @@ static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu, · 3 unmodified lines
163 counter += perf_event_read_value(pmc->perf_event, &enabled,105 counter += perf_event_read_value(pmc->perf_event, &enabled,
164 &running);106 &running);
165 107
108 if (!kvm_pmc_is_64bit(pmc))
109 counter = lower_32_bits(counter);
110
166 return counter;111 return counter;
167}112}
168 113
@@ -173,22 +118,37 @@ static u64 kvm_pmu_get_pair_counter_value(struct kvm_vcpu *vcpu, · 4 unmodified lines
173 */118 */
174u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)119u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
175{120{
176 u64 counter;
177 struct kvm_pmu *pmu = &vcpu->arch.pmu;
178 struct kvm_pmc *pmc = &pmu->pmc[select_idx];
179
180 if (!kvm_vcpu_has_pmu(vcpu))121 if (!kvm_vcpu_has_pmu(vcpu))
181 return 0;122 return 0;
182 123
183 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);124 return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx));
125}
184 126
185 if (kvm_pmu_pmc_is_chained(pmc) &&127static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force)
186 kvm_pmu_idx_is_high_counter(select_idx))128{
187 counter = upper_32_bits(counter);129 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
188 else if (select_idx != ARMV8_PMU_CYCLE_IDX)130 u64 reg;
189 counter = lower_32_bits(counter);
190 131
191 return counter;132 kvm_pmu_release_perf_event(pmc);
133
134 reg = counter_index_to_reg(pmc->idx);
135
136 if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX &&
137 !force) {
138 /*
139 * Even with PMUv3p5, AArch32 cannot write to the top
140 * 32bit of the counters. The only possible course of
141 * action is to use PMCR.P, which will reset them to
142 * 0 (the only use of the 'force' parameter).
143 */
144 val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32);
145 val |= lower_32_bits(val);
146 }
147
148 __vcpu_sys_reg(vcpu, reg) = val;
149
150 /* Recreate the perf event to reflect the updated sample_period */
151 kvm_pmu_create_perf_event(pmc);
192}152}
193 153
194/**154/**
@@ -199,17 +159,10 @@ u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) · 4 unmodified lines
199 */159 */
200void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)160void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
201{161{
202 u64 reg;
203
204 if (!kvm_vcpu_has_pmu(vcpu))162 if (!kvm_vcpu_has_pmu(vcpu))
205 return;163 return;
206 164
207 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)165 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false);
208 ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
209 __vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
210
211 /* Recreate the perf event to reflect the updated sample_period */
212 kvm_pmu_create_perf_event(vcpu, select_idx);
213}166}
214 167
215/**168/**
@@ -218,7 +171,6 @@ void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) · 2 unmodified lines
218 */171 */
219static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)172static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
220{173{
221 pmc = kvm_pmu_get_canonical_pmc(pmc);
222 if (pmc->perf_event) {174 if (pmc->perf_event) {
223 perf_event_disable(pmc->perf_event);175 perf_event_disable(pmc->perf_event);
224 perf_event_release_kernel(pmc->perf_event);176 perf_event_release_kernel(pmc->perf_event);
@@ -232,29 +184,20 @@ static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) · 7 unmodified lines
232 *184 *
233 * If this counter has been configured to monitor some event, release it here.185 * If this counter has been configured to monitor some event, release it here.
234 */186 */
235static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)187static void kvm_pmu_stop_counter(struct kvm_pmc *pmc)
236{188{
237 u64 counter, reg, val;189 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
190 u64 reg, val;
238 191
239 pmc = kvm_pmu_get_canonical_pmc(pmc);
240 if (!pmc->perf_event)192 if (!pmc->perf_event)
241 return;193 return;
242 194
243 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);195 val = kvm_pmu_get_pmc_value(pmc);
244 196
245 if (pmc->idx == ARMV8_PMU_CYCLE_IDX) {197 reg = counter_index_to_reg(pmc->idx);
246 reg = PMCCNTR_EL0;
247 val = counter;
248 } else {
249 reg = PMEVCNTR0_EL0 + pmc->idx;
250 val = lower_32_bits(counter);
251 }
252 198
253 __vcpu_sys_reg(vcpu, reg) = val;199 __vcpu_sys_reg(vcpu, reg) = val;
254 200
255 if (kvm_pmu_pmc_is_chained(pmc))
256 __vcpu_sys_reg(vcpu, reg + 1) = upper_32_bits(counter);
257
258 kvm_pmu_release_perf_event(pmc);201 kvm_pmu_release_perf_event(pmc);
259}202}
260 203
@@ -280,13 +223,10 @@ void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) · 19 unmodified lines
280void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)223void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
281{224{
282 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);225 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
283 struct kvm_pmu *pmu = &vcpu->arch.pmu;
284 int i;226 int i;
285 227
286 for_each_set_bit(i, &mask, 32)228 for_each_set_bit(i, &mask, 32)
287 kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);229 kvm_pmu_stop_counter(kvm_vcpu_idx_to_pmc(vcpu, i));
288
289 bitmap_zero(vcpu->arch.pmu.chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
290}230}
291 231
292/**232/**
@@ -297,10 +237,9 @@ void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) · 4 unmodified lines
297void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)237void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
298{238{
299 int i;239 int i;
300 struct kvm_pmu *pmu = &vcpu->arch.pmu;
301 240
302 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)241 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
303 kvm_pmu_release_perf_event(&pmu->pmc[i]);242 kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i));
304 irq_work_sync(&vcpu->arch.pmu.overflow_work);243 irq_work_sync(&vcpu->arch.pmu.overflow_work);
305}244}
306 245
@@ -325,9 +264,6 @@ u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) · 18 unmodified lines
325void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)264void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
326{265{
327 int i;266 int i;
328 struct kvm_pmu *pmu = &vcpu->arch.pmu;
329 struct kvm_pmc *pmc;
330
331 if (!kvm_vcpu_has_pmu(vcpu))267 if (!kvm_vcpu_has_pmu(vcpu))
332 return;268 return;
333 269
@@ -335,17 +271,16 @@ void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) · 1 unmodified line
335 return;271 return;
336 272
337 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {273 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
274 struct kvm_pmc *pmc;
275
338 if (!(val & BIT(i)))276 if (!(val & BIT(i)))
339 continue;277 continue;
340 278
341 pmc = &pmu->pmc[i];279 pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
342
343 /* A change in the enable state may affect the chain state */
344 kvm_pmu_update_pmc_chained(vcpu, i);
345 kvm_pmu_create_perf_event(vcpu, i);
346 280
347 /* At this point, pmc must be the canonical */281 if (!pmc->perf_event) {
348 if (pmc->perf_event) {282 kvm_pmu_create_perf_event(pmc);
283 } else {
349 perf_event_enable(pmc->perf_event);284 perf_event_enable(pmc->perf_event);
350 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)285 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
351 kvm_debug("fail to enable perf event\n");286 kvm_debug("fail to enable perf event\n");
@@ -363,23 +298,18 @@ void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) · 11 unmodified lines
363void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)298void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
364{299{
365 int i;300 int i;
366 struct kvm_pmu *pmu = &vcpu->arch.pmu;
367 struct kvm_pmc *pmc;
368 301
369 if (!kvm_vcpu_has_pmu(vcpu) || !val)302 if (!kvm_vcpu_has_pmu(vcpu) || !val)
370 return;303 return;
371 304
372 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {305 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
306 struct kvm_pmc *pmc;
307
373 if (!(val & BIT(i)))308 if (!(val & BIT(i)))
374 continue;309 continue;
375 310
376 pmc = &pmu->pmc[i];311 pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
377
378 /* A change in the enable state may affect the chain state */
379 kvm_pmu_update_pmc_chained(vcpu, i);
380 kvm_pmu_create_perf_event(vcpu, i);
381 312
382 /* At this point, pmc must be the canonical */
383 if (pmc->perf_event)313 if (pmc->perf_event)
384 perf_event_disable(pmc->perf_event);314 perf_event_disable(pmc->perf_event);
385 }315 }
@@ -476,14 +406,69 @@ void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) · 90 unmodified lines
476static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work)406static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work)
477{407{
478 struct kvm_vcpu *vcpu;408 struct kvm_vcpu *vcpu;
479 struct kvm_pmu *pmu;
480
481 pmu = container_of(work, struct kvm_pmu, overflow_work);
482 vcpu = kvm_pmc_to_vcpu(pmu->pmc);
483 409
410 vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work);
484 kvm_vcpu_kick(vcpu);411 kvm_vcpu_kick(vcpu);
485}412}
486 413
414/*
415 * Perform an increment on any of the counters described in @mask,
416 * generating the overflow if required, and propagate it as a chained
417 * event if possible.
418 */
419static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu,
420 unsigned long mask, u32 event)
421{
422 int i;
423
424 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
425 return;
426
427 /* Weed out disabled counters */
428 mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
429
430 for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) {
431 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
432 u64 type, reg;
433
434 /* Filter on event type */
435 type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i));
436 type &= kvm_pmu_event_mask(vcpu->kvm);
437 if (type != event)
438 continue;
439
440 /* Increment this counter */
441 reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1;
442 if (!kvm_pmc_is_64bit(pmc))
443 reg = lower_32_bits(reg);
444 __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) = reg;
445
446 /* No overflow? move on */
447 if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg))
448 continue;
449
450 /* Mark overflow */
451 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
452
453 if (kvm_pmu_counter_can_chain(pmc))
454 kvm_pmu_counter_increment(vcpu, BIT(i + 1),
455 ARMV8_PMUV3_PERFCTR_CHAIN);
456 }
457}
458
459/* Compute the sample period for a given counter value */
460static u64 compute_period(struct kvm_pmc *pmc, u64 counter)
461{
462 u64 val;
463
464 if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc))
465 val = (-counter) & GENMASK(63, 0);
466 else
467 val = (-counter) & GENMASK(31, 0);
468
469 return val;
470}
471
487/**472/**
488 * When the perf event overflows, set the overflow status and inform the vcpu.473 * When the perf event overflows, set the overflow status and inform the vcpu.
489 */474 */
@@ -503,10 +488,7 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 13 unmodified lines
503 * Reset the sample period to the architectural limit,488 * Reset the sample period to the architectural limit,
504 * i.e. the point where the counter overflows.489 * i.e. the point where the counter overflows.
505 */490 */
506 period = -(local64_read(&perf_event->count));491 period = compute_period(pmc, local64_read(&perf_event->count));
507
508 if (!kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
509 period &= GENMASK(31, 0);
510 492
511 local64_set(&perf_event->hw.period_left, 0);493 local64_set(&perf_event->hw.period_left, 0);
512 perf_event->attr.sample_period = period;494 perf_event->attr.sample_period = period;
@@ -514,6 +496,10 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 1 unmodified line
514 496
515 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);497 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
516 498
499 if (kvm_pmu_counter_can_chain(pmc))
500 kvm_pmu_counter_increment(vcpu, BIT(idx + 1),
501 ARMV8_PMUV3_PERFCTR_CHAIN);
502
517 if (kvm_pmu_overflow_status(vcpu)) {503 if (kvm_pmu_overflow_status(vcpu)) {
518 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);504 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
519 505
@@ -533,50 +519,7 @@ static void kvm_pmu_perf_overflow(struct perf_event *perf_event, · 13 unmodified lines
533 */519 */
534void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)520void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
535{521{
536 struct kvm_pmu *pmu = &vcpu->arch.pmu;522 kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR);
537 int i;
538
539 if (!kvm_vcpu_has_pmu(vcpu))
540 return;
541
542 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
543 return;
544
545 /* Weed out disabled counters */
546 val &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
547
548 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
549 u64 type, reg;
550
551 if (!(val & BIT(i)))
552 continue;
553
554 /* PMSWINC only applies to ... SW_INC! */
555 type = __vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i);
556 type &= kvm_pmu_event_mask(vcpu->kvm);
557 if (type != ARMV8_PMUV3_PERFCTR_SW_INCR)
558 continue;
559
560 /* increment this even SW_INC counter */
561 reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
562 reg = lower_32_bits(reg);
563 __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
564
565 if (reg) /* no overflow on the low part */
566 continue;
567
568 if (kvm_pmu_pmc_is_chained(&pmu->pmc[i])) {
569 /* increment the high counter */
570 reg = __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) + 1;
571 reg = lower_32_bits(reg);
572 __vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i + 1) = reg;
573 if (!reg) /* mark overflow on the high counter */
574 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i + 1);
575 } else {
576 /* mark overflow on low counter */
577 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
578 }
579 }
580}523}
581 524
582/**525/**
@@ -591,6 +534,12 @@ void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) · 8 unmodified lines
591 if (!kvm_vcpu_has_pmu(vcpu))534 if (!kvm_vcpu_has_pmu(vcpu))
592 return;535 return;
593 536
537 /* Fixup PMCR_EL0 to reconcile the PMU version and the LP bit */
538 if (!kvm_pmu_is_3p5(vcpu))
539 val &= ~ARMV8_PMU_PMCR_LP;
540
541 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
542
594 if (val & ARMV8_PMU_PMCR_E) {543 if (val & ARMV8_PMU_PMCR_E) {
595 kvm_pmu_enable_counter_mask(vcpu,544 kvm_pmu_enable_counter_mask(vcpu,
596 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0));545 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0));
@@ -606,49 +555,44 @@ void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) · 9 unmodified lines
606 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);555 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
607 mask &= ~BIT(ARMV8_PMU_CYCLE_IDX);556 mask &= ~BIT(ARMV8_PMU_CYCLE_IDX);
608 for_each_set_bit(i, &mask, 32)557 for_each_set_bit(i, &mask, 32)
609 kvm_pmu_set_counter_value(vcpu, i, 0);558 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true);
610 }559 }
611}560}
612 561
613static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)562static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc)
614{563{
564 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
615 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&565 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
616 (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));566 (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx));
617}567}
618 568
619/**569/**
620 * kvm_pmu_create_perf_event - create a perf event for a counter570 * kvm_pmu_create_perf_event - create a perf event for a counter
621 * @vcpu: The vcpu pointer571 * @pmc: Counter context
622 * @select_idx: The number of selected counter
623 */572 */
624static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx)573static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc)
625{574{
575 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
626 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu;576 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu;
627 struct kvm_pmu *pmu = &vcpu->arch.pmu;
628 struct kvm_pmc *pmc;
629 struct perf_event *event;577 struct perf_event *event;
630 struct perf_event_attr attr;578 struct perf_event_attr attr;
631 u64 eventsel, counter, reg, data;579 u64 eventsel, reg, data;
632 580
633 /*581 reg = counter_index_to_evtreg(pmc->idx);
634 * For chained counters the event type and filtering attributes are
635 * obtained from the low/even counter. We also use this counter to
636 * determine if the event is enabled/disabled.
637 */
638 pmc = kvm_pmu_get_canonical_pmc(&pmu->pmc[select_idx]);
639
640 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
641 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + pmc->idx;
642 data = __vcpu_sys_reg(vcpu, reg);582 data = __vcpu_sys_reg(vcpu, reg);
643 583
644 kvm_pmu_stop_counter(vcpu, pmc);584 kvm_pmu_stop_counter(pmc);
645 if (pmc->idx == ARMV8_PMU_CYCLE_IDX)585 if (pmc->idx == ARMV8_PMU_CYCLE_IDX)
646 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES;586 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES;
647 else587 else
648 eventsel = data & kvm_pmu_event_mask(vcpu->kvm);588 eventsel = data & kvm_pmu_event_mask(vcpu->kvm);
649 589
650 /* Software increment event doesn't need to be backed by a perf event */590 /*
651 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR)591 * Neither SW increment nor chained events need to be backed
592 * by a perf event.
593 */
594 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR ||
595 eventsel == ARMV8_PMUV3_PERFCTR_CHAIN)
652 return;596 return;
653 597
654 /*598 /*
@@ -663,37 +607,25 @@ static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx) · 8 unmodified lines
663 attr.type = arm_pmu->pmu.type;607 attr.type = arm_pmu->pmu.type;
664 attr.size = sizeof(attr);608 attr.size = sizeof(attr);
665 attr.pinned = 1;609 attr.pinned = 1;
666 attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, pmc->idx);610 attr.disabled = !kvm_pmu_counter_is_enabled(pmc);
667 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;611 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
668 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;612 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
669 attr.exclude_hv = 1; /* Don't count EL2 events */613 attr.exclude_hv = 1; /* Don't count EL2 events */
670 attr.exclude_host = 1; /* Don't count host events */614 attr.exclude_host = 1; /* Don't count host events */
671 attr.config = eventsel;615 attr.config = eventsel;
672 616
673 counter = kvm_pmu_get_pair_counter_value(vcpu, pmc);617 /*
674 618 * If counting with a 64bit counter, advertise it to the perf
675 if (kvm_pmu_pmc_is_chained(pmc)) {619 * code, carefully dealing with the initial sample period
676 /**620 * which also depends on the overflow.
677 * The initial sample period (overflow count) of an event. For621 */
678 * chained counters we only support overflow interrupts on the622 if (kvm_pmc_is_64bit(pmc))
679 * high counter.623 attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT;
680 */
681 attr.sample_period = (-counter) & GENMASK(63, 0);
682 attr.config1 |= PERF_ATTR_CFG1_KVM_PMU_CHAINED;
683 624
684 event = perf_event_create_kernel_counter(&attr, -1, current,625 attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc));
685 kvm_pmu_perf_overflow,
686 pmc + 1);
687 } else {
688 /* The initial sample period (overflow count) of an event. */
689 if (kvm_pmu_idx_is_64bit(vcpu, pmc->idx))
690 attr.sample_period = (-counter) & GENMASK(63, 0);
691 else
692 attr.sample_period = (-counter) & GENMASK(31, 0);
693 626
694 event = perf_event_create_kernel_counter(&attr, -1, current,627 event = perf_event_create_kernel_counter(&attr, -1, current,
695 kvm_pmu_perf_overflow, pmc);628 kvm_pmu_perf_overflow, pmc);
696 }
697 629
698 if (IS_ERR(event)) {630 if (IS_ERR(event)) {
699 pr_err_once("kvm: pmu event creation failed %ld\n",631 pr_err_once("kvm: pmu event creation failed %ld\n",
@@ -704,41 +636,6 @@ static void kvm_pmu_create_perf_event(struct kvm_vcpu *vcpu, u64 select_idx) · 4 unmodified lines
704 pmc->perf_event = event;636 pmc->perf_event = event;
705}637}
706 638
707/**
708 * kvm_pmu_update_pmc_chained - update chained bitmap
709 * @vcpu: The vcpu pointer
710 * @select_idx: The number of selected counter
711 *
712 * Update the chained bitmap based on the event type written in the
713 * typer register and the enable state of the odd register.
714 */
715static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx)
716{
717 struct kvm_pmu *pmu = &vcpu->arch.pmu;
718 struct kvm_pmc *pmc = &pmu->pmc[select_idx], *canonical_pmc;
719 bool new_state, old_state;
720
721 old_state = kvm_pmu_pmc_is_chained(pmc);
722 new_state = kvm_pmu_idx_has_chain_evtype(vcpu, pmc->idx) &&
723 kvm_pmu_counter_is_enabled(vcpu, pmc->idx | 0x1);
724
725 if (old_state == new_state)
726 return;
727
728 canonical_pmc = kvm_pmu_get_canonical_pmc(pmc);
729 kvm_pmu_stop_counter(vcpu, canonical_pmc);
730 if (new_state) {
731 /*
732 * During promotion from !chained to chained we must ensure
733 * the adjacent counter is stopped and its event destroyed
734 */
735 kvm_pmu_stop_counter(vcpu, kvm_pmu_get_alternate_pmc(pmc));
736 set_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
737 return;
738 }
739 clear_bit(pmc->idx >> 1, vcpu->arch.pmu.chained);
740}
741
742/**639/**
743 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event640 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
744 * @vcpu: The vcpu pointer641 * @vcpu: The vcpu pointer
@@ -752,6 +649,7 @@ static void kvm_pmu_update_pmc_chained(struct kvm_vcpu *vcpu, u64 select_idx) · 7 unmodified lines
752void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,649void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
753 u64 select_idx)650 u64 select_idx)
754{651{
652 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx);
755 u64 reg, mask;653 u64 reg, mask;
756 654
757 if (!kvm_vcpu_has_pmu(vcpu))655 if (!kvm_vcpu_has_pmu(vcpu))
@@ -761,20 +659,19 @@ void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, · 3 unmodified lines
761 mask &= ~ARMV8_PMU_EVTYPE_EVENT;659 mask &= ~ARMV8_PMU_EVTYPE_EVENT;
762 mask |= kvm_pmu_event_mask(vcpu->kvm);660 mask |= kvm_pmu_event_mask(vcpu->kvm);
763 661
764 reg = (select_idx == ARMV8_PMU_CYCLE_IDX)662 reg = counter_index_to_evtreg(pmc->idx);
765 ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + select_idx;
766 663
767 __vcpu_sys_reg(vcpu, reg) = data & mask;664 __vcpu_sys_reg(vcpu, reg) = data & mask;
768 665
769 kvm_pmu_update_pmc_chained(vcpu, select_idx);666 kvm_pmu_create_perf_event(pmc);
770 kvm_pmu_create_perf_event(vcpu, select_idx);
771}667}
772 668
773void kvm_host_pmu_init(struct arm_pmu *pmu)669void kvm_host_pmu_init(struct arm_pmu *pmu)
774{670{
775 struct arm_pmu_entry *entry;671 struct arm_pmu_entry *entry;
776 672
777 if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)673 if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
674 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
778 return;675 return;
779 676
780 mutex_lock(&arm_pmus_lock);677 mutex_lock(&arm_pmus_lock);
@@ -827,7 +724,7 @@ static struct arm_pmu *kvm_pmu_probe_armpmu(void) · 46 unmodified lines
827 724
828 if (event->pmu) {725 if (event->pmu) {
829 pmu = to_arm_pmu(event->pmu);726 pmu = to_arm_pmu(event->pmu);
830 if (pmu->pmuver == 0 ||727 if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
831 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)728 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
832 pmu = NULL;729 pmu = NULL;
833 }730 }
@@ -849,6 +746,8 @@ u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) · 15 unmodified lines
849 746
850 if (!pmceid1) {747 if (!pmceid1) {
851 val = read_sysreg(pmceid0_el0);748 val = read_sysreg(pmceid0_el0);
749 /* always support CHAIN */
750 val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN);
852 base = 0;751 base = 0;
853 } else {752 } else {
854 val = read_sysreg(pmceid1_el0);753 val = read_sysreg(pmceid1_el0);
@@ -1150,3 +1049,14 @@ int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) · 295 unmodified lines
1150 1049
1151 return -ENXIO;1050 return -ENXIO;
1152}1051}
1052
1053u8 kvm_arm_pmu_get_pmuver_limit(void)
1054{
1055 u64 tmp;
1056
1057 tmp = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1058 tmp = cpuid_feature_cap_perfmon_field(tmp,
1059 ID_AA64DFR0_EL1_PMUVer_SHIFT,
1060 ID_AA64DFR0_EL1_PMUVer_V3P5);
1061 return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), tmp);
1062}
arch/arm64/kvm/reset.c+0−29
@@ -395,32 +395,3 @@ int kvm_set_ipa_limit(void) · 394 unmodified lines
395 395
396 return 0;396 return 0;
397}397}
398
399int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type)
400{
401 u64 mmfr0, mmfr1;
402 u32 phys_shift;
403
404 if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
405 return -EINVAL;
406
407 phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type);
408 if (phys_shift) {
409 if (phys_shift > kvm_ipa_limit ||
410 phys_shift < ARM64_MIN_PARANGE_BITS)
411 return -EINVAL;
412 } else {
413 phys_shift = KVM_PHYS_SHIFT;
414 if (phys_shift > kvm_ipa_limit) {
415 pr_warn_once("%s using unsupported default IPA limit, upgrade your VMM\n",
416 current->comm);
417 return -EINVAL;
418 }
419 }
420
421 mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
422 mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
423 kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
424
425 return 0;
426}
arch/arm64/kvm/sys_regs.c+135−22
@@ -639,22 +639,18 @@ static void reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) · 638 unmodified lines
639 639
640static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)640static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
641{641{
642 u64 pmcr, val;642 u64 pmcr;
643 643
644 /* No PMU available, PMCR_EL0 may UNDEF... */644 /* No PMU available, PMCR_EL0 may UNDEF... */
645 if (!kvm_arm_support_pmu_v3())645 if (!kvm_arm_support_pmu_v3())
646 return;646 return;
647 647
648 pmcr = read_sysreg(pmcr_el0);648 /* Only preserve PMCR_EL0.N, and reset the rest to 0 */
649 /*649 pmcr = read_sysreg(pmcr_el0) & ARMV8_PMU_PMCR_N_MASK;
650 * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
651 * except PMCR.E resetting to zero.
652 */
653 val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
654 | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
655 if (!kvm_supports_32bit_el0())650 if (!kvm_supports_32bit_el0())
656 val |= ARMV8_PMU_PMCR_LC;651 pmcr |= ARMV8_PMU_PMCR_LC;
657 __vcpu_sys_reg(vcpu, r->reg) = val;652
653 __vcpu_sys_reg(vcpu, r->reg) = pmcr;
658}654}
659 655
660static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)656static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
@@ -697,13 +693,15 @@ static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · 36 unmodified lines
697 return false;693 return false;
698 694
699 if (p->is_write) {695 if (p->is_write) {
700 /* Only update writeable bits of PMCR */696 /*
697 * Only update writeable bits of PMCR (continuing into
698 * kvm_pmu_handle_pmcr() as well)
699 */
701 val = __vcpu_sys_reg(vcpu, PMCR_EL0);700 val = __vcpu_sys_reg(vcpu, PMCR_EL0);
702 val &= ~ARMV8_PMU_PMCR_MASK;701 val &= ~ARMV8_PMU_PMCR_MASK;
703 val |= p->regval & ARMV8_PMU_PMCR_MASK;702 val |= p->regval & ARMV8_PMU_PMCR_MASK;
704 if (!kvm_supports_32bit_el0())703 if (!kvm_supports_32bit_el0())
705 val |= ARMV8_PMU_PMCR_LC;704 val |= ARMV8_PMU_PMCR_LC;
706 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
707 kvm_pmu_handle_pmcr(vcpu, val);705 kvm_pmu_handle_pmcr(vcpu, val);
708 kvm_vcpu_pmu_restore_guest(vcpu);706 kvm_vcpu_pmu_restore_guest(vcpu);
709 } else {707 } else {
@@ -1062,6 +1060,40 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, · 352 unmodified lines
1062 return true;1060 return true;
1063}1061}
1064 1062
1063static u8 vcpu_pmuver(const struct kvm_vcpu *vcpu)
1064{
1065 if (kvm_vcpu_has_pmu(vcpu))
1066 return vcpu->kvm->arch.dfr0_pmuver.imp;
1067
1068 return vcpu->kvm->arch.dfr0_pmuver.unimp;
1069}
1070
1071static u8 perfmon_to_pmuver(u8 perfmon)
1072{
1073 switch (perfmon) {
1074 case ID_DFR0_EL1_PerfMon_PMUv3:
1075 return ID_AA64DFR0_EL1_PMUVer_IMP;
1076 case ID_DFR0_EL1_PerfMon_IMPDEF:
1077 return ID_AA64DFR0_EL1_PMUVer_IMP_DEF;
1078 default:
1079 /* Anything ARMv8.1+ and NI have the same value. For now. */
1080 return perfmon;
1081 }
1082}
1083
1084static u8 pmuver_to_perfmon(u8 pmuver)
1085{
1086 switch (pmuver) {
1087 case ID_AA64DFR0_EL1_PMUVer_IMP:
1088 return ID_DFR0_EL1_PerfMon_PMUv3;
1089 case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
1090 return ID_DFR0_EL1_PerfMon_IMPDEF;
1091 default:
1092 /* Anything ARMv8.1+ and NI have the same value. For now. */
1093 return pmuver;
1094 }
1095}
1096
1065/* Read a sanitised cpufeature ID register by sys_reg_desc */1097/* Read a sanitised cpufeature ID register by sys_reg_desc */
1066static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r)1098static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r)
1067{1099{
@@ -1111,18 +1143,17 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r · 43 unmodified lines
1111 /* Limit debug to ARMv8.0 */1143 /* Limit debug to ARMv8.0 */
1112 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer);1144 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer);
1113 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6);1145 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6);
1114 /* Limit guests to PMUv3 for ARMv8.4 */1146 /* Set PMUver to the required version */
1115 val = cpuid_feature_cap_perfmon_field(val,1147 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer);
1116 ID_AA64DFR0_EL1_PMUVer_SHIFT,1148 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer),
1117 kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_EL1_PMUVer_V3P4 : 0);1149 vcpu_pmuver(vcpu));
1118 /* Hide SPE from guests */1150 /* Hide SPE from guests */
1119 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer);1151 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer);
1120 break;1152 break;
1121 case SYS_ID_DFR0_EL1:1153 case SYS_ID_DFR0_EL1:
1122 /* Limit guests to PMUv3 for ARMv8.4 */1154 val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon);
1123 val = cpuid_feature_cap_perfmon_field(val,1155 val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon),
1124 ID_DFR0_PERFMON_SHIFT,1156 pmuver_to_perfmon(vcpu_pmuver(vcpu)));
1125 kvm_vcpu_has_pmu(vcpu) ? ID_DFR0_PERFMON_8_4 : 0);
1126 break;1157 break;
1127 }1158 }
1128 1159
@@ -1222,6 +1253,85 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, · 93 unmodified lines
1222 return 0;1253 return 0;
1223}1254}
1224 1255
1256static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1257 const struct sys_reg_desc *rd,
1258 u64 val)
1259{
1260 u8 pmuver, host_pmuver;
1261 bool valid_pmu;
1262
1263 host_pmuver = kvm_arm_pmu_get_pmuver_limit();
1264
1265 /*
1266 * Allow AA64DFR0_EL1.PMUver to be set from userspace as long
1267 * as it doesn't promise more than what the HW gives us. We
1268 * allow an IMPDEF PMU though, only if no PMU is supported
1269 * (KVM backward compatibility handling).
1270 */
1271 pmuver = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), val);
1272 if ((pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF && pmuver > host_pmuver))
1273 return -EINVAL;
1274
1275 valid_pmu = (pmuver != 0 && pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF);
1276
1277 /* Make sure view register and PMU support do match */
1278 if (kvm_vcpu_has_pmu(vcpu) != valid_pmu)
1279 return -EINVAL;
1280
1281 /* We can only differ with PMUver, and anything else is an error */
1282 val ^= read_id_reg(vcpu, rd);
1283 val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer);
1284 if (val)
1285 return -EINVAL;
1286
1287 if (valid_pmu)
1288 vcpu->kvm->arch.dfr0_pmuver.imp = pmuver;
1289 else
1290 vcpu->kvm->arch.dfr0_pmuver.unimp = pmuver;
1291
1292 return 0;
1293}
1294
1295static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1296 const struct sys_reg_desc *rd,
1297 u64 val)
1298{
1299 u8 perfmon, host_perfmon;
1300 bool valid_pmu;
1301
1302 host_perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1303
1304 /*
1305 * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1306 * it doesn't promise more than what the HW gives us on the
1307 * AArch64 side (as everything is emulated with that), and
1308 * that this is a PMUv3.
1309 */
1310 perfmon = FIELD_GET(ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon), val);
1311 if ((perfmon != ID_DFR0_EL1_PerfMon_IMPDEF && perfmon > host_perfmon) ||
1312 (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3))
1313 return -EINVAL;
1314
1315 valid_pmu = (perfmon != 0 && perfmon != ID_DFR0_EL1_PerfMon_IMPDEF);
1316
1317 /* Make sure view register and PMU support do match */
1318 if (kvm_vcpu_has_pmu(vcpu) != valid_pmu)
1319 return -EINVAL;
1320
1321 /* We can only differ with PerfMon, and anything else is an error */
1322 val ^= read_id_reg(vcpu, rd);
1323 val &= ~ARM64_FEATURE_MASK(ID_DFR0_EL1_PerfMon);
1324 if (val)
1325 return -EINVAL;
1326
1327 if (valid_pmu)
1328 vcpu->kvm->arch.dfr0_pmuver.imp = perfmon_to_pmuver(perfmon);
1329 else
1330 vcpu->kvm->arch.dfr0_pmuver.unimp = perfmon_to_pmuver(perfmon);
1331
1332 return 0;
1333}
1334
1225/*1335/*
1226 * cpufeature ID register user accessors1336 * cpufeature ID register user accessors
1227 *1337 *
@@ -1443,7 +1553,9 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 215 unmodified lines
1443 /* CRm=1 */1553 /* CRm=1 */
1444 AA32_ID_SANITISED(ID_PFR0_EL1),1554 AA32_ID_SANITISED(ID_PFR0_EL1),
1445 AA32_ID_SANITISED(ID_PFR1_EL1),1555 AA32_ID_SANITISED(ID_PFR1_EL1),
1446 AA32_ID_SANITISED(ID_DFR0_EL1),1556 { SYS_DESC(SYS_ID_DFR0_EL1), .access = access_id_reg,
1557 .get_user = get_id_reg, .set_user = set_id_dfr0_el1,
1558 .visibility = aa32_id_visibility, },
1447 ID_HIDDEN(ID_AFR0_EL1),1559 ID_HIDDEN(ID_AFR0_EL1),
1448 AA32_ID_SANITISED(ID_MMFR0_EL1),1560 AA32_ID_SANITISED(ID_MMFR0_EL1),
1449 AA32_ID_SANITISED(ID_MMFR1_EL1),1561 AA32_ID_SANITISED(ID_MMFR1_EL1),
@@ -1483,7 +1595,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 33 unmodified lines
1483 ID_UNALLOCATED(4,7),1595 ID_UNALLOCATED(4,7),
1484 1596
1485 /* CRm=5 */1597 /* CRm=5 */
1486 ID_SANITISED(ID_AA64DFR0_EL1),1598 { SYS_DESC(SYS_ID_AA64DFR0_EL1), .access = access_id_reg,
1599 .get_user = get_id_reg, .set_user = set_id_aa64dfr0_el1, },
1487 ID_SANITISED(ID_AA64DFR1_EL1),1600 ID_SANITISED(ID_AA64DFR1_EL1),
1488 ID_UNALLOCATED(5,2),1601 ID_UNALLOCATED(5,2),
1489 ID_UNALLOCATED(5,3),1602 ID_UNALLOCATED(5,3),
arch/arm64/kvm/vgic/vgic-its.c+20−0
@@ -2743,6 +2743,7 @@ static int vgic_its_has_attr(struct kvm_device *dev, · 2742 unmodified lines
2743static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)2743static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2744{2744{
2745 const struct vgic_its_abi *abi = vgic_its_get_abi(its);2745 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2746 struct vgic_dist *dist = &kvm->arch.vgic;
2746 int ret = 0;2747 int ret = 0;
2747 2748
2748 if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */2749 if (attr == KVM_DEV_ARM_VGIC_CTRL_INIT) /* Nothing to do */
@@ -2762,7 +2763,9 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr) · 13 unmodified lines
2762 vgic_its_reset(kvm, its);2763 vgic_its_reset(kvm, its);
2763 break;2764 break;
2764 case KVM_DEV_ARM_ITS_SAVE_TABLES:2765 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2766 dist->save_its_tables_in_progress = true;
2765 ret = abi->save_tables(its);2767 ret = abi->save_tables(its);
2768 dist->save_its_tables_in_progress = false;
2766 break;2769 break;
2767 case KVM_DEV_ARM_ITS_RESTORE_TABLES:2770 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2768 ret = abi->restore_tables(its);2771 ret = abi->restore_tables(its);
@@ -2775,6 +2778,23 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr) · 6 unmodified lines
2775 return ret;2778 return ret;
2776}2779}
2777 2780
2781/*
2782 * kvm_arch_allow_write_without_running_vcpu - allow writing guest memory
2783 * without the running VCPU when dirty ring is enabled.
2784 *
2785 * The running VCPU is required to track dirty guest pages when dirty ring
2786 * is enabled. Otherwise, the backup bitmap should be used to track the
2787 * dirty guest pages. When vgic/its tables are being saved, the backup
2788 * bitmap is used to track the dirty guest pages due to the missed running
2789 * VCPU in the period.
2790 */
2791bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
2792{
2793 struct vgic_dist *dist = &kvm->arch.vgic;
2794
2795 return dist->save_its_tables_in_progress;
2796}
2797
2778static int vgic_its_set_attr(struct kvm_device *dev,2798static int vgic_its_set_attr(struct kvm_device *dev,
2779 struct kvm_device_attr *attr)2799 struct kvm_device_attr *attr)
2780{2800{
arch/arm64/lib/mte.S+1−1
@@ -18,7 +18,7 @@ · 17 unmodified lines
18 */18 */
19 .macro multitag_transfer_size, reg, tmp19 .macro multitag_transfer_size, reg, tmp
20 mrs_s \reg, SYS_GMID_EL120 mrs_s \reg, SYS_GMID_EL1
21 ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_SIZE21 ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
22 mov \tmp, #422 mov \tmp, #4
23 lsl \reg, \tmp, \reg23 lsl \reg, \tmp, \reg
24 .endm24 .endm
arch/arm64/mm/copypage.c+5−2
@@ -21,9 +21,12 @@ void copy_highpage(struct page *to, struct page *from) · 20 unmodified lines
21 21
22 copy_page(kto, kfrom);22 copy_page(kto, kfrom);
23 23
24 if (system_supports_mte() && test_bit(PG_mte_tagged, &from->flags)) {24 if (system_supports_mte() && page_mte_tagged(from)) {
25 set_bit(PG_mte_tagged, &to->flags);25 page_kasan_tag_reset(to);
26 /* It's a new page, shouldn't have been tagged yet */
27 WARN_ON_ONCE(!try_page_mte_tagging(to));
26 mte_copy_page_tags(kto, kfrom);28 mte_copy_page_tags(kto, kfrom);
29 set_page_mte_tagged(to);
27 }30 }
28}31}
29EXPORT_SYMBOL(copy_highpage);32EXPORT_SYMBOL(copy_highpage);
arch/arm64/mm/fault.c+3−1
@@ -937,6 +937,8 @@ struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, · 936 unmodified lines
937 937
938void tag_clear_highpage(struct page *page)938void tag_clear_highpage(struct page *page)
939{939{
940 /* Newly allocated page, shouldn't have been tagged yet */
941 WARN_ON_ONCE(!try_page_mte_tagging(page));
940 mte_zero_clear_page_tags(page_address(page));942 mte_zero_clear_page_tags(page_address(page));
941 set_bit(PG_mte_tagged, &page->flags);943 set_page_mte_tagged(page);
942}944}
arch/arm64/mm/mteswap.c+6−10
@@ -24,7 +24,7 @@ int mte_save_tags(struct page *page) · 23 unmodified lines
24{24{
25 void *tag_storage, *ret;25 void *tag_storage, *ret;
26 26
27 if (!test_bit(PG_mte_tagged, &page->flags))27 if (!page_mte_tagged(page))
28 return 0;28 return 0;
29 29
30 tag_storage = mte_allocate_tag_storage();30 tag_storage = mte_allocate_tag_storage();
@@ -46,21 +46,17 @@ int mte_save_tags(struct page *page) · 15 unmodified lines
46 return 0;46 return 0;
47}47}
48 48
49bool mte_restore_tags(swp_entry_t entry, struct page *page)49void mte_restore_tags(swp_entry_t entry, struct page *page)
50{50{
51 void *tags = xa_load(&mte_pages, entry.val);51 void *tags = xa_load(&mte_pages, entry.val);
52 52
53 if (!tags)53 if (!tags)
54 return false;54 return;
55 55
56 /*56 if (try_page_mte_tagging(page)) {
57 * Test PG_mte_tagged again in case it was racing with another
58 * set_pte_at().
59 */
60 if (!test_and_set_bit(PG_mte_tagged, &page->flags))
61 mte_restore_page_tags(page_address(page), tags);57 mte_restore_page_tags(page_address(page), tags);
62 58 set_page_mte_tagged(page);
63 return true;59 }
64}60}
65 61
66void mte_invalidate_tags(int type, pgoff_t offset)62void mte_invalidate_tags(int type, pgoff_t offset)
arch/arm64/tools/gen-sysreg.awk+1−1
@@ -33,7 +33,7 @@ function expect_fields(nf) { · 32 unmodified lines
33# Print a CPP macro definition, padded with spaces so that the macro bodies33# Print a CPP macro definition, padded with spaces so that the macro bodies
34# line up in a column34# line up in a column
35function define(name, val) {35function define(name, val) {
36 printf "%-48s%s\n", "#define " name, val36 printf "%-56s%s\n", "#define " name, val
37}37}
38 38
39# Print standard BITMASK/SHIFT/WIDTH CPP definitions for a field39# Print standard BITMASK/SHIFT/WIDTH CPP definitions for a field
arch/arm64/tools/sysreg+754−0
@@ -46,6 +46,760 @@ · 45 unmodified lines
46# feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration46# feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
47# item ACCDATA) though it may be more taseful to do something else.47# item ACCDATA) though it may be more taseful to do something else.
48 48
49Sysreg ID_PFR0_EL1 3 0 0 1 0
50Res0 63:32
51Enum 31:28 RAS
52 0b0000 NI
53 0b0001 RAS
54 0b0010 RASv1p1
55EndEnum
56Enum 27:24 DIT
57 0b0000 NI
58 0b0001 IMP
59EndEnum
60Enum 23:20 AMU
61 0b0000 NI
62 0b0001 AMUv1
63 0b0010 AMUv1p1
64EndEnum
65Enum 19:16 CSV2
66 0b0000 UNDISCLOSED
67 0b0001 IMP
68 0b0010 CSV2p1
69EndEnum
70Enum 15:12 State3
71 0b0000 NI
72 0b0001 IMP
73EndEnum
74Enum 11:8 State2
75 0b0000 NI
76 0b0001 NO_CV
77 0b0010 CV
78EndEnum
79Enum 7:4 State1
80 0b0000 NI
81 0b0001 THUMB
82 0b0010 THUMB2
83EndEnum
84Enum 3:0 State0
85 0b0000 NI
86 0b0001 IMP
87EndEnum
88EndSysreg
89
90Sysreg ID_PFR1_EL1 3 0 0 1 1
91Res0 63:32
92Enum 31:28 GIC
93 0b0000 NI
94 0b0001 GICv3
95 0b0010 GICv4p1
96EndEnum
97Enum 27:24 Virt_frac
98 0b0000 NI
99 0b0001 IMP
100EndEnum
101Enum 23:20 Sec_frac
102 0b0000 NI
103 0b0001 WALK_DISABLE
104 0b0010 SECURE_MEMORY
105EndEnum
106Enum 19:16 GenTimer
107 0b0000 NI
108 0b0001 IMP
109 0b0010 ECV
110EndEnum
111Enum 15:12 Virtualization
112 0b0000 NI
113 0b0001 IMP
114EndEnum
115Enum 11:8 MProgMod
116 0b0000 NI
117 0b0001 IMP
118EndEnum
119Enum 7:4 Security
120 0b0000 NI
121 0b0001 EL3
122 0b0001 NSACR_RFR
123EndEnum
124Enum 3:0 ProgMod
125 0b0000 NI
126 0b0001 IMP
127EndEnum
128EndSysreg
129
130Sysreg ID_DFR0_EL1 3 0 0 1 2
131Res0 63:32
132Enum 31:28 TraceFilt
133 0b0000 NI
134 0b0001 IMP
135EndEnum
136Enum 27:24 PerfMon
137 0b0000 NI
138 0b0001 PMUv1
139 0b0010 PMUv2
140 0b0011 PMUv3
141 0b0100 PMUv3p1
142 0b0101 PMUv3p4
143 0b0110 PMUv3p5
144 0b0111 PMUv3p7
145 0b1000 PMUv3p8
146 0b1111 IMPDEF
147EndEnum
148Enum 23:20 MProfDbg
149 0b0000 NI
150 0b0001 IMP
151EndEnum
152Enum 19:16 MMapTrc
153 0b0000 NI
154 0b0001 IMP
155EndEnum
156Enum 15:12 CopTrc
157 0b0000 NI
158 0b0001 IMP
159EndEnum
160Enum 11:8 MMapDbg
161 0b0000 NI
162 0b0100 Armv7
163 0b0101 Armv7p1
164EndEnum
165Field 7:4 CopSDbg
166Enum 3:0 CopDbg
167 0b0000 NI
168 0b0010 Armv6
169 0b0011 Armv6p1
170 0b0100 Armv7
171 0b0101 Armv7p1
172 0b0110 Armv8
173 0b0111 VHE
174 0b1000 Debugv8p2
175 0b1001 Debugv8p4
176 0b1010 Debugv8p8
177EndEnum
178EndSysreg
179
180Sysreg ID_AFR0_EL1 3 0 0 1 3
181Res0 63:16
182Field 15:12 IMPDEF3
183Field 11:8 IMPDEF2
184Field 7:4 IMPDEF1
185Field 3:0 IMPDEF0
186EndSysreg
187
188Sysreg ID_MMFR0_EL1 3 0 0 1 4
189Res0 63:32
190Enum 31:28 InnerShr
191 0b0000 NC
192 0b0001 HW
193 0b1111 IGNORED
194EndEnum
195Enum 27:24 FCSE
196 0b0000 NI
197 0b0001 IMP
198EndEnum
199Enum 23:20 AuxReg
200 0b0000 NI
201 0b0001 ACTLR
202 0b0010 AIFSR
203EndEnum
204Enum 19:16 TCM
205 0b0000 NI
206 0b0001 IMPDEF
207 0b0010 TCM
208 0b0011 TCM_DMA
209EndEnum
210Enum 15:12 ShareLvl
211 0b0000 ONE
212 0b0001 TWO
213EndEnum
214Enum 11:8 OuterShr
215 0b0000 NC
216 0b0001 HW
217 0b1111 IGNORED
218EndEnum
219Enum 7:4 PMSA
220 0b0000 NI
221 0b0001 IMPDEF
222 0b0010 PMSAv6
223 0b0011 PMSAv7
224EndEnum
225Enum 3:0 VMSA
226 0b0000 NI
227 0b0001 IMPDEF
228 0b0010 VMSAv6
229 0b0011 VMSAv7
230 0b0100 VMSAv7_PXN
231 0b0101 VMSAv7_LONG
232EndEnum
233EndSysreg
234
235Sysreg ID_MMFR1_EL1 3 0 0 1 5
236Res0 63:32
237Enum 31:28 BPred
238 0b0000 NI
239 0b0001 BP_SW_MANGED
240 0b0010 BP_ASID_AWARE
241 0b0011 BP_NOSNOOP
242 0b0100 BP_INVISIBLE
243EndEnum
244Enum 27:24 L1TstCln
245 0b0000 NI
246 0b0001 NOINVALIDATE
247 0b0010 INVALIDATE
248EndEnum
249Enum 23:20 L1Uni
250 0b0000 NI
251 0b0001 INVALIDATE
252 0b0010 CLEAN_AND_INVALIDATE
253EndEnum
254Enum 19:16 L1Hvd
255 0b0000 NI
256 0b0001 INVALIDATE_ISIDE_ONLY
257 0b0010 INVALIDATE
258 0b0011 CLEAN_AND_INVALIDATE
259EndEnum
260Enum 15:12 L1UniSW
261 0b0000 NI
262 0b0001 CLEAN
263 0b0010 CLEAN_AND_INVALIDATE
264 0b0011 INVALIDATE
265EndEnum
266Enum 11:8 L1HvdSW
267 0b0000 NI
268 0b0001 CLEAN_AND_INVALIDATE
269 0b0010 INVALIDATE_DSIDE_ONLY
270 0b0011 INVALIDATE
271EndEnum
272Enum 7:4 L1UniVA
273 0b0000 NI
274 0b0001 CLEAN_AND_INVALIDATE
275 0b0010 INVALIDATE_BP
276EndEnum
277Enum 3:0 L1HvdVA
278 0b0000 NI
279 0b0001 CLEAN_AND_INVALIDATE
280 0b0010 INVALIDATE_BP
281EndEnum
282EndSysreg
283
284Sysreg ID_MMFR2_EL1 3 0 0 1 6
285Res0 63:32
286Enum 31:28 HWAccFlg
287 0b0000 NI
288 0b0001 IMP
289EndEnum
290Enum 27:24 WFIStall
291 0b0000 NI
292 0b0001 IMP
293EndEnum
294Enum 23:20 MemBarr
295 0b0000 NI
296 0b0001 DSB_ONLY
297 0b0010 IMP
298EndEnum
299Enum 19:16 UniTLB
300 0b0000 NI
301 0b0001 BY_VA
302 0b0010 BY_MATCH_ASID
303 0b0011 BY_ALL_ASID
304 0b0100 OTHER_TLBS
305 0b0101 BROADCAST
306 0b0110 BY_IPA
307EndEnum
308Enum 15:12 HvdTLB
309 0b0000 NI
310EndEnum
311Enum 11:8 L1HvdRng
312 0b0000 NI
313 0b0001 IMP
314EndEnum
315Enum 7:4 L1HvdBG
316 0b0000 NI
317 0b0001 IMP
318EndEnum
319Enum 3:0 L1HvdFG
320 0b0000 NI
321 0b0001 IMP
322EndEnum
323EndSysreg
324
325Sysreg ID_MMFR3_EL1 3 0 0 1 7
326Res0 63:32
327Enum 31:28 Supersec
328 0b0000 IMP
329 0b1111 NI
330EndEnum
331Enum 27:24 CMemSz
332 0b0000 4GB
333 0b0001 64GB
334 0b0010 1TB
335EndEnum
336Enum 23:20 CohWalk
337 0b0000 NI
338 0b0001 IMP
339EndEnum
340Enum 19:16 PAN
341 0b0000 NI
342 0b0001 PAN
343 0b0010 PAN2
344EndEnum
345Enum 15:12 MaintBcst
346 0b0000 NI
347 0b0001 NO_TLB
348 0b0010 ALL
349EndEnum
350Enum 11:8 BPMaint
351 0b0000 NI
352 0b0001 ALL
353 0b0010 BY_VA
354EndEnum
355Enum 7:4 CMaintSW
356 0b0000 NI
357 0b0001 IMP
358EndEnum
359Enum 3:0 CMaintVA
360 0b0000 NI
361 0b0001 IMP
362EndEnum
363EndSysreg
364
365Sysreg ID_ISAR0_EL1 3 0 0 2 0
366Res0 63:28
367Enum 27:24 Divide
368 0b0000 NI
369 0b0001 xDIV_T32
370 0b0010 xDIV_A32
371EndEnum
372Enum 23:20 Debug
373 0b0000 NI
374 0b0001 IMP
375EndEnum
376Enum 19:16 Coproc
377 0b0000 NI
378 0b0001 MRC
379 0b0010 MRC2
380 0b0011 MRRC
381 0b0100 MRRC2
382EndEnum
383Enum 15:12 CmpBranch
384 0b0000 NI
385 0b0001 IMP
386EndEnum
387Enum 11:8 BitField
388 0b0000 NI
389 0b0001 IMP
390EndEnum
391Enum 7:4 BitCount
392 0b0000 NI
393 0b0001 IMP
394EndEnum
395Enum 3:0 Swap
396 0b0000 NI
397 0b0001 IMP
398EndEnum
399EndSysreg
400
401Sysreg ID_ISAR1_EL1 3 0 0 2 1
402Res0 63:32
403Enum 31:28 Jazelle
404 0b0000 NI
405 0b0001 IMP
406EndEnum
407Enum 27:24 Interwork
408 0b0000 NI
409 0b0001 BX
410 0b0010 BLX
411 0b0011 A32_BX
412EndEnum
413Enum 23:20 Immediate
414 0b0000 NI
415 0b0001 IMP
416EndEnum
417Enum 19:16 IfThen
418 0b0000 NI
419 0b0001 IMP
420EndEnum
421Enum 15:12 Extend
422 0b0000 NI
423 0b0001 SXTB
424 0b0010 SXTB16
425EndEnum
426Enum 11:8 Except_AR
427 0b0000 NI
428 0b0001 IMP
429EndEnum
430Enum 7:4 Except
431 0b0000 NI
432 0b0001 IMP
433EndEnum
434Enum 3:0 Endian
435 0b0000 NI
436 0b0001 IMP
437EndEnum
438EndSysreg
439
440Sysreg ID_ISAR2_EL1 3 0 0 2 2
441Res0 63:32
442Enum 31:28 Reversal
443 0b0000 NI
444 0b0001 REV
445 0b0010 RBIT
446EndEnum
447Enum 27:24 PSR_AR
448 0b0000 NI
449 0b0001 IMP
450EndEnum
451Enum 23:20 MultU
452 0b0000 NI
453 0b0001 UMULL
454 0b0010 UMAAL
455EndEnum
456Enum 19:16 MultS
457 0b0000 NI
458 0b0001 SMULL
459 0b0010 SMLABB
460 0b0011 SMLAD
461EndEnum
462Enum 15:12 Mult
463 0b0000 NI
464 0b0001 MLA
465 0b0010 MLS
466EndEnum
467Enum 11:8 MultiAccessInt
468 0b0000 NI
469 0b0001 RESTARTABLE
470 0b0010 CONTINUABLE
471EndEnum
472Enum 7:4 MemHint
473 0b0000 NI
474 0b0001 PLD
475 0b0010 PLD2
476 0b0011 PLI
477 0b0100 PLDW
478EndEnum
479Enum 3:0 LoadStore
480 0b0000 NI
481 0b0001 DOUBLE
482 0b0010 ACQUIRE
483EndEnum
484EndSysreg
485
486Sysreg ID_ISAR3_EL1 3 0 0 2 3
487Res0 63:32
488Enum 31:28 T32EE
489 0b0000 NI
490 0b0001 IMP
491EndEnum
492Enum 27:24 TrueNOP
493 0b0000 NI
494 0b0001 IMP
495EndEnum
496Enum 23:20 T32Copy
497 0b0000 NI
498 0b0001 IMP
499EndEnum
500Enum 19:16 TabBranch
501 0b0000 NI
502 0b0001 IMP
503EndEnum
504Enum 15:12 SynchPrim
505 0b0000 NI
506 0b0001 EXCLUSIVE
507 0b0010 DOUBLE
508EndEnum
509Enum 11:8 SVC
510 0b0000 NI
511 0b0001 IMP
512EndEnum
513Enum 7:4 SIMD
514 0b0000 NI
515 0b0001 SSAT
516 0b0011 PKHBT
517EndEnum
518Enum 3:0 Saturate
519 0b0000 NI
520 0b0001 IMP
521EndEnum
522EndSysreg
523
524Sysreg ID_ISAR4_EL1 3 0 0 2 4
525Res0 63:32
526Enum 31:28 SWP_frac
527 0b0000 NI
528 0b0001 IMP
529EndEnum
530Enum 27:24 PSR_M
531 0b0000 NI
532 0b0001 IMP
533EndEnum
534Enum 23:20 SynchPrim_frac
535 0b0000 NI
536 0b0011 IMP
537EndEnum
538Enum 19:16 Barrier
539 0b0000 NI
540 0b0001 IMP
541EndEnum
542Enum 15:12 SMC
543 0b0000 NI
544 0b0001 IMP
545EndEnum
546Enum 11:8 Writeback
547 0b0000 NI
548 0b0001 IMP
549EndEnum
550Enum 7:4 WithShifts
551 0b0000 NI
552 0b0001 LSL3
553 0b0011 LS
554 0b0100 REG
555EndEnum
556Enum 3:0 Unpriv
557 0b0000 NI
558 0b0001 REG_BYTE
559 0b0010 SIGNED_HALFWORD
560EndEnum
561EndSysreg
562
563Sysreg ID_ISAR5_EL1 3 0 0 2 5
564Res0 63:32
565Enum 31:28 VCMA
566 0b0000 NI
567 0b0001 IMP
568EndEnum
569Enum 27:24 RDM
570 0b0000 NI
571 0b0001 IMP
572EndEnum
573Res0 23:20
574Enum 19:16 CRC32
575 0b0000 NI
576 0b0001 IMP
577EndEnum
578Enum 15:12 SHA2
579 0b0000 NI
580 0b0001 IMP
581EndEnum
582Enum 11:8 SHA1
583 0b0000 NI
584 0b0001 IMP
585EndEnum
586Enum 7:4 AES
587 0b0000 NI
588 0b0001 IMP
589 0b0010 VMULL
590EndEnum
591Enum 3:0 SEVL
592 0b0000 NI
593 0b0001 IMP
594EndEnum
595EndSysreg
596
597Sysreg ID_ISAR6_EL1 3 0 0 2 7
598Res0 63:28
599Enum 27:24 I8MM
600 0b0000 NI
601 0b0001 IMP
602EndEnum
603Enum 23:20 BF16
604 0b0000 NI
605 0b0001 IMP
606EndEnum
607Enum 19:16 SPECRES
608 0b0000 NI
609 0b0001 IMP
610EndEnum
611Enum 15:12 SB
612 0b0000 NI
613 0b0001 IMP
614EndEnum
615Enum 11:8 FHM
616 0b0000 NI
617 0b0001 IMP
618EndEnum
619Enum 7:4 DP
620 0b0000 NI
621 0b0001 IMP
622EndEnum
623Enum 3:0 JSCVT
624 0b0000 NI
625 0b0001 IMP
626EndEnum
627EndSysreg
628
629Sysreg ID_MMFR4_EL1 3 0 0 2 6
630Res0 63:32
631Enum 31:28 EVT
632 0b0000 NI
633 0b0001 NO_TLBIS
634 0b0010 TLBIS
635EndEnum
636Enum 27:24 CCIDX
637 0b0000 NI
638 0b0001 IMP
639EndEnum
640Enum 23:20 LSM
641 0b0000 NI
642 0b0001 IMP
643EndEnum
644Enum 19:16 HPDS
645 0b0000 NI
646 0b0001 AA32HPD
647 0b0010 HPDS2
648EndEnum
649Enum 15:12 CnP
650 0b0000 NI
651 0b0001 IMP
652EndEnum
653Enum 11:8 XNX
654 0b0000 NI
655 0b0001 IMP
656EndEnum
657Enum 7:4 AC2
658 0b0000 NI
659 0b0001 IMP
660EndEnum
661Enum 3:0 SpecSEI
662 0b0000 NI
663 0b0001 IMP
664EndEnum
665EndSysreg
666
667Sysreg MVFR0_EL1 3 0 0 3 0
668Res0 63:32
669Enum 31:28 FPRound
670 0b0000 NI
671 0b0001 IMP
672EndEnum
673Enum 27:24 FPShVec
674 0b0000 NI
675 0b0001 IMP
676EndEnum
677Enum 23:20 FPSqrt
678 0b0000 NI
679 0b0001 IMP
680EndEnum
681Enum 19:16 FPDivide
682 0b0000 NI
683 0b0001 IMP
684EndEnum
685Enum 15:12 FPTrap
686 0b0000 NI
687 0b0001 IMP
688EndEnum
689Enum 11:8 FPDP
690 0b0000 NI
691 0b0001 VFPv2
692 0b0001 VFPv3
693EndEnum
694Enum 7:4 FPSP
695 0b0000 NI
696 0b0001 VFPv2
697 0b0001 VFPv3
698EndEnum
699Enum 3:0 SIMDReg
700 0b0000 NI
701 0b0001 IMP_16x64
702 0b0001 IMP_32x64
703EndEnum
704EndSysreg
705
706Sysreg MVFR1_EL1 3 0 0 3 1
707Res0 63:32
708Enum 31:28 SIMDFMAC
709 0b0000 NI
710 0b0001 IMP
711EndEnum
712Enum 27:24 FPHP
713 0b0000 NI
714 0b0001 FPHP
715 0b0010 FPHP_CONV
716 0b0011 FP16
717EndEnum
718Enum 23:20 SIMDHP
719 0b0000 NI
720 0b0001 SIMDHP
721 0b0001 SIMDHP_FLOAT
722EndEnum
723Enum 19:16 SIMDSP
724 0b0000 NI
725 0b0001 IMP
726EndEnum
727Enum 15:12 SIMDInt
728 0b0000 NI
729 0b0001 IMP
730EndEnum
731Enum 11:8 SIMDLS
732 0b0000 NI
733 0b0001 IMP
734EndEnum
735Enum 7:4 FPDNaN
736 0b0000 NI
737 0b0001 IMP
738EndEnum
739Enum 3:0 FPFtZ
740 0b0000 NI
741 0b0001 IMP
742EndEnum
743EndSysreg
744
745Sysreg MVFR2_EL1 3 0 0 3 2
746Res0 63:8
747Enum 7:4 FPMisc
748 0b0000 NI
749 0b0001 FP
750 0b0010 FP_DIRECTED_ROUNDING
751 0b0011 FP_ROUNDING
752 0b0100 FP_MAX_MIN
753EndEnum
754Enum 3:0 SIMDMisc
755 0b0000 NI
756 0b0001 SIMD_DIRECTED_ROUNDING
757 0b0010 SIMD_ROUNDING
758 0b0011 SIMD_MAX_MIN
759EndEnum
760EndSysreg
761
762Sysreg ID_PFR2_EL1 3 0 0 3 4
763Res0 63:12
764Enum 11:8 RAS_frac
765 0b0000 NI
766 0b0001 RASv1p1
767EndEnum
768Enum 7:4 SSBS
769 0b0000 NI
770 0b0001 IMP
771EndEnum
772Enum 3:0 CSV3
773 0b0000 NI
774 0b0001 IMP
775EndEnum
776EndSysreg
777
778Sysreg ID_DFR1_EL1 3 0 0 3 5
779Res0 63:8
780Enum 7:4 HPMN0
781 0b0000 NI
782 0b0001 IMP
783EndEnum
784Enum 3:0 MTPMU
785 0b0000 IMPDEF
786 0b0001 IMP
787 0b1111 NI
788EndEnum
789EndSysreg
790
791Sysreg ID_MMFR5_EL1 3 0 0 3 6
792Res0 63:8
793Enum 7:4 nTLBPA
794 0b0000 NI
795 0b0001 IMP
796EndEnum
797Enum 3:0 ETS
798 0b0000 NI
799 0b0001 IMP
800EndEnum
801EndSysreg
802
49Sysreg ID_AA64PFR0_EL1 3 0 0 4 0803Sysreg ID_AA64PFR0_EL1 3 0 0 4 0
50Enum 63:60 CSV3804Enum 63:60 CSV3
51 0b0000 NI805 0b0000 NI
arch/x86/include/asm/kvm_host.h+0−2
@@ -2154,8 +2154,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu) · 2153 unmodified lines
2154#endif2154#endif
2155}2155}
2156 2156
2157int kvm_cpu_dirty_log_size(void);
2158
2159int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);2157int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
2160 2158
2161#define KVM_CLOCK_VALID_FLAGS \2159#define KVM_CLOCK_VALID_FLAGS \
arch/x86/kvm/x86.c+6−9
@@ -10208,20 +10208,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · 10207 unmodified lines
10208 10208
10209 bool req_immediate_exit = false;10209 bool req_immediate_exit = false;
10210 10210
10211 /* Forbid vmenter if vcpu dirty ring is soft-full */
10212 if (unlikely(vcpu->kvm->dirty_ring_size &&
10213 kvm_dirty_ring_soft_full(&vcpu->dirty_ring))) {
10214 vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
10215 trace_kvm_dirty_ring_exit(vcpu);
10216 r = 0;
10217 goto out;
10218 }
10219
10220 if (kvm_request_pending(vcpu)) {10211 if (kvm_request_pending(vcpu)) {
10221 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {10212 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
10222 r = -EIO;10213 r = -EIO;
10223 goto out;10214 goto out;
10224 }10215 }
10216
10217 if (kvm_dirty_ring_check_request(vcpu)) {
10218 r = 0;
10219 goto out;
10220 }
10221
10225 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {10222 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
10226 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {10223 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
10227 r = 0;10224 r = 0;
fs/proc/page.c+2−1
@@ -219,8 +219,9 @@ u64 stable_page_flags(struct page *page) · 218 unmodified lines
219 u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);219 u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
220 u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);220 u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
221 u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);221 u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
222#ifdef CONFIG_64BIT222#ifdef CONFIG_ARCH_USES_PG_ARCH_X
223 u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2);223 u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2);
224 u |= kpf_copy_bit(k, KPF_ARCH_3, PG_arch_3);
224#endif225#endif
225 226
226 return u;227 return u;
include/kvm/arm_pmu.h+13−2
@@ -11,7 +11,6 @@ · 10 unmodified lines
11#include <asm/perf_event.h>11#include <asm/perf_event.h>
12 12
13#define ARMV8_PMU_CYCLE_IDX (ARMV8_PMU_MAX_COUNTERS - 1)13#define ARMV8_PMU_CYCLE_IDX (ARMV8_PMU_MAX_COUNTERS - 1)
14#define ARMV8_PMU_MAX_COUNTER_PAIRS ((ARMV8_PMU_MAX_COUNTERS + 1) >> 1)
15 14
16#ifdef CONFIG_HW_PERF_EVENTS15#ifdef CONFIG_HW_PERF_EVENTS
17 16
@@ -29,7 +28,6 @@ struct kvm_pmu { · 11 unmodified lines
29 struct irq_work overflow_work;28 struct irq_work overflow_work;
30 struct kvm_pmu_events events;29 struct kvm_pmu_events events;
31 struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS];30 struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS];
32 DECLARE_BITMAP(chained, ARMV8_PMU_MAX_COUNTER_PAIRS);
33 int irq_num;31 int irq_num;
34 bool created;32 bool created;
35 bool irq_level;33 bool irq_level;
@@ -91,6 +89,14 @@ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); · 55 unmodified lines
91 vcpu->arch.pmu.events = *kvm_get_pmu_events(); \89 vcpu->arch.pmu.events = *kvm_get_pmu_events(); \
92 } while (0)90 } while (0)
93 91
92/*
93 * Evaluates as true when emulating PMUv3p5, and false otherwise.
94 */
95#define kvm_pmu_is_3p5(vcpu) \
96 (vcpu->kvm->arch.dfr0_pmuver.imp >= ID_AA64DFR0_EL1_PMUVer_V3P5)
97
98u8 kvm_arm_pmu_get_pmuver_limit(void);
99
94#else100#else
95struct kvm_pmu {101struct kvm_pmu {
96};102};
@@ -153,9 +159,14 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) · 56 unmodified lines
153}159}
154 160
155#define kvm_vcpu_has_pmu(vcpu) ({ false; })161#define kvm_vcpu_has_pmu(vcpu) ({ false; })
162#define kvm_pmu_is_3p5(vcpu) ({ false; })
156static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {}163static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {}
157static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}164static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {}
158static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}165static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {}
166static inline u8 kvm_arm_pmu_get_pmuver_limit(void)
167{
168 return 0;
169}
159 170
160#endif171#endif
161 172
include/kvm/arm_vgic.h+1−0
@@ -263,6 +263,7 @@ struct vgic_dist { · 262 unmodified lines
263 struct vgic_io_device dist_iodev;263 struct vgic_io_device dist_iodev;
264 264
265 bool has_its;265 bool has_its;
266 bool save_its_tables_in_progress;
266 267
267 /*268 /*
268 * Contains the attributes and gpa of the LPI configuration table.269 * Contains the attributes and gpa of the LPI configuration table.
include/linux/kernel-page-flags.h+1−0
@@ -18,5 +18,6 @@ · 17 unmodified lines
18#define KPF_UNCACHED 3918#define KPF_UNCACHED 39
19#define KPF_SOFTDIRTY 4019#define KPF_SOFTDIRTY 40
20#define KPF_ARCH_2 4120#define KPF_ARCH_2 41
21#define KPF_ARCH_3 42
21 22
22#endif /* LINUX_KERNEL_PAGE_FLAGS_H */23#endif /* LINUX_KERNEL_PAGE_FLAGS_H */
include/linux/kvm_dirty_ring.h+12−8
@@ -37,6 +37,11 @@ static inline u32 kvm_dirty_ring_get_rsvd_entries(void) · 36 unmodified lines
37 return 0;37 return 0;
38}38}
39 39
40static inline bool kvm_use_dirty_bitmap(struct kvm *kvm)
41{
42 return true;
43}
44
40static inline int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring,45static inline int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring,
41 int index, u32 size)46 int index, u32 size)
42{47{
@@ -49,7 +54,7 @@ static inline int kvm_dirty_ring_reset(struct kvm *kvm, · 6 unmodified lines
49 return 0;54 return 0;
50}55}
51 56
52static inline void kvm_dirty_ring_push(struct kvm_dirty_ring *ring,57static inline void kvm_dirty_ring_push(struct kvm_vcpu *vcpu,
53 u32 slot, u64 offset)58 u32 slot, u64 offset)
54{59{
55}60}
@@ -64,13 +69,11 @@ static inline void kvm_dirty_ring_free(struct kvm_dirty_ring *ring) · 8 unmodified lines
64{69{
65}70}
66 71
67static inline bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
68{
69 return true;
70}
71
72#else /* CONFIG_HAVE_KVM_DIRTY_RING */72#else /* CONFIG_HAVE_KVM_DIRTY_RING */
73 73
74int kvm_cpu_dirty_log_size(void);
75bool kvm_use_dirty_bitmap(struct kvm *kvm);
76bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm);
74u32 kvm_dirty_ring_get_rsvd_entries(void);77u32 kvm_dirty_ring_get_rsvd_entries(void);
75int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size);78int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size);
76 79
@@ -84,13 +87,14 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring); · 7 unmodified lines
84 * returns =0: successfully pushed87 * returns =0: successfully pushed
85 * <0: unable to push, need to wait88 * <0: unable to push, need to wait
86 */89 */
87void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset);90void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset);
91
92bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu);
88 93
89/* for use in vm_operations_struct */94/* for use in vm_operations_struct */
90struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset);95struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset);
91 96
92void kvm_dirty_ring_free(struct kvm_dirty_ring *ring);97void kvm_dirty_ring_free(struct kvm_dirty_ring *ring);
93bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring);
94 98
95#endif /* CONFIG_HAVE_KVM_DIRTY_RING */99#endif /* CONFIG_HAVE_KVM_DIRTY_RING */
96 100
include/linux/kvm_host.h+6−4
@@ -163,10 +163,11 @@ static inline bool is_error_page(struct page *page) · 162 unmodified lines
163 * Architecture-independent vcpu->requests bit members163 * Architecture-independent vcpu->requests bit members
164 * Bits 3-7 are reserved for more arch-independent bits.164 * Bits 3-7 are reserved for more arch-independent bits.
165 */165 */
166#define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)166#define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
167#define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)167#define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
168#define KVM_REQ_UNBLOCK 2168#define KVM_REQ_UNBLOCK 2
169#define KVM_REQUEST_ARCH_BASE 8169#define KVM_REQ_DIRTY_RING_SOFT_FULL 3
170#define KVM_REQUEST_ARCH_BASE 8
170 171
171/*172/*
172 * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to173 * KVM_REQ_OUTSIDE_GUEST_MODE exists is purely as way to force the vCPU to
@@ -795,6 +796,7 @@ struct kvm { · 622 unmodified lines
795 pid_t userspace_pid;796 pid_t userspace_pid;
796 unsigned int max_halt_poll_ns;797 unsigned int max_halt_poll_ns;
797 u32 dirty_ring_size;798 u32 dirty_ring_size;
799 bool dirty_ring_with_bitmap;
798 bool vm_bugged;800 bool vm_bugged;
799 bool vm_dead;801 bool vm_dead;
800 802
include/linux/page-flags.h+2−1
@@ -132,8 +132,9 @@ enum pageflags { · 131 unmodified lines
132 PG_young,132 PG_young,
133 PG_idle,133 PG_idle,
134#endif134#endif
135#ifdef CONFIG_64BIT135#ifdef CONFIG_ARCH_USES_PG_ARCH_X
136 PG_arch_2,136 PG_arch_2,
137 PG_arch_3,
137#endif138#endif
138#ifdef CONFIG_KASAN_HW_TAGS139#ifdef CONFIG_KASAN_HW_TAGS
139 PG_skip_kasan_poison,140 PG_skip_kasan_poison,
include/trace/events/mmflags.h+5−4
@@ -91,10 +91,10 @@ · 90 unmodified lines
91#define IF_HAVE_PG_IDLE(flag,string)91#define IF_HAVE_PG_IDLE(flag,string)
92#endif92#endif
93 93
94#ifdef CONFIG_64BIT94#ifdef CONFIG_ARCH_USES_PG_ARCH_X
95#define IF_HAVE_PG_ARCH_2(flag,string) ,{1UL << flag, string}95#define IF_HAVE_PG_ARCH_X(flag,string) ,{1UL << flag, string}
96#else96#else
97#define IF_HAVE_PG_ARCH_2(flag,string)97#define IF_HAVE_PG_ARCH_X(flag,string)
98#endif98#endif
99 99
100#ifdef CONFIG_KASAN_HW_TAGS100#ifdef CONFIG_KASAN_HW_TAGS
@@ -130,7 +130,8 @@ IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \ · 29 unmodified lines
130IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \130IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \
131IF_HAVE_PG_IDLE(PG_young, "young" ) \131IF_HAVE_PG_IDLE(PG_young, "young" ) \
132IF_HAVE_PG_IDLE(PG_idle, "idle" ) \132IF_HAVE_PG_IDLE(PG_idle, "idle" ) \
133IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) \133IF_HAVE_PG_ARCH_X(PG_arch_2, "arch_2" ) \
134IF_HAVE_PG_ARCH_X(PG_arch_3, "arch_3" ) \
134IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison")135IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison")
135 136
136#define show_page_flags(flags) \137#define show_page_flags(flags) \
include/uapi/linux/kvm.h+1−0
@@ -1182,6 +1182,7 @@ struct kvm_ppc_resize_hpt { · 1181 unmodified lines
1182#define KVM_CAP_S390_CPU_TOPOLOGY 2221182#define KVM_CAP_S390_CPU_TOPOLOGY 222
1183#define KVM_CAP_DIRTY_LOG_RING_ACQ_REL 2231183#define KVM_CAP_DIRTY_LOG_RING_ACQ_REL 223
1184#define KVM_CAP_S390_PROTECTED_ASYNC_DISABLE 2241184#define KVM_CAP_S390_PROTECTED_ASYNC_DISABLE 224
1185#define KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP 225
1185 1186
1186#ifdef KVM_CAP_IRQ_ROUTING1187#ifdef KVM_CAP_IRQ_ROUTING
1187 1188
mm/Kconfig+8−0
@@ -1005,6 +1005,14 @@ config ARCH_USES_HIGH_VMA_FLAGS · 1004 unmodified lines
1005config ARCH_HAS_PKEYS1005config ARCH_HAS_PKEYS
1006 bool1006 bool
1007 1007
1008config ARCH_USES_PG_ARCH_X
1009 bool
1010 help
1011 Enable the definition of PG_arch_x page flags with x > 1. Only
1012 suitable for 64-bit architectures with CONFIG_FLATMEM or
1013 CONFIG_SPARSEMEM_VMEMMAP enabled, otherwise there may not be
1014 enough room for additional bits in page->flags.
1015
1008config VM_EVENT_COUNTERS1016config VM_EVENT_COUNTERS
1009 default y1017 default y
1010 bool "Enable VM event counters for /proc/vmstat" if EXPERT1018 bool "Enable VM event counters for /proc/vmstat" if EXPERT
mm/huge_memory.c+2−1
@@ -2444,8 +2444,9 @@ static void __split_huge_page_tail(struct page *head, int tail, · 2443 unmodified lines
2444 (1L << PG_workingset) |2444 (1L << PG_workingset) |
2445 (1L << PG_locked) |2445 (1L << PG_locked) |
2446 (1L << PG_unevictable) |2446 (1L << PG_unevictable) |
2447#ifdef CONFIG_64BIT2447#ifdef CONFIG_ARCH_USES_PG_ARCH_X
2448 (1L << PG_arch_2) |2448 (1L << PG_arch_2) |
2449 (1L << PG_arch_3) |
2449#endif2450#endif
2450 (1L << PG_dirty) |2451 (1L << PG_dirty) |
2451 LRU_GEN_MASK | LRU_REFS_MASK));2452 LRU_GEN_MASK | LRU_REFS_MASK));
tools/include/linux/bitfield.h+176−0
@@ -0,0 +1,176 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2014 Felix Fietkau <nbd@nbd.name>
4 * Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
5 */
6
7#ifndef _LINUX_BITFIELD_H
8#define _LINUX_BITFIELD_H
9
10#include <linux/build_bug.h>
11#include <asm/byteorder.h>
12
13/*
14 * Bitfield access macros
15 *
16 * FIELD_{GET,PREP} macros take as first parameter shifted mask
17 * from which they extract the base mask and shift amount.
18 * Mask must be a compilation time constant.
19 *
20 * Example:
21 *
22 * #define REG_FIELD_A GENMASK(6, 0)
23 * #define REG_FIELD_B BIT(7)
24 * #define REG_FIELD_C GENMASK(15, 8)
25 * #define REG_FIELD_D GENMASK(31, 16)
26 *
27 * Get:
28 * a = FIELD_GET(REG_FIELD_A, reg);
29 * b = FIELD_GET(REG_FIELD_B, reg);
30 *
31 * Set:
32 * reg = FIELD_PREP(REG_FIELD_A, 1) |
33 * FIELD_PREP(REG_FIELD_B, 0) |
34 * FIELD_PREP(REG_FIELD_C, c) |
35 * FIELD_PREP(REG_FIELD_D, 0x40);
36 *
37 * Modify:
38 * reg &= ~REG_FIELD_C;
39 * reg |= FIELD_PREP(REG_FIELD_C, c);
40 */
41
42#define __bf_shf(x) (__builtin_ffsll(x) - 1)
43
44#define __scalar_type_to_unsigned_cases(type) \
45 unsigned type: (unsigned type)0, \
46 signed type: (unsigned type)0
47
48#define __unsigned_scalar_typeof(x) typeof( \
49 _Generic((x), \
50 char: (unsigned char)0, \
51 __scalar_type_to_unsigned_cases(char), \
52 __scalar_type_to_unsigned_cases(short), \
53 __scalar_type_to_unsigned_cases(int), \
54 __scalar_type_to_unsigned_cases(long), \
55 __scalar_type_to_unsigned_cases(long long), \
56 default: (x)))
57
58#define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x))
59
60#define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \
61 ({ \
62 BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \
63 _pfx "mask is not constant"); \
64 BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero"); \
65 BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ? \
66 ~((_mask) >> __bf_shf(_mask)) & (_val) : 0, \
67 _pfx "value too large for the field"); \
68 BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \
69 __bf_cast_unsigned(_reg, ~0ull), \
70 _pfx "type of reg too small for mask"); \
71 __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \
72 (1ULL << __bf_shf(_mask))); \
73 })
74
75/**
76 * FIELD_MAX() - produce the maximum value representable by a field
77 * @_mask: shifted mask defining the field's length and position
78 *
79 * FIELD_MAX() returns the maximum value that can be held in the field
80 * specified by @_mask.
81 */
82#define FIELD_MAX(_mask) \
83 ({ \
84 __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_MAX: "); \
85 (typeof(_mask))((_mask) >> __bf_shf(_mask)); \
86 })
87
88/**
89 * FIELD_FIT() - check if value fits in the field
90 * @_mask: shifted mask defining the field's length and position
91 * @_val: value to test against the field
92 *
93 * Return: true if @_val can fit inside @_mask, false if @_val is too big.
94 */
95#define FIELD_FIT(_mask, _val) \
96 ({ \
97 __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_FIT: "); \
98 !((((typeof(_mask))_val) << __bf_shf(_mask)) & ~(_mask)); \
99 })
100
101/**
102 * FIELD_PREP() - prepare a bitfield element
103 * @_mask: shifted mask defining the field's length and position
104 * @_val: value to put in the field
105 *
106 * FIELD_PREP() masks and shifts up the value. The result should
107 * be combined with other fields of the bitfield using logical OR.
108 */
109#define FIELD_PREP(_mask, _val) \
110 ({ \
111 __BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: "); \
112 ((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask); \
113 })
114
115/**
116 * FIELD_GET() - extract a bitfield element
117 * @_mask: shifted mask defining the field's length and position
118 * @_reg: value of entire bitfield
119 *
120 * FIELD_GET() extracts the field specified by @_mask from the
121 * bitfield passed in as @_reg by masking and shifting it down.
122 */
123#define FIELD_GET(_mask, _reg) \
124 ({ \
125 __BF_FIELD_CHECK(_mask, _reg, 0U, "FIELD_GET: "); \
126 (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
127 })
128
129extern void __compiletime_error("value doesn't fit into mask")
130__field_overflow(void);
131extern void __compiletime_error("bad bitfield mask")
132__bad_mask(void);
133static __always_inline u64 field_multiplier(u64 field)
134{
135 if ((field | (field - 1)) & ((field | (field - 1)) + 1))
136 __bad_mask();
137 return field & -field;
138}
139static __always_inline u64 field_mask(u64 field)
140{
141 return field / field_multiplier(field);
142}
143#define field_max(field) ((typeof(field))field_mask(field))
144#define ____MAKE_OP(type,base,to,from) \
145static __always_inline __##type type##_encode_bits(base v, base field) \
146{ \
147 if (__builtin_constant_p(v) && (v & ~field_mask(field))) \
148 __field_overflow(); \
149 return to((v & field_mask(field)) * field_multiplier(field)); \
150} \
151static __always_inline __##type type##_replace_bits(__##type old, \
152 base val, base field) \
153{ \
154 return (old & ~to(field)) | type##_encode_bits(val, field); \
155} \
156static __always_inline void type##p_replace_bits(__##type *p, \
157 base val, base field) \
158{ \
159 *p = (*p & ~to(field)) | type##_encode_bits(val, field); \
160} \
161static __always_inline base type##_get_bits(__##type v, base field) \
162{ \
163 return (from(v) & field)/field_multiplier(field); \
164}
165#define __MAKE_OP(size) \
166 ____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu) \
167 ____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu) \
168 ____MAKE_OP(u##size,u##size,,)
169____MAKE_OP(u8,u8,,)
170__MAKE_OP(16)
171__MAKE_OP(32)
172__MAKE_OP(64)
173#undef __MAKE_OP
174#undef ____MAKE_OP
175
176#endif
tools/testing/selftests/kvm/.gitignore+1−0
@@ -4,6 +4,7 @@ · 3 unmodified lines
4/aarch64/debug-exceptions4/aarch64/debug-exceptions
5/aarch64/get-reg-list5/aarch64/get-reg-list
6/aarch64/hypercalls6/aarch64/hypercalls
7/aarch64/page_fault_test
7/aarch64/psci_test8/aarch64/psci_test
8/aarch64/vcpu_width_config9/aarch64/vcpu_width_config
9/aarch64/vgic_init10/aarch64/vgic_init
tools/testing/selftests/kvm/Makefile+3−0
@@ -48,6 +48,7 @@ LIBKVM += lib/rbtree.c · 47 unmodified lines
48LIBKVM += lib/sparsebit.c48LIBKVM += lib/sparsebit.c
49LIBKVM += lib/test_util.c49LIBKVM += lib/test_util.c
50LIBKVM += lib/ucall_common.c50LIBKVM += lib/ucall_common.c
51LIBKVM += lib/userfaultfd_util.c
51 52
52LIBKVM_STRING += lib/string_override.c53LIBKVM_STRING += lib/string_override.c
53 54
@@ -158,10 +159,12 @@ TEST_GEN_PROGS_aarch64 += aarch64/arch_timer · 104 unmodified lines
158TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions159TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions
159TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list160TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list
160TEST_GEN_PROGS_aarch64 += aarch64/hypercalls161TEST_GEN_PROGS_aarch64 += aarch64/hypercalls
162TEST_GEN_PROGS_aarch64 += aarch64/page_fault_test
161TEST_GEN_PROGS_aarch64 += aarch64/psci_test163TEST_GEN_PROGS_aarch64 += aarch64/psci_test
162TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config164TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config
163TEST_GEN_PROGS_aarch64 += aarch64/vgic_init165TEST_GEN_PROGS_aarch64 += aarch64/vgic_init
164TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq166TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq
167TEST_GEN_PROGS_aarch64 += access_tracking_perf_test
165TEST_GEN_PROGS_aarch64 += demand_paging_test168TEST_GEN_PROGS_aarch64 += demand_paging_test
166TEST_GEN_PROGS_aarch64 += dirty_log_test169TEST_GEN_PROGS_aarch64 += dirty_log_test
167TEST_GEN_PROGS_aarch64 += dirty_log_perf_test170TEST_GEN_PROGS_aarch64 += dirty_log_perf_test
tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c+2−1
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include "kvm_util.h"13#include "kvm_util.h"
14#include "processor.h"14#include "processor.h"
15#include "test_util.h"15#include "test_util.h"
16#include <linux/bitfield.h>
16 17
17#define BAD_ID_REG_VAL 0x1badc0deul18#define BAD_ID_REG_VAL 0x1badc0deul
18 19
@@ -145,7 +146,7 @@ static bool vcpu_aarch64_only(struct kvm_vcpu *vcpu) · 126 unmodified lines
145 146
146 vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1), &val);147 vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1), &val);
147 148
148 el0 = (val & ARM64_FEATURE_MASK(ID_AA64PFR0_EL0)) >> ID_AA64PFR0_EL0_SHIFT;149 el0 = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0), val);
149 return el0 == ID_AA64PFR0_ELx_64BIT_ONLY;150 return el0 == ID_AA64PFR0_ELx_64BIT_ONLY;
150}151}
151 152
tools/testing/selftests/kvm/aarch64/debug-exceptions.c+239−72
@@ -2,6 +2,7 @@ · 1 unmodified line
2#include <test_util.h>2#include <test_util.h>
3#include <kvm_util.h>3#include <kvm_util.h>
4#include <processor.h>4#include <processor.h>
5#include <linux/bitfield.h>
5 6
6#define MDSCR_KDE (1 << 13)7#define MDSCR_KDE (1 << 13)
7#define MDSCR_MDE (1 << 15)8#define MDSCR_MDE (1 << 15)
@@ -11,17 +12,24 @@ · 3 unmodified lines
11#define DBGBCR_EXEC (0x0 << 3)12#define DBGBCR_EXEC (0x0 << 3)
12#define DBGBCR_EL1 (0x1 << 1)13#define DBGBCR_EL1 (0x1 << 1)
13#define DBGBCR_E (0x1 << 0)14#define DBGBCR_E (0x1 << 0)
15#define DBGBCR_LBN_SHIFT 16
16#define DBGBCR_BT_SHIFT 20
17#define DBGBCR_BT_ADDR_LINK_CTX (0x1 << DBGBCR_BT_SHIFT)
18#define DBGBCR_BT_CTX_LINK (0x3 << DBGBCR_BT_SHIFT)
14 19
15#define DBGWCR_LEN8 (0xff << 5)20#define DBGWCR_LEN8 (0xff << 5)
16#define DBGWCR_RD (0x1 << 3)21#define DBGWCR_RD (0x1 << 3)
17#define DBGWCR_WR (0x2 << 3)22#define DBGWCR_WR (0x2 << 3)
18#define DBGWCR_EL1 (0x1 << 1)23#define DBGWCR_EL1 (0x1 << 1)
19#define DBGWCR_E (0x1 << 0)24#define DBGWCR_E (0x1 << 0)
25#define DBGWCR_LBN_SHIFT 16
26#define DBGWCR_WT_SHIFT 20
27#define DBGWCR_WT_LINK (0x1 << DBGWCR_WT_SHIFT)
20 28
21#define SPSR_D (1 << 9)29#define SPSR_D (1 << 9)
22#define SPSR_SS (1 << 21)30#define SPSR_SS (1 << 21)
23 31
24extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start;32extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start, hw_bp_ctx;
25extern unsigned char iter_ss_begin, iter_ss_end;33extern unsigned char iter_ss_begin, iter_ss_end;
26static volatile uint64_t sw_bp_addr, hw_bp_addr;34static volatile uint64_t sw_bp_addr, hw_bp_addr;
27static volatile uint64_t wp_addr, wp_data_addr;35static volatile uint64_t wp_addr, wp_data_addr;
@@ -29,8 +37,74 @@ static volatile uint64_t svc_addr; · 1 unmodified line
29static volatile uint64_t ss_addr[4], ss_idx;37static volatile uint64_t ss_addr[4], ss_idx;
30#define PC(v) ((uint64_t)&(v))38#define PC(v) ((uint64_t)&(v))
31 39
40#define GEN_DEBUG_WRITE_REG(reg_name) \
41static void write_##reg_name(int num, uint64_t val) \
42{ \
43 switch (num) { \
44 case 0: \
45 write_sysreg(val, reg_name##0_el1); \
46 break; \
47 case 1: \
48 write_sysreg(val, reg_name##1_el1); \
49 break; \
50 case 2: \
51 write_sysreg(val, reg_name##2_el1); \
52 break; \
53 case 3: \
54 write_sysreg(val, reg_name##3_el1); \
55 break; \
56 case 4: \
57 write_sysreg(val, reg_name##4_el1); \
58 break; \
59 case 5: \
60 write_sysreg(val, reg_name##5_el1); \
61 break; \
62 case 6: \
63 write_sysreg(val, reg_name##6_el1); \
64 break; \
65 case 7: \
66 write_sysreg(val, reg_name##7_el1); \
67 break; \
68 case 8: \
69 write_sysreg(val, reg_name##8_el1); \
70 break; \
71 case 9: \
72 write_sysreg(val, reg_name##9_el1); \
73 break; \
74 case 10: \
75 write_sysreg(val, reg_name##10_el1); \
76 break; \
77 case 11: \
78 write_sysreg(val, reg_name##11_el1); \
79 break; \
80 case 12: \
81 write_sysreg(val, reg_name##12_el1); \
82 break; \
83 case 13: \
84 write_sysreg(val, reg_name##13_el1); \
85 break; \
86 case 14: \
87 write_sysreg(val, reg_name##14_el1); \
88 break; \
89 case 15: \
90 write_sysreg(val, reg_name##15_el1); \
91 break; \
92 default: \
93 GUEST_ASSERT(0); \
94 } \
95}
96
97/* Define write_dbgbcr()/write_dbgbvr()/write_dbgwcr()/write_dbgwvr() */
98GEN_DEBUG_WRITE_REG(dbgbcr)
99GEN_DEBUG_WRITE_REG(dbgbvr)
100GEN_DEBUG_WRITE_REG(dbgwcr)
101GEN_DEBUG_WRITE_REG(dbgwvr)
102
32static void reset_debug_state(void)103static void reset_debug_state(void)
33{104{
105 uint8_t brps, wrps, i;
106 uint64_t dfr0;
107
34 asm volatile("msr daifset, #8");108 asm volatile("msr daifset, #8");
35 109
36 write_sysreg(0, osdlr_el1);110 write_sysreg(0, osdlr_el1);
@@ -38,11 +112,21 @@ static void reset_debug_state(void) · 1 unmodified line
38 isb();112 isb();
39 113
40 write_sysreg(0, mdscr_el1);114 write_sysreg(0, mdscr_el1);
41 /* This test only uses the first bp and wp slot. */115 write_sysreg(0, contextidr_el1);
42 write_sysreg(0, dbgbvr0_el1);116
43 write_sysreg(0, dbgbcr0_el1);117 /* Reset all bcr/bvr/wcr/wvr registers */
44 write_sysreg(0, dbgwcr0_el1);118 dfr0 = read_sysreg(id_aa64dfr0_el1);
45 write_sysreg(0, dbgwvr0_el1);119 brps = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_BRPS), dfr0);
120 for (i = 0; i <= brps; i++) {
121 write_dbgbcr(i, 0);
122 write_dbgbvr(i, 0);
123 }
124 wrps = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_WRPS), dfr0);
125 for (i = 0; i <= wrps; i++) {
126 write_dbgwcr(i, 0);
127 write_dbgwvr(i, 0);
128 }
129
46 isb();130 isb();
47}131}
48 132
@@ -54,16 +138,10 @@ static void enable_os_lock(void) · 5 unmodified lines
54 GUEST_ASSERT(read_sysreg(oslsr_el1) & 2);138 GUEST_ASSERT(read_sysreg(oslsr_el1) & 2);
55}139}
56 140
57static void install_wp(uint64_t addr)141static void enable_monitor_debug_exceptions(void)
58{142{
59 uint32_t wcr;
60 uint32_t mdscr;143 uint32_t mdscr;
61 144
62 wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E;
63 write_sysreg(wcr, dbgwcr0_el1);
64 write_sysreg(addr, dbgwvr0_el1);
65 isb();
66
67 asm volatile("msr daifclr, #8");145 asm volatile("msr daifclr, #8");
68 146
69 mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE;147 mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE;
@@ -71,21 +149,76 @@ static void install_wp(uint64_t addr) · 1 unmodified line
71 isb();149 isb();
72}150}
73 151
74static void install_hw_bp(uint64_t addr)152static void install_wp(uint8_t wpn, uint64_t addr)
153{
154 uint32_t wcr;
155
156 wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E;
157 write_dbgwcr(wpn, wcr);
158 write_dbgwvr(wpn, addr);
159
160 isb();
161
162 enable_monitor_debug_exceptions();
163}
164
165static void install_hw_bp(uint8_t bpn, uint64_t addr)
75{166{
76 uint32_t bcr;167 uint32_t bcr;
77 uint32_t mdscr;
78 168
79 bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E;169 bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E;
80 write_sysreg(bcr, dbgbcr0_el1);170 write_dbgbcr(bpn, bcr);
81 write_sysreg(addr, dbgbvr0_el1);171 write_dbgbvr(bpn, addr);
82 isb();172 isb();
83 173
84 asm volatile("msr daifclr, #8");174 enable_monitor_debug_exceptions();
175}
85 176
86 mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE;177static void install_wp_ctx(uint8_t addr_wp, uint8_t ctx_bp, uint64_t addr,
87 write_sysreg(mdscr, mdscr_el1);178 uint64_t ctx)
179{
180 uint32_t wcr;
181 uint64_t ctx_bcr;
182
183 /* Setup a context-aware breakpoint for Linked Context ID Match */
184 ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
185 DBGBCR_BT_CTX_LINK;
186 write_dbgbcr(ctx_bp, ctx_bcr);
187 write_dbgbvr(ctx_bp, ctx);
188
189 /* Setup a linked watchpoint (linked to the context-aware breakpoint) */
190 wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E |
191 DBGWCR_WT_LINK | ((uint32_t)ctx_bp << DBGWCR_LBN_SHIFT);
192 write_dbgwcr(addr_wp, wcr);
193 write_dbgwvr(addr_wp, addr);
88 isb();194 isb();
195
196 enable_monitor_debug_exceptions();
197}
198
199void install_hw_bp_ctx(uint8_t addr_bp, uint8_t ctx_bp, uint64_t addr,
200 uint64_t ctx)
201{
202 uint32_t addr_bcr, ctx_bcr;
203
204 /* Setup a context-aware breakpoint for Linked Context ID Match */
205 ctx_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
206 DBGBCR_BT_CTX_LINK;
207 write_dbgbcr(ctx_bp, ctx_bcr);
208 write_dbgbvr(ctx_bp, ctx);
209
210 /*
211 * Setup a normal breakpoint for Linked Address Match, and link it
212 * to the context-aware breakpoint.
213 */
214 addr_bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E |
215 DBGBCR_BT_ADDR_LINK_CTX |
216 ((uint32_t)ctx_bp << DBGBCR_LBN_SHIFT);
217 write_dbgbcr(addr_bp, addr_bcr);
218 write_dbgbvr(addr_bp, addr);
219 isb();
220
221 enable_monitor_debug_exceptions();
89}222}
90 223
91static void install_ss(void)224static void install_ss(void)
@@ -101,52 +234,42 @@ static void install_ss(void) · 9 unmodified lines
101 234
102static volatile char write_data;235static volatile char write_data;
103 236
104static void guest_code(void)237static void guest_code(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
105{238{
106 GUEST_SYNC(0);239 uint64_t ctx = 0xabcdef; /* a random context number */
107 240
108 /* Software-breakpoint */241 /* Software-breakpoint */
109 reset_debug_state();242 reset_debug_state();
110 asm volatile("sw_bp: brk #0");243 asm volatile("sw_bp: brk #0");
111 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp));244 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp));
112 245
113 GUEST_SYNC(1);
114
115 /* Hardware-breakpoint */246 /* Hardware-breakpoint */
116 reset_debug_state();247 reset_debug_state();
117 install_hw_bp(PC(hw_bp));248 install_hw_bp(bpn, PC(hw_bp));
118 asm volatile("hw_bp: nop");249 asm volatile("hw_bp: nop");
119 GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp));250 GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp));
120 251
121 GUEST_SYNC(2);
122
123 /* Hardware-breakpoint + svc */252 /* Hardware-breakpoint + svc */
124 reset_debug_state();253 reset_debug_state();
125 install_hw_bp(PC(bp_svc));254 install_hw_bp(bpn, PC(bp_svc));
126 asm volatile("bp_svc: svc #0");255 asm volatile("bp_svc: svc #0");
127 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_svc));256 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_svc));
128 GUEST_ASSERT_EQ(svc_addr, PC(bp_svc) + 4);257 GUEST_ASSERT_EQ(svc_addr, PC(bp_svc) + 4);
129 258
130 GUEST_SYNC(3);
131
132 /* Hardware-breakpoint + software-breakpoint */259 /* Hardware-breakpoint + software-breakpoint */
133 reset_debug_state();260 reset_debug_state();
134 install_hw_bp(PC(bp_brk));261 install_hw_bp(bpn, PC(bp_brk));
135 asm volatile("bp_brk: brk #0");262 asm volatile("bp_brk: brk #0");
136 GUEST_ASSERT_EQ(sw_bp_addr, PC(bp_brk));263 GUEST_ASSERT_EQ(sw_bp_addr, PC(bp_brk));
137 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_brk));264 GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_brk));
138 265
139 GUEST_SYNC(4);
140
141 /* Watchpoint */266 /* Watchpoint */
142 reset_debug_state();267 reset_debug_state();
143 install_wp(PC(write_data));268 install_wp(wpn, PC(write_data));
144 write_data = 'x';269 write_data = 'x';
145 GUEST_ASSERT_EQ(write_data, 'x');270 GUEST_ASSERT_EQ(write_data, 'x');
146 GUEST_ASSERT_EQ(wp_data_addr, PC(write_data));271 GUEST_ASSERT_EQ(wp_data_addr, PC(write_data));
147 272
148 GUEST_SYNC(5);
149
150 /* Single-step */273 /* Single-step */
151 reset_debug_state();274 reset_debug_state();
152 install_ss();275 install_ss();
@@ -160,8 +283,6 @@ static void guest_code(void) · 7 unmodified lines
160 GUEST_ASSERT_EQ(ss_addr[1], PC(ss_start) + 4);283 GUEST_ASSERT_EQ(ss_addr[1], PC(ss_start) + 4);
161 GUEST_ASSERT_EQ(ss_addr[2], PC(ss_start) + 8);284 GUEST_ASSERT_EQ(ss_addr[2], PC(ss_start) + 8);
162 285
163 GUEST_SYNC(6);
164
165 /* OS Lock does not block software-breakpoint */286 /* OS Lock does not block software-breakpoint */
166 reset_debug_state();287 reset_debug_state();
167 enable_os_lock();288 enable_os_lock();
@@ -169,30 +290,24 @@ static void guest_code(void) · 1 unmodified line
169 asm volatile("sw_bp2: brk #0");290 asm volatile("sw_bp2: brk #0");
170 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp2));291 GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp2));
171 292
172 GUEST_SYNC(7);
173
174 /* OS Lock blocking hardware-breakpoint */293 /* OS Lock blocking hardware-breakpoint */
175 reset_debug_state();294 reset_debug_state();
176 enable_os_lock();295 enable_os_lock();
177 install_hw_bp(PC(hw_bp2));296 install_hw_bp(bpn, PC(hw_bp2));
178 hw_bp_addr = 0;297 hw_bp_addr = 0;
179 asm volatile("hw_bp2: nop");298 asm volatile("hw_bp2: nop");
180 GUEST_ASSERT_EQ(hw_bp_addr, 0);299 GUEST_ASSERT_EQ(hw_bp_addr, 0);
181 300
182 GUEST_SYNC(8);
183
184 /* OS Lock blocking watchpoint */301 /* OS Lock blocking watchpoint */
185 reset_debug_state();302 reset_debug_state();
186 enable_os_lock();303 enable_os_lock();
187 write_data = '\0';304 write_data = '\0';
188 wp_data_addr = 0;305 wp_data_addr = 0;
189 install_wp(PC(write_data));306 install_wp(wpn, PC(write_data));
190 write_data = 'x';307 write_data = 'x';
191 GUEST_ASSERT_EQ(write_data, 'x');308 GUEST_ASSERT_EQ(write_data, 'x');
192 GUEST_ASSERT_EQ(wp_data_addr, 0);309 GUEST_ASSERT_EQ(wp_data_addr, 0);
193 310
194 GUEST_SYNC(9);
195
196 /* OS Lock blocking single-step */311 /* OS Lock blocking single-step */
197 reset_debug_state();312 reset_debug_state();
198 enable_os_lock();313 enable_os_lock();
@@ -205,6 +320,27 @@ static void guest_code(void) · 6 unmodified lines
205 : : : "x0");320 : : : "x0");
206 GUEST_ASSERT_EQ(ss_addr[0], 0);321 GUEST_ASSERT_EQ(ss_addr[0], 0);
207 322
323 /* Linked hardware-breakpoint */
324 hw_bp_addr = 0;
325 reset_debug_state();
326 install_hw_bp_ctx(bpn, ctx_bpn, PC(hw_bp_ctx), ctx);
327 /* Set context id */
328 write_sysreg(ctx, contextidr_el1);
329 isb();
330 asm volatile("hw_bp_ctx: nop");
331 write_sysreg(0, contextidr_el1);
332 GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp_ctx));
333
334 /* Linked watchpoint */
335 reset_debug_state();
336 install_wp_ctx(wpn, ctx_bpn, PC(write_data), ctx);
337 /* Set context id */
338 write_sysreg(ctx, contextidr_el1);
339 isb();
340 write_data = 'x';
341 GUEST_ASSERT_EQ(write_data, 'x');
342 GUEST_ASSERT_EQ(wp_data_addr, PC(write_data));
343
208 GUEST_DONE();344 GUEST_DONE();
209}345}
210 346
@@ -276,20 +412,16 @@ static void guest_code_ss(int test_cnt) · 65 unmodified lines
276 GUEST_DONE();412 GUEST_DONE();
277}413}
278 414
279static int debug_version(struct kvm_vcpu *vcpu)415static int debug_version(uint64_t id_aa64dfr0)
280{416{
281 uint64_t id_aa64dfr0;417 return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), id_aa64dfr0);
282
283 vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1), &id_aa64dfr0);
284 return id_aa64dfr0 & 0xf;
285}418}
286 419
287static void test_guest_debug_exceptions(void)420static void test_guest_debug_exceptions(uint8_t bpn, uint8_t wpn, uint8_t ctx_bpn)
288{421{
289 struct kvm_vcpu *vcpu;422 struct kvm_vcpu *vcpu;
290 struct kvm_vm *vm;423 struct kvm_vm *vm;
291 struct ucall uc;424 struct ucall uc;
292 int stage;
293 425
294 vm = vm_create_with_one_vcpu(&vcpu, guest_code);426 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
295 427
@@ -307,23 +439,19 @@ static void test_guest_debug_exceptions(void) · 11 unmodified lines
307 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,439 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
308 ESR_EC_SVC64, guest_svc_handler);440 ESR_EC_SVC64, guest_svc_handler);
309 441
310 for (stage = 0; stage < 11; stage++) {442 /* Specify bpn/wpn/ctx_bpn to be tested */
311 vcpu_run(vcpu);443 vcpu_args_set(vcpu, 3, bpn, wpn, ctx_bpn);
312 444 pr_debug("Use bpn#%d, wpn#%d and ctx_bpn#%d\n", bpn, wpn, ctx_bpn);
313 switch (get_ucall(vcpu, &uc)) {445
314 case UCALL_SYNC:446 vcpu_run(vcpu);
315 TEST_ASSERT(uc.args[1] == stage,447 switch (get_ucall(vcpu, &uc)) {
316 "Stage %d: Unexpected sync ucall, got %lx",448 case UCALL_ABORT:
317 stage, (ulong)uc.args[1]);449 REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");
318 break;450 break;
319 case UCALL_ABORT:451 case UCALL_DONE:
320 REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");452 goto done;
321 break;453 default:
322 case UCALL_DONE:454 TEST_FAIL("Unknown ucall %lu", uc.cmd);
323 goto done;
324 default:
325 TEST_FAIL("Unknown ucall %lu", uc.cmd);
326 }
327 }455 }
328 456
329done:457done:
@@ -400,6 +528,43 @@ void test_single_step_from_userspace(int test_cnt) · 70 unmodified lines
400 kvm_vm_free(vm);528 kvm_vm_free(vm);
401}529}
402 530
531/*
532 * Run debug testing using the various breakpoint#, watchpoint# and
533 * context-aware breakpoint# with the given ID_AA64DFR0_EL1 configuration.
534 */
535void test_guest_debug_exceptions_all(uint64_t aa64dfr0)
536{
537 uint8_t brp_num, wrp_num, ctx_brp_num, normal_brp_num, ctx_brp_base;
538 int b, w, c;
539
540 /* Number of breakpoints */
541 brp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_BRPS), aa64dfr0) + 1;
542 __TEST_REQUIRE(brp_num >= 2, "At least two breakpoints are required");
543
544 /* Number of watchpoints */
545 wrp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_WRPS), aa64dfr0) + 1;
546
547 /* Number of context aware breakpoints */
548 ctx_brp_num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_CTX_CMPS), aa64dfr0) + 1;
549
550 pr_debug("%s brp_num:%d, wrp_num:%d, ctx_brp_num:%d\n", __func__,
551 brp_num, wrp_num, ctx_brp_num);
552
553 /* Number of normal (non-context aware) breakpoints */
554 normal_brp_num = brp_num - ctx_brp_num;
555
556 /* Lowest context aware breakpoint number */
557 ctx_brp_base = normal_brp_num;
558
559 /* Run tests with all supported breakpoints/watchpoints */
560 for (c = ctx_brp_base; c < ctx_brp_base + ctx_brp_num; c++) {
561 for (b = 0; b < normal_brp_num; b++) {
562 for (w = 0; w < wrp_num; w++)
563 test_guest_debug_exceptions(b, w, c);
564 }
565 }
566}
567
403static void help(char *name)568static void help(char *name)
404{569{
405 puts("");570 puts("");
@@ -414,9 +579,11 @@ int main(int argc, char *argv[]) · 8 unmodified lines
414 struct kvm_vm *vm;579 struct kvm_vm *vm;
415 int opt;580 int opt;
416 int ss_iteration = 10000;581 int ss_iteration = 10000;
582 uint64_t aa64dfr0;
417 583
418 vm = vm_create_with_one_vcpu(&vcpu, guest_code);584 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
419 __TEST_REQUIRE(debug_version(vcpu) >= 6,585 vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64DFR0_EL1), &aa64dfr0);
586 __TEST_REQUIRE(debug_version(aa64dfr0) >= 6,
420 "Armv8 debug architecture not supported.");587 "Armv8 debug architecture not supported.");
421 kvm_vm_free(vm);588 kvm_vm_free(vm);
422 589
@@ -432,7 +599,7 @@ int main(int argc, char *argv[]) · 9 unmodified lines
432 }599 }
433 }600 }
434 601
435 test_guest_debug_exceptions();602 test_guest_debug_exceptions_all(aa64dfr0);
436 test_single_step_from_userspace(ss_iteration);603 test_single_step_from_userspace(ss_iteration);
437 604
438 return 0;605 return 0;
tools/testing/selftests/kvm/aarch64/page_fault_test.c+1117−0
@@ -0,0 +1,1117 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * page_fault_test.c - Test stage 2 faults.
4 *
5 * This test tries different combinations of guest accesses (e.g., write,
6 * S1PTW), backing source type (e.g., anon) and types of faults (e.g., read on
7 * hugetlbfs with a hole). It checks that the expected handling method is
8 * called (e.g., uffd faults with the right address and write/read flag).
9 */
10
11#define _GNU_SOURCE
12#include <linux/bitmap.h>
13#include <fcntl.h>
14#include <test_util.h>
15#include <kvm_util.h>
16#include <processor.h>
17#include <asm/sysreg.h>
18#include <linux/bitfield.h>
19#include "guest_modes.h"
20#include "userfaultfd_util.h"
21
22/* Guest virtual addresses that point to the test page and its PTE. */
23#define TEST_GVA 0xc0000000
24#define TEST_EXEC_GVA (TEST_GVA + 0x8)
25#define TEST_PTE_GVA 0xb0000000
26#define TEST_DATA 0x0123456789ABCDEF
27
28static uint64_t *guest_test_memory = (uint64_t *)TEST_GVA;
29
30#define CMD_NONE (0)
31#define CMD_SKIP_TEST (1ULL << 1)
32#define CMD_HOLE_PT (1ULL << 2)
33#define CMD_HOLE_DATA (1ULL << 3)
34#define CMD_CHECK_WRITE_IN_DIRTY_LOG (1ULL << 4)
35#define CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG (1ULL << 5)
36#define CMD_CHECK_NO_WRITE_IN_DIRTY_LOG (1ULL << 6)
37#define CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG (1ULL << 7)
38#define CMD_SET_PTE_AF (1ULL << 8)
39
40#define PREPARE_FN_NR 10
41#define CHECK_FN_NR 10
42
43static struct event_cnt {
44 int mmio_exits;
45 int fail_vcpu_runs;
46 int uffd_faults;
47 /* uffd_faults is incremented from multiple threads. */
48 pthread_mutex_t uffd_faults_mutex;
49} events;
50
51struct test_desc {
52 const char *name;
53 uint64_t mem_mark_cmd;
54 /* Skip the test if any prepare function returns false */
55 bool (*guest_prepare[PREPARE_FN_NR])(void);
56 void (*guest_test)(void);
57 void (*guest_test_check[CHECK_FN_NR])(void);
58 uffd_handler_t uffd_pt_handler;
59 uffd_handler_t uffd_data_handler;
60 void (*dabt_handler)(struct ex_regs *regs);
61 void (*iabt_handler)(struct ex_regs *regs);
62 void (*mmio_handler)(struct kvm_vm *vm, struct kvm_run *run);
63 void (*fail_vcpu_run_handler)(int ret);
64 uint32_t pt_memslot_flags;
65 uint32_t data_memslot_flags;
66 bool skip;
67 struct event_cnt expected_events;
68};
69
70struct test_params {
71 enum vm_mem_backing_src_type src_type;
72 struct test_desc *test_desc;
73};
74
75static inline void flush_tlb_page(uint64_t vaddr)
76{
77 uint64_t page = vaddr >> 12;
78
79 dsb(ishst);
80 asm volatile("tlbi vaae1is, %0" :: "r" (page));
81 dsb(ish);
82 isb();
83}
84
85static void guest_write64(void)
86{
87 uint64_t val;
88
89 WRITE_ONCE(*guest_test_memory, TEST_DATA);
90 val = READ_ONCE(*guest_test_memory);
91 GUEST_ASSERT_EQ(val, TEST_DATA);
92}
93
94/* Check the system for atomic instructions. */
95static bool guest_check_lse(void)
96{
97 uint64_t isar0 = read_sysreg(id_aa64isar0_el1);
98 uint64_t atomic;
99
100 atomic = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR0_ATOMICS), isar0);
101 return atomic >= 2;
102}
103
104static bool guest_check_dc_zva(void)
105{
106 uint64_t dczid = read_sysreg(dczid_el0);
107 uint64_t dzp = FIELD_GET(ARM64_FEATURE_MASK(DCZID_DZP), dczid);
108
109 return dzp == 0;
110}
111
112/* Compare and swap instruction. */
113static void guest_cas(void)
114{
115 uint64_t val;
116
117 GUEST_ASSERT(guest_check_lse());
118 asm volatile(".arch_extension lse\n"
119 "casal %0, %1, [%2]\n"
120 :: "r" (0), "r" (TEST_DATA), "r" (guest_test_memory));
121 val = READ_ONCE(*guest_test_memory);
122 GUEST_ASSERT_EQ(val, TEST_DATA);
123}
124
125static void guest_read64(void)
126{
127 uint64_t val;
128
129 val = READ_ONCE(*guest_test_memory);
130 GUEST_ASSERT_EQ(val, 0);
131}
132
133/* Address translation instruction */
134static void guest_at(void)
135{
136 uint64_t par;
137
138 asm volatile("at s1e1r, %0" :: "r" (guest_test_memory));
139 par = read_sysreg(par_el1);
140 isb();
141
142 /* Bit 1 indicates whether the AT was successful */
143 GUEST_ASSERT_EQ(par & 1, 0);
144}
145
146/*
147 * The size of the block written by "dc zva" is guaranteed to be between (2 <<
148 * 0) and (2 << 9), which is safe in our case as we need the write to happen
149 * for at least a word, and not more than a page.
150 */
151static void guest_dc_zva(void)
152{
153 uint16_t val;
154
155 asm volatile("dc zva, %0" :: "r" (guest_test_memory));
156 dsb(ish);
157 val = READ_ONCE(*guest_test_memory);
158 GUEST_ASSERT_EQ(val, 0);
159}
160
161/*
162 * Pre-indexing loads and stores don't have a valid syndrome (ESR_EL2.ISV==0).
163 * And that's special because KVM must take special care with those: they
164 * should still count as accesses for dirty logging or user-faulting, but
165 * should be handled differently on mmio.
166 */
167static void guest_ld_preidx(void)
168{
169 uint64_t val;
170 uint64_t addr = TEST_GVA - 8;
171
172 /*
173 * This ends up accessing "TEST_GVA + 8 - 8", where "TEST_GVA - 8" is
174 * in a gap between memslots not backing by anything.
175 */
176 asm volatile("ldr %0, [%1, #8]!"
177 : "=r" (val), "+r" (addr));
178 GUEST_ASSERT_EQ(val, 0);
179 GUEST_ASSERT_EQ(addr, TEST_GVA);
180}
181
182static void guest_st_preidx(void)
183{
184 uint64_t val = TEST_DATA;
185 uint64_t addr = TEST_GVA - 8;
186
187 asm volatile("str %0, [%1, #8]!"
188 : "+r" (val), "+r" (addr));
189
190 GUEST_ASSERT_EQ(addr, TEST_GVA);
191 val = READ_ONCE(*guest_test_memory);
192}
193
194static bool guest_set_ha(void)
195{
196 uint64_t mmfr1 = read_sysreg(id_aa64mmfr1_el1);
197 uint64_t hadbs, tcr;
198
199 /* Skip if HA is not supported. */
200 hadbs = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_HADBS), mmfr1);
201 if (hadbs == 0)
202 return false;
203
204 tcr = read_sysreg(tcr_el1) | TCR_EL1_HA;
205 write_sysreg(tcr, tcr_el1);
206 isb();
207
208 return true;
209}
210
211static bool guest_clear_pte_af(void)
212{
213 *((uint64_t *)TEST_PTE_GVA) &= ~PTE_AF;
214 flush_tlb_page(TEST_GVA);
215
216 return true;
217}
218
219static void guest_check_pte_af(void)
220{
221 dsb(ish);
222 GUEST_ASSERT_EQ(*((uint64_t *)TEST_PTE_GVA) & PTE_AF, PTE_AF);
223}
224
225static void guest_check_write_in_dirty_log(void)
226{
227 GUEST_SYNC(CMD_CHECK_WRITE_IN_DIRTY_LOG);
228}
229
230static void guest_check_no_write_in_dirty_log(void)
231{
232 GUEST_SYNC(CMD_CHECK_NO_WRITE_IN_DIRTY_LOG);
233}
234
235static void guest_check_s1ptw_wr_in_dirty_log(void)
236{
237 GUEST_SYNC(CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG);
238}
239
240static void guest_exec(void)
241{
242 int (*code)(void) = (int (*)(void))TEST_EXEC_GVA;
243 int ret;
244
245 ret = code();
246 GUEST_ASSERT_EQ(ret, 0x77);
247}
248
249static bool guest_prepare(struct test_desc *test)
250{
251 bool (*prepare_fn)(void);
252 int i;
253
254 for (i = 0; i < PREPARE_FN_NR; i++) {
255 prepare_fn = test->guest_prepare[i];
256 if (prepare_fn && !prepare_fn())
257 return false;
258 }
259
260 return true;
261}
262
263static void guest_test_check(struct test_desc *test)
264{
265 void (*check_fn)(void);
266 int i;
267
268 for (i = 0; i < CHECK_FN_NR; i++) {
269 check_fn = test->guest_test_check[i];
270 if (check_fn)
271 check_fn();
272 }
273}
274
275static void guest_code(struct test_desc *test)
276{
277 if (!guest_prepare(test))
278 GUEST_SYNC(CMD_SKIP_TEST);
279
280 GUEST_SYNC(test->mem_mark_cmd);
281
282 if (test->guest_test)
283 test->guest_test();
284
285 guest_test_check(test);
286 GUEST_DONE();
287}
288
289static void no_dabt_handler(struct ex_regs *regs)
290{
291 GUEST_ASSERT_1(false, read_sysreg(far_el1));
292}
293
294static void no_iabt_handler(struct ex_regs *regs)
295{
296 GUEST_ASSERT_1(false, regs->pc);
297}
298
299static struct uffd_args {
300 char *copy;
301 void *hva;
302 uint64_t paging_size;
303} pt_args, data_args;
304
305/* Returns true to continue the test, and false if it should be skipped. */
306static int uffd_generic_handler(int uffd_mode, int uffd, struct uffd_msg *msg,
307 struct uffd_args *args, bool expect_write)
308{
309 uint64_t addr = msg->arg.pagefault.address;
310 uint64_t flags = msg->arg.pagefault.flags;
311 struct uffdio_copy copy;
312 int ret;
313
314 TEST_ASSERT(uffd_mode == UFFDIO_REGISTER_MODE_MISSING,
315 "The only expected UFFD mode is MISSING");
316 ASSERT_EQ(!!(flags & UFFD_PAGEFAULT_FLAG_WRITE), expect_write);
317 ASSERT_EQ(addr, (uint64_t)args->hva);
318
319 pr_debug("uffd fault: addr=%p write=%d\n",
320 (void *)addr, !!(flags & UFFD_PAGEFAULT_FLAG_WRITE));
321
322 copy.src = (uint64_t)args->copy;
323 copy.dst = addr;
324 copy.len = args->paging_size;
325 copy.mode = 0;
326
327 ret = ioctl(uffd, UFFDIO_COPY, &copy);
328 if (ret == -1) {
329 pr_info("Failed UFFDIO_COPY in 0x%lx with errno: %d\n",
330 addr, errno);
331 return ret;
332 }
333
334 pthread_mutex_lock(&events.uffd_faults_mutex);
335 events.uffd_faults += 1;
336 pthread_mutex_unlock(&events.uffd_faults_mutex);
337 return 0;
338}
339
340static int uffd_pt_write_handler(int mode, int uffd, struct uffd_msg *msg)
341{
342 return uffd_generic_handler(mode, uffd, msg, &pt_args, true);
343}
344
345static int uffd_data_write_handler(int mode, int uffd, struct uffd_msg *msg)
346{
347 return uffd_generic_handler(mode, uffd, msg, &data_args, true);
348}
349
350static int uffd_data_read_handler(int mode, int uffd, struct uffd_msg *msg)
351{
352 return uffd_generic_handler(mode, uffd, msg, &data_args, false);
353}
354
355static void setup_uffd_args(struct userspace_mem_region *region,
356 struct uffd_args *args)
357{
358 args->hva = (void *)region->region.userspace_addr;
359 args->paging_size = region->region.memory_size;
360
361 args->copy = malloc(args->paging_size);
362 TEST_ASSERT(args->copy, "Failed to allocate data copy.");
363 memcpy(args->copy, args->hva, args->paging_size);
364}
365
366static void setup_uffd(struct kvm_vm *vm, struct test_params *p,
367 struct uffd_desc **pt_uffd, struct uffd_desc **data_uffd)
368{
369 struct test_desc *test = p->test_desc;
370 int uffd_mode = UFFDIO_REGISTER_MODE_MISSING;
371
372 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_PT), &pt_args);
373 setup_uffd_args(vm_get_mem_region(vm, MEM_REGION_TEST_DATA), &data_args);
374
375 *pt_uffd = NULL;
376 if (test->uffd_pt_handler)
377 *pt_uffd = uffd_setup_demand_paging(uffd_mode, 0,
378 pt_args.hva,
379 pt_args.paging_size,
380 test->uffd_pt_handler);
381
382 *data_uffd = NULL;
383 if (test->uffd_data_handler)
384 *data_uffd = uffd_setup_demand_paging(uffd_mode, 0,
385 data_args.hva,
386 data_args.paging_size,
387 test->uffd_data_handler);
388}
389
390static void free_uffd(struct test_desc *test, struct uffd_desc *pt_uffd,
391 struct uffd_desc *data_uffd)
392{
393 if (test->uffd_pt_handler)
394 uffd_stop_demand_paging(pt_uffd);
395 if (test->uffd_data_handler)
396 uffd_stop_demand_paging(data_uffd);
397
398 free(pt_args.copy);
399 free(data_args.copy);
400}
401
402static int uffd_no_handler(int mode, int uffd, struct uffd_msg *msg)
403{
404 TEST_FAIL("There was no UFFD fault expected.");
405 return -1;
406}
407
408/* Returns false if the test should be skipped. */
409static bool punch_hole_in_backing_store(struct kvm_vm *vm,
410 struct userspace_mem_region *region)
411{
412 void *hva = (void *)region->region.userspace_addr;
413 uint64_t paging_size = region->region.memory_size;
414 int ret, fd = region->fd;
415
416 if (fd != -1) {
417 ret = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
418 0, paging_size);
419 TEST_ASSERT(ret == 0, "fallocate failed\n");
420 } else {
421 ret = madvise(hva, paging_size, MADV_DONTNEED);
422 TEST_ASSERT(ret == 0, "madvise failed\n");
423 }
424
425 return true;
426}
427
428static void mmio_on_test_gpa_handler(struct kvm_vm *vm, struct kvm_run *run)
429{
430 struct userspace_mem_region *region;
431 void *hva;
432
433 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
434 hva = (void *)region->region.userspace_addr;
435
436 ASSERT_EQ(run->mmio.phys_addr, region->region.guest_phys_addr);
437
438 memcpy(hva, run->mmio.data, run->mmio.len);
439 events.mmio_exits += 1;
440}
441
442static void mmio_no_handler(struct kvm_vm *vm, struct kvm_run *run)
443{
444 uint64_t data;
445
446 memcpy(&data, run->mmio.data, sizeof(data));
447 pr_debug("addr=%lld len=%d w=%d data=%lx\n",
448 run->mmio.phys_addr, run->mmio.len,
449 run->mmio.is_write, data);
450 TEST_FAIL("There was no MMIO exit expected.");
451}
452
453static bool check_write_in_dirty_log(struct kvm_vm *vm,
454 struct userspace_mem_region *region,
455 uint64_t host_pg_nr)
456{
457 unsigned long *bmap;
458 bool first_page_dirty;
459 uint64_t size = region->region.memory_size;
460
461 /* getpage_size() is not always equal to vm->page_size */
462 bmap = bitmap_zalloc(size / getpagesize());
463 kvm_vm_get_dirty_log(vm, region->region.slot, bmap);
464 first_page_dirty = test_bit(host_pg_nr, bmap);
465 free(bmap);
466 return first_page_dirty;
467}
468
469/* Returns true to continue the test, and false if it should be skipped. */
470static bool handle_cmd(struct kvm_vm *vm, int cmd)
471{
472 struct userspace_mem_region *data_region, *pt_region;
473 bool continue_test = true;
474
475 data_region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
476 pt_region = vm_get_mem_region(vm, MEM_REGION_PT);
477
478 if (cmd == CMD_SKIP_TEST)
479 continue_test = false;
480
481 if (cmd & CMD_HOLE_PT)
482 continue_test = punch_hole_in_backing_store(vm, pt_region);
483 if (cmd & CMD_HOLE_DATA)
484 continue_test = punch_hole_in_backing_store(vm, data_region);
485 if (cmd & CMD_CHECK_WRITE_IN_DIRTY_LOG)
486 TEST_ASSERT(check_write_in_dirty_log(vm, data_region, 0),
487 "Missing write in dirty log");
488 if (cmd & CMD_CHECK_S1PTW_WR_IN_DIRTY_LOG)
489 TEST_ASSERT(check_write_in_dirty_log(vm, pt_region, 0),
490 "Missing s1ptw write in dirty log");
491 if (cmd & CMD_CHECK_NO_WRITE_IN_DIRTY_LOG)
492 TEST_ASSERT(!check_write_in_dirty_log(vm, data_region, 0),
493 "Unexpected write in dirty log");
494 if (cmd & CMD_CHECK_NO_S1PTW_WR_IN_DIRTY_LOG)
495 TEST_ASSERT(!check_write_in_dirty_log(vm, pt_region, 0),
496 "Unexpected s1ptw write in dirty log");
497
498 return continue_test;
499}
500
501void fail_vcpu_run_no_handler(int ret)
502{
503 TEST_FAIL("Unexpected vcpu run failure\n");
504}
505
506void fail_vcpu_run_mmio_no_syndrome_handler(int ret)
507{
508 TEST_ASSERT(errno == ENOSYS,
509 "The mmio handler should have returned not implemented.");
510 events.fail_vcpu_runs += 1;
511}
512
513typedef uint32_t aarch64_insn_t;
514extern aarch64_insn_t __exec_test[2];
515
516noinline void __return_0x77(void)
517{
518 asm volatile("__exec_test: mov x0, #0x77\n"
519 "ret\n");
520}
521
522/*
523 * Note that this function runs on the host before the test VM starts: there's
524 * no need to sync the D$ and I$ caches.
525 */
526static void load_exec_code_for_test(struct kvm_vm *vm)
527{
528 uint64_t *code;
529 struct userspace_mem_region *region;
530 void *hva;
531
532 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
533 hva = (void *)region->region.userspace_addr;
534
535 assert(TEST_EXEC_GVA > TEST_GVA);
536 code = hva + TEST_EXEC_GVA - TEST_GVA;
537 memcpy(code, __exec_test, sizeof(__exec_test));
538}
539
540static void setup_abort_handlers(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
541 struct test_desc *test)
542{
543 vm_init_descriptor_tables(vm);
544 vcpu_init_descriptor_tables(vcpu);
545
546 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
547 ESR_EC_DABT, no_dabt_handler);
548 vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT,
549 ESR_EC_IABT, no_iabt_handler);
550}
551
552static void setup_gva_maps(struct kvm_vm *vm)
553{
554 struct userspace_mem_region *region;
555 uint64_t pte_gpa;
556
557 region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
558 /* Map TEST_GVA first. This will install a new PTE. */
559 virt_pg_map(vm, TEST_GVA, region->region.guest_phys_addr);
560 /* Then map TEST_PTE_GVA to the above PTE. */
561 pte_gpa = addr_hva2gpa(vm, virt_get_pte_hva(vm, TEST_GVA));
562 virt_pg_map(vm, TEST_PTE_GVA, pte_gpa);
563}
564
565enum pf_test_memslots {
566 CODE_AND_DATA_MEMSLOT,
567 PAGE_TABLE_MEMSLOT,
568 TEST_DATA_MEMSLOT,
569};
570
571/*
572 * Create a memslot for code and data at pfn=0, and test-data and PT ones
573 * at max_gfn.
574 */
575static void setup_memslots(struct kvm_vm *vm, struct test_params *p)
576{
577 uint64_t backing_src_pagesz = get_backing_src_pagesz(p->src_type);
578 uint64_t guest_page_size = vm->page_size;
579 uint64_t max_gfn = vm_compute_max_gfn(vm);
580 /* Enough for 2M of code when using 4K guest pages. */
581 uint64_t code_npages = 512;
582 uint64_t pt_size, data_size, data_gpa;
583
584 /*
585 * This test requires 1 pgd, 2 pud, 4 pmd, and 6 pte pages when using
586 * VM_MODE_P48V48_4K. Note that the .text takes ~1.6MBs. That's 13
587 * pages. VM_MODE_P48V48_4K is the mode with most PT pages; let's use
588 * twice that just in case.
589 */
590 pt_size = 26 * guest_page_size;
591
592 /* memslot sizes and gpa's must be aligned to the backing page size */
593 pt_size = align_up(pt_size, backing_src_pagesz);
594 data_size = align_up(guest_page_size, backing_src_pagesz);
595 data_gpa = (max_gfn * guest_page_size) - data_size;
596 data_gpa = align_down(data_gpa, backing_src_pagesz);
597
598 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0,
599 CODE_AND_DATA_MEMSLOT, code_npages, 0);
600 vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT;
601 vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT;
602
603 vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size,
604 PAGE_TABLE_MEMSLOT, pt_size / guest_page_size,
605 p->test_desc->pt_memslot_flags);
606 vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT;
607
608 vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT,
609 data_size / guest_page_size,
610 p->test_desc->data_memslot_flags);
611 vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT;
612}
613
614static void setup_ucall(struct kvm_vm *vm)
615{
616 struct userspace_mem_region *region = vm_get_mem_region(vm, MEM_REGION_TEST_DATA);
617
618 ucall_init(vm, region->region.guest_phys_addr + region->region.memory_size);
619}
620
621static void setup_default_handlers(struct test_desc *test)
622{
623 if (!test->mmio_handler)
624 test->mmio_handler = mmio_no_handler;
625
626 if (!test->fail_vcpu_run_handler)
627 test->fail_vcpu_run_handler = fail_vcpu_run_no_handler;
628}
629
630static void check_event_counts(struct test_desc *test)
631{
632 ASSERT_EQ(test->expected_events.uffd_faults, events.uffd_faults);
633 ASSERT_EQ(test->expected_events.mmio_exits, events.mmio_exits);
634 ASSERT_EQ(test->expected_events.fail_vcpu_runs, events.fail_vcpu_runs);
635}
636
637static void print_test_banner(enum vm_guest_mode mode, struct test_params *p)
638{
639 struct test_desc *test = p->test_desc;
640
641 pr_debug("Test: %s\n", test->name);
642 pr_debug("Testing guest mode: %s\n", vm_guest_mode_string(mode));
643 pr_debug("Testing memory backing src type: %s\n",
644 vm_mem_backing_src_alias(p->src_type)->name);
645}
646
647static void reset_event_counts(void)
648{
649 memset(&events, 0, sizeof(events));
650}
651
652/*
653 * This function either succeeds, skips the test (after setting test->skip), or
654 * fails with a TEST_FAIL that aborts all tests.
655 */
656static void vcpu_run_loop(struct kvm_vm *vm, struct kvm_vcpu *vcpu,
657 struct test_desc *test)
658{
659 struct kvm_run *run;
660 struct ucall uc;
661 int ret;
662
663 run = vcpu->run;
664
665 for (;;) {
666 ret = _vcpu_run(vcpu);
667 if (ret) {
668 test->fail_vcpu_run_handler(ret);
669 goto done;
670 }
671
672 switch (get_ucall(vcpu, &uc)) {
673 case UCALL_SYNC:
674 if (!handle_cmd(vm, uc.args[1])) {
675 test->skip = true;
676 goto done;
677 }
678 break;
679 case UCALL_ABORT:
680 REPORT_GUEST_ASSERT_2(uc, "values: %#lx, %#lx");
681 break;
682 case UCALL_DONE:
683 goto done;
684 case UCALL_NONE:
685 if (run->exit_reason == KVM_EXIT_MMIO)
686 test->mmio_handler(vm, run);
687 break;
688 default:
689 TEST_FAIL("Unknown ucall %lu", uc.cmd);
690 }
691 }
692
693done:
694 pr_debug(test->skip ? "Skipped.\n" : "Done.\n");
695}
696
697static void run_test(enum vm_guest_mode mode, void *arg)
698{
699 struct test_params *p = (struct test_params *)arg;
700 struct test_desc *test = p->test_desc;
701 struct kvm_vm *vm;
702 struct kvm_vcpu *vcpu;
703 struct uffd_desc *pt_uffd, *data_uffd;
704
705 print_test_banner(mode, p);
706
707 vm = ____vm_create(mode);
708 setup_memslots(vm, p);
709 kvm_vm_elf_load(vm, program_invocation_name);
710 setup_ucall(vm);
711 vcpu = vm_vcpu_add(vm, 0, guest_code);
712
713 setup_gva_maps(vm);
714
715 reset_event_counts();
716
717 /*
718 * Set some code in the data memslot for the guest to execute (only
719 * applicable to the EXEC tests). This has to be done before
720 * setup_uffd() as that function copies the memslot data for the uffd
721 * handler.
722 */
723 load_exec_code_for_test(vm);
724 setup_uffd(vm, p, &pt_uffd, &data_uffd);
725 setup_abort_handlers(vm, vcpu, test);
726 setup_default_handlers(test);
727 vcpu_args_set(vcpu, 1, test);
728
729 vcpu_run_loop(vm, vcpu, test);
730
731 kvm_vm_free(vm);
732 free_uffd(test, pt_uffd, data_uffd);
733
734 /*
735 * Make sure we check the events after the uffd threads have exited,
736 * which means they updated their respective event counters.
737 */
738 if (!test->skip)
739 check_event_counts(test);
740}
741
742static void help(char *name)
743{
744 puts("");
745 printf("usage: %s [-h] [-s mem-type]\n", name);
746 puts("");
747 guest_modes_help();
748 backing_src_help("-s");
749 puts("");
750}
751
752#define SNAME(s) #s
753#define SCAT2(a, b) SNAME(a ## _ ## b)
754#define SCAT3(a, b, c) SCAT2(a, SCAT2(b, c))
755#define SCAT4(a, b, c, d) SCAT2(a, SCAT3(b, c, d))
756
757#define _CHECK(_test) _CHECK_##_test
758#define _PREPARE(_test) _PREPARE_##_test
759#define _PREPARE_guest_read64 NULL
760#define _PREPARE_guest_ld_preidx NULL
761#define _PREPARE_guest_write64 NULL
762#define _PREPARE_guest_st_preidx NULL
763#define _PREPARE_guest_exec NULL
764#define _PREPARE_guest_at NULL
765#define _PREPARE_guest_dc_zva guest_check_dc_zva
766#define _PREPARE_guest_cas guest_check_lse
767
768/* With or without access flag checks */
769#define _PREPARE_with_af guest_set_ha, guest_clear_pte_af
770#define _PREPARE_no_af NULL
771#define _CHECK_with_af guest_check_pte_af
772#define _CHECK_no_af NULL
773
774/* Performs an access and checks that no faults were triggered. */
775#define TEST_ACCESS(_access, _with_af, _mark_cmd) \
776{ \
777 .name = SCAT3(_access, _with_af, #_mark_cmd), \
778 .guest_prepare = { _PREPARE(_with_af), \
779 _PREPARE(_access) }, \
780 .mem_mark_cmd = _mark_cmd, \
781 .guest_test = _access, \
782 .guest_test_check = { _CHECK(_with_af) }, \
783 .expected_events = { 0 }, \
784}
785
786#define TEST_UFFD(_access, _with_af, _mark_cmd, \
787 _uffd_data_handler, _uffd_pt_handler, _uffd_faults) \
788{ \
789 .name = SCAT4(uffd, _access, _with_af, #_mark_cmd), \
790 .guest_prepare = { _PREPARE(_with_af), \
791 _PREPARE(_access) }, \
792 .guest_test = _access, \
793 .mem_mark_cmd = _mark_cmd, \
794 .guest_test_check = { _CHECK(_with_af) }, \
795 .uffd_data_handler = _uffd_data_handler, \
796 .uffd_pt_handler = _uffd_pt_handler, \
797 .expected_events = { .uffd_faults = _uffd_faults, }, \
798}
799
800#define TEST_DIRTY_LOG(_access, _with_af, _test_check) \
801{ \
802 .name = SCAT3(dirty_log, _access, _with_af), \
803 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
804 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
805 .guest_prepare = { _PREPARE(_with_af), \
806 _PREPARE(_access) }, \
807 .guest_test = _access, \
808 .guest_test_check = { _CHECK(_with_af), _test_check, \
809 guest_check_s1ptw_wr_in_dirty_log}, \
810 .expected_events = { 0 }, \
811}
812
813#define TEST_UFFD_AND_DIRTY_LOG(_access, _with_af, _uffd_data_handler, \
814 _uffd_faults, _test_check) \
815{ \
816 .name = SCAT3(uffd_and_dirty_log, _access, _with_af), \
817 .data_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
818 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
819 .guest_prepare = { _PREPARE(_with_af), \
820 _PREPARE(_access) }, \
821 .guest_test = _access, \
822 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
823 .guest_test_check = { _CHECK(_with_af), _test_check }, \
824 .uffd_data_handler = _uffd_data_handler, \
825 .uffd_pt_handler = uffd_pt_write_handler, \
826 .expected_events = { .uffd_faults = _uffd_faults, }, \
827}
828
829#define TEST_RO_MEMSLOT(_access, _mmio_handler, _mmio_exits) \
830{ \
831 .name = SCAT3(ro_memslot, _access, _with_af), \
832 .data_memslot_flags = KVM_MEM_READONLY, \
833 .guest_prepare = { _PREPARE(_access) }, \
834 .guest_test = _access, \
835 .mmio_handler = _mmio_handler, \
836 .expected_events = { .mmio_exits = _mmio_exits }, \
837}
838
839#define TEST_RO_MEMSLOT_NO_SYNDROME(_access) \
840{ \
841 .name = SCAT2(ro_memslot_no_syndrome, _access), \
842 .data_memslot_flags = KVM_MEM_READONLY, \
843 .guest_test = _access, \
844 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
845 .expected_events = { .fail_vcpu_runs = 1 }, \
846}
847
848#define TEST_RO_MEMSLOT_AND_DIRTY_LOG(_access, _mmio_handler, _mmio_exits, \
849 _test_check) \
850{ \
851 .name = SCAT3(ro_memslot, _access, _with_af), \
852 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \
853 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
854 .guest_prepare = { _PREPARE(_access) }, \
855 .guest_test = _access, \
856 .guest_test_check = { _test_check }, \
857 .mmio_handler = _mmio_handler, \
858 .expected_events = { .mmio_exits = _mmio_exits}, \
859}
860
861#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(_access, _test_check) \
862{ \
863 .name = SCAT2(ro_memslot_no_syn_and_dlog, _access), \
864 .data_memslot_flags = KVM_MEM_READONLY | KVM_MEM_LOG_DIRTY_PAGES, \
865 .pt_memslot_flags = KVM_MEM_LOG_DIRTY_PAGES, \
866 .guest_test = _access, \
867 .guest_test_check = { _test_check }, \
868 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
869 .expected_events = { .fail_vcpu_runs = 1 }, \
870}
871
872#define TEST_RO_MEMSLOT_AND_UFFD(_access, _mmio_handler, _mmio_exits, \
873 _uffd_data_handler, _uffd_faults) \
874{ \
875 .name = SCAT2(ro_memslot_uffd, _access), \
876 .data_memslot_flags = KVM_MEM_READONLY, \
877 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
878 .guest_prepare = { _PREPARE(_access) }, \
879 .guest_test = _access, \
880 .uffd_data_handler = _uffd_data_handler, \
881 .uffd_pt_handler = uffd_pt_write_handler, \
882 .mmio_handler = _mmio_handler, \
883 .expected_events = { .mmio_exits = _mmio_exits, \
884 .uffd_faults = _uffd_faults }, \
885}
886
887#define TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(_access, _uffd_data_handler, \
888 _uffd_faults) \
889{ \
890 .name = SCAT2(ro_memslot_no_syndrome, _access), \
891 .data_memslot_flags = KVM_MEM_READONLY, \
892 .mem_mark_cmd = CMD_HOLE_DATA | CMD_HOLE_PT, \
893 .guest_test = _access, \
894 .uffd_data_handler = _uffd_data_handler, \
895 .uffd_pt_handler = uffd_pt_write_handler, \
896 .fail_vcpu_run_handler = fail_vcpu_run_mmio_no_syndrome_handler, \
897 .expected_events = { .fail_vcpu_runs = 1, \
898 .uffd_faults = _uffd_faults }, \
899}
900
901static struct test_desc tests[] = {
902
903 /* Check that HW is setting the Access Flag (AF) (sanity checks). */
904 TEST_ACCESS(guest_read64, with_af, CMD_NONE),
905 TEST_ACCESS(guest_ld_preidx, with_af, CMD_NONE),
906 TEST_ACCESS(guest_cas, with_af, CMD_NONE),
907 TEST_ACCESS(guest_write64, with_af, CMD_NONE),
908 TEST_ACCESS(guest_st_preidx, with_af, CMD_NONE),
909 TEST_ACCESS(guest_dc_zva, with_af, CMD_NONE),
910 TEST_ACCESS(guest_exec, with_af, CMD_NONE),
911
912 /*
913 * Punch a hole in the data backing store, and then try multiple
914 * accesses: reads should rturn zeroes, and writes should
915 * re-populate the page. Moreover, the test also check that no
916 * exception was generated in the guest. Note that this
917 * reading/writing behavior is the same as reading/writing a
918 * punched page (with fallocate(FALLOC_FL_PUNCH_HOLE)) from
919 * userspace.
920 */
921 TEST_ACCESS(guest_read64, no_af, CMD_HOLE_DATA),
922 TEST_ACCESS(guest_cas, no_af, CMD_HOLE_DATA),
923 TEST_ACCESS(guest_ld_preidx, no_af, CMD_HOLE_DATA),
924 TEST_ACCESS(guest_write64, no_af, CMD_HOLE_DATA),
925 TEST_ACCESS(guest_st_preidx, no_af, CMD_HOLE_DATA),
926 TEST_ACCESS(guest_at, no_af, CMD_HOLE_DATA),
927 TEST_ACCESS(guest_dc_zva, no_af, CMD_HOLE_DATA),
928
929 /*
930 * Punch holes in the data and PT backing stores and mark them for
931 * userfaultfd handling. This should result in 2 faults: the access
932 * on the data backing store, and its respective S1 page table walk
933 * (S1PTW).
934 */
935 TEST_UFFD(guest_read64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
936 uffd_data_read_handler, uffd_pt_write_handler, 2),
937 /* no_af should also lead to a PT write. */
938 TEST_UFFD(guest_read64, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,
939 uffd_data_read_handler, uffd_pt_write_handler, 2),
940 /* Note how that cas invokes the read handler. */
941 TEST_UFFD(guest_cas, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
942 uffd_data_read_handler, uffd_pt_write_handler, 2),
943 /*
944 * Can't test guest_at with_af as it's IMPDEF whether the AF is set.
945 * The S1PTW fault should still be marked as a write.
946 */
947 TEST_UFFD(guest_at, no_af, CMD_HOLE_DATA | CMD_HOLE_PT,
948 uffd_data_read_handler, uffd_pt_write_handler, 1),
949 TEST_UFFD(guest_ld_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
950 uffd_data_read_handler, uffd_pt_write_handler, 2),
951 TEST_UFFD(guest_write64, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
952 uffd_data_write_handler, uffd_pt_write_handler, 2),
953 TEST_UFFD(guest_dc_zva, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
954 uffd_data_write_handler, uffd_pt_write_handler, 2),
955 TEST_UFFD(guest_st_preidx, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
956 uffd_data_write_handler, uffd_pt_write_handler, 2),
957 TEST_UFFD(guest_exec, with_af, CMD_HOLE_DATA | CMD_HOLE_PT,
958 uffd_data_read_handler, uffd_pt_write_handler, 2),
959
960 /*
961 * Try accesses when the data and PT memory regions are both
962 * tracked for dirty logging.
963 */
964 TEST_DIRTY_LOG(guest_read64, with_af, guest_check_no_write_in_dirty_log),
965 /* no_af should also lead to a PT write. */
966 TEST_DIRTY_LOG(guest_read64, no_af, guest_check_no_write_in_dirty_log),
967 TEST_DIRTY_LOG(guest_ld_preidx, with_af, guest_check_no_write_in_dirty_log),
968 TEST_DIRTY_LOG(guest_at, no_af, guest_check_no_write_in_dirty_log),
969 TEST_DIRTY_LOG(guest_exec, with_af, guest_check_no_write_in_dirty_log),
970 TEST_DIRTY_LOG(guest_write64, with_af, guest_check_write_in_dirty_log),
971 TEST_DIRTY_LOG(guest_cas, with_af, guest_check_write_in_dirty_log),
972 TEST_DIRTY_LOG(guest_dc_zva, with_af, guest_check_write_in_dirty_log),
973 TEST_DIRTY_LOG(guest_st_preidx, with_af, guest_check_write_in_dirty_log),
974
975 /*
976 * Access when the data and PT memory regions are both marked for
977 * dirty logging and UFFD at the same time. The expected result is
978 * that writes should mark the dirty log and trigger a userfaultfd
979 * write fault. Reads/execs should result in a read userfaultfd
980 * fault, and nothing in the dirty log. Any S1PTW should result in
981 * a write in the dirty log and a userfaultfd write.
982 */
983 TEST_UFFD_AND_DIRTY_LOG(guest_read64, with_af, uffd_data_read_handler, 2,
984 guest_check_no_write_in_dirty_log),
985 /* no_af should also lead to a PT write. */
986 TEST_UFFD_AND_DIRTY_LOG(guest_read64, no_af, uffd_data_read_handler, 2,
987 guest_check_no_write_in_dirty_log),
988 TEST_UFFD_AND_DIRTY_LOG(guest_ld_preidx, with_af, uffd_data_read_handler,
989 2, guest_check_no_write_in_dirty_log),
990 TEST_UFFD_AND_DIRTY_LOG(guest_at, with_af, 0, 1,
991 guest_check_no_write_in_dirty_log),
992 TEST_UFFD_AND_DIRTY_LOG(guest_exec, with_af, uffd_data_read_handler, 2,
993 guest_check_no_write_in_dirty_log),
994 TEST_UFFD_AND_DIRTY_LOG(guest_write64, with_af, uffd_data_write_handler,
995 2, guest_check_write_in_dirty_log),
996 TEST_UFFD_AND_DIRTY_LOG(guest_cas, with_af, uffd_data_read_handler, 2,
997 guest_check_write_in_dirty_log),
998 TEST_UFFD_AND_DIRTY_LOG(guest_dc_zva, with_af, uffd_data_write_handler,
999 2, guest_check_write_in_dirty_log),
1000 TEST_UFFD_AND_DIRTY_LOG(guest_st_preidx, with_af,
1001 uffd_data_write_handler, 2,
1002 guest_check_write_in_dirty_log),
1003
1004 /*
1005 * Try accesses when the data memory region is marked read-only
1006 * (with KVM_MEM_READONLY). Writes with a syndrome result in an
1007 * MMIO exit, writes with no syndrome (e.g., CAS) result in a
1008 * failed vcpu run, and reads/execs with and without syndroms do
1009 * not fault.
1010 */
1011 TEST_RO_MEMSLOT(guest_read64, 0, 0),
1012 TEST_RO_MEMSLOT(guest_ld_preidx, 0, 0),
1013 TEST_RO_MEMSLOT(guest_at, 0, 0),
1014 TEST_RO_MEMSLOT(guest_exec, 0, 0),
1015 TEST_RO_MEMSLOT(guest_write64, mmio_on_test_gpa_handler, 1),
1016 TEST_RO_MEMSLOT_NO_SYNDROME(guest_dc_zva),
1017 TEST_RO_MEMSLOT_NO_SYNDROME(guest_cas),
1018 TEST_RO_MEMSLOT_NO_SYNDROME(guest_st_preidx),
1019
1020 /*
1021 * Access when both the data region is both read-only and marked
1022 * for dirty logging at the same time. The expected result is that
1023 * for writes there should be no write in the dirty log. The
1024 * readonly handling is the same as if the memslot was not marked
1025 * for dirty logging: writes with a syndrome result in an MMIO
1026 * exit, and writes with no syndrome result in a failed vcpu run.
1027 */
1028 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_read64, 0, 0,
1029 guest_check_no_write_in_dirty_log),
1030 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_ld_preidx, 0, 0,
1031 guest_check_no_write_in_dirty_log),
1032 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_at, 0, 0,
1033 guest_check_no_write_in_dirty_log),
1034 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_exec, 0, 0,
1035 guest_check_no_write_in_dirty_log),
1036 TEST_RO_MEMSLOT_AND_DIRTY_LOG(guest_write64, mmio_on_test_gpa_handler,
1037 1, guest_check_no_write_in_dirty_log),
1038 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_dc_zva,
1039 guest_check_no_write_in_dirty_log),
1040 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_cas,
1041 guest_check_no_write_in_dirty_log),
1042 TEST_RO_MEMSLOT_NO_SYNDROME_AND_DIRTY_LOG(guest_st_preidx,
1043 guest_check_no_write_in_dirty_log),
1044
1045 /*
1046 * Access when the data region is both read-only and punched with
1047 * holes tracked with userfaultfd. The expected result is the
1048 * union of both userfaultfd and read-only behaviors. For example,
1049 * write accesses result in a userfaultfd write fault and an MMIO
1050 * exit. Writes with no syndrome result in a failed vcpu run and
1051 * no userfaultfd write fault. Reads result in userfaultfd getting
1052 * triggered.
1053 */
1054 TEST_RO_MEMSLOT_AND_UFFD(guest_read64, 0, 0,
1055 uffd_data_read_handler, 2),
1056 TEST_RO_MEMSLOT_AND_UFFD(guest_ld_preidx, 0, 0,
1057 uffd_data_read_handler, 2),
1058 TEST_RO_MEMSLOT_AND_UFFD(guest_at, 0, 0,
1059 uffd_no_handler, 1),
1060 TEST_RO_MEMSLOT_AND_UFFD(guest_exec, 0, 0,
1061 uffd_data_read_handler, 2),
1062 TEST_RO_MEMSLOT_AND_UFFD(guest_write64, mmio_on_test_gpa_handler, 1,
1063 uffd_data_write_handler, 2),
1064 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_cas,
1065 uffd_data_read_handler, 2),
1066 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_dc_zva,
1067 uffd_no_handler, 1),
1068 TEST_RO_MEMSLOT_NO_SYNDROME_AND_UFFD(guest_st_preidx,
1069 uffd_no_handler, 1),
1070
1071 { 0 }
1072};
1073
1074static void for_each_test_and_guest_mode(enum vm_mem_backing_src_type src_type)
1075{
1076 struct test_desc *t;
1077
1078 for (t = &tests[0]; t->name; t++) {
1079 if (t->skip)
1080 continue;
1081
1082 struct test_params p = {
1083 .src_type = src_type,
1084 .test_desc = t,
1085 };
1086
1087 for_each_guest_mode(run_test, &p);
1088 }
1089}
1090
1091int main(int argc, char *argv[])
1092{
1093 enum vm_mem_backing_src_type src_type;
1094 int opt;
1095
1096 setbuf(stdout, NULL);
1097
1098 src_type = DEFAULT_VM_MEM_SRC;
1099
1100 while ((opt = getopt(argc, argv, "hm:s:")) != -1) {
1101 switch (opt) {
1102 case 'm':
1103 guest_modes_cmdline(optarg);
1104 break;
1105 case 's':
1106 src_type = parse_backing_src_type(optarg);
1107 break;
1108 case 'h':
1109 default:
1110 help(argv[0]);
1111 exit(0);
1112 }
1113 }
1114
1115 for_each_test_and_guest_mode(src_type);
1116 return 0;
1117}
tools/testing/selftests/kvm/access_tracking_perf_test.c+1−7
@@ -58,9 +58,6 @@ static enum { · 57 unmodified lines
58 ITERATION_MARK_IDLE,58 ITERATION_MARK_IDLE,
59} iteration_work;59} iteration_work;
60 60
61/* Set to true when vCPU threads should exit. */
62static bool done;
63
64/* The iteration that was last completed by each vCPU. */61/* The iteration that was last completed by each vCPU. */
65static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];62static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
66 63
@@ -211,7 +208,7 @@ static bool spin_wait_for_next_iteration(int *current_iteration) · 144 unmodified lines
211 int last_iteration = *current_iteration;208 int last_iteration = *current_iteration;
212 209
213 do {210 do {
214 if (READ_ONCE(done))211 if (READ_ONCE(memstress_args.stop_vcpus))
215 return false;212 return false;
216 213
217 *current_iteration = READ_ONCE(iteration);214 *current_iteration = READ_ONCE(iteration);
@@ -321,9 +318,6 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 103 unmodified lines
321 mark_memory_idle(vm, nr_vcpus);318 mark_memory_idle(vm, nr_vcpus);
322 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory");319 access_memory(vm, nr_vcpus, ACCESS_READ, "Reading from idle memory");
323 320
324 /* Set done to signal the vCPU threads to exit */
325 done = true;
326
327 memstress_join_vcpu_threads(nr_vcpus);321 memstress_join_vcpu_threads(nr_vcpus);
328 memstress_destroy_vm(vm);322 memstress_destroy_vm(vm);
329}323}
tools/testing/selftests/kvm/demand_paging_test.c+30−198
@@ -22,23 +22,13 @@ · 21 unmodified lines
22#include "test_util.h"22#include "test_util.h"
23#include "memstress.h"23#include "memstress.h"
24#include "guest_modes.h"24#include "guest_modes.h"
25#include "userfaultfd_util.h"
25 26
26#ifdef __NR_userfaultfd27#ifdef __NR_userfaultfd
27 28
28#ifdef PRINT_PER_PAGE_UPDATES
29#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
30#else
31#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
32#endif
33
34#ifdef PRINT_PER_VCPU_UPDATES
35#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
36#else
37#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
38#endif
39
40static int nr_vcpus = 1;29static int nr_vcpus = 1;
41static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;30static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
31
42static size_t demand_paging_size;32static size_t demand_paging_size;
43static char *guest_data_prototype;33static char *guest_data_prototype;
44 34
@@ -67,9 +57,11 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) · 22 unmodified lines
67 ts_diff.tv_sec, ts_diff.tv_nsec);57 ts_diff.tv_sec, ts_diff.tv_nsec);
68}58}
69 59
70static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr)60static int handle_uffd_page_request(int uffd_mode, int uffd,
61 struct uffd_msg *msg)
71{62{
72 pid_t tid = syscall(__NR_gettid);63 pid_t tid = syscall(__NR_gettid);
64 uint64_t addr = msg->arg.pagefault.address;
73 struct timespec start;65 struct timespec start;
74 struct timespec ts_diff;66 struct timespec ts_diff;
75 int r;67 int r;
@@ -116,174 +108,32 @@ static int handle_uffd_page_request(int uffd_mode, int uffd, uint64_t addr) · 40 unmodified lines
116 return 0;108 return 0;
117}109}
118 110
119bool quit_uffd_thread;111struct test_params {
120
121struct uffd_handler_args {
122 int uffd_mode;112 int uffd_mode;
123 int uffd;113 useconds_t uffd_delay;
124 int pipefd;114 enum vm_mem_backing_src_type src_type;
125 useconds_t delay;115 bool partition_vcpu_memory_access;
126};116};
127 117
128static void *uffd_handler_thread_fn(void *arg)118static void prefault_mem(void *alias, uint64_t len)
129{
130 struct uffd_handler_args *uffd_args = (struct uffd_handler_args *)arg;
131 int uffd = uffd_args->uffd;
132 int pipefd = uffd_args->pipefd;
133 useconds_t delay = uffd_args->delay;
134 int64_t pages = 0;
135 struct timespec start;
136 struct timespec ts_diff;
137
138 clock_gettime(CLOCK_MONOTONIC, &start);
139 while (!quit_uffd_thread) {
140 struct uffd_msg msg;
141 struct pollfd pollfd[2];
142 char tmp_chr;
143 int r;
144 uint64_t addr;
145
146 pollfd[0].fd = uffd;
147 pollfd[0].events = POLLIN;
148 pollfd[1].fd = pipefd;
149 pollfd[1].events = POLLIN;
150
151 r = poll(pollfd, 2, -1);
152 switch (r) {
153 case -1:
154 pr_info("poll err");
155 continue;
156 case 0:
157 continue;
158 case 1:
159 break;
160 default:
161 pr_info("Polling uffd returned %d", r);
162 return NULL;
163 }
164
165 if (pollfd[0].revents & POLLERR) {
166 pr_info("uffd revents has POLLERR");
167 return NULL;
168 }
169
170 if (pollfd[1].revents & POLLIN) {
171 r = read(pollfd[1].fd, &tmp_chr, 1);
172 TEST_ASSERT(r == 1,
173 "Error reading pipefd in UFFD thread\n");
174 return NULL;
175 }
176
177 if (!(pollfd[0].revents & POLLIN))
178 continue;
179
180 r = read(uffd, &msg, sizeof(msg));
181 if (r == -1) {
182 if (errno == EAGAIN)
183 continue;
184 pr_info("Read of uffd got errno %d\n", errno);
185 return NULL;
186 }
187
188 if (r != sizeof(msg)) {
189 pr_info("Read on uffd returned unexpected size: %d bytes", r);
190 return NULL;
191 }
192
193 if (!(msg.event & UFFD_EVENT_PAGEFAULT))
194 continue;
195
196 if (delay)
197 usleep(delay);
198 addr = msg.arg.pagefault.address;
199 r = handle_uffd_page_request(uffd_args->uffd_mode, uffd, addr);
200 if (r < 0)
201 return NULL;
202 pages++;
203 }
204
205 ts_diff = timespec_elapsed(start);
206 PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
207 pages, ts_diff.tv_sec, ts_diff.tv_nsec,
208 pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
209
210 return NULL;
211}
212
213static void setup_demand_paging(struct kvm_vm *vm,
214 pthread_t *uffd_handler_thread, int pipefd,
215 int uffd_mode, useconds_t uffd_delay,
216 struct uffd_handler_args *uffd_args,
217 void *hva, void *alias, uint64_t len)
218{119{
219 bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);120 size_t p;
220 int uffd;
221 struct uffdio_api uffdio_api;
222 struct uffdio_register uffdio_register;
223 uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
224 int ret;
225 121
226 PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",122 TEST_ASSERT(alias != NULL, "Alias required for minor faults");
227 is_minor ? "MINOR" : "MISSING",123 for (p = 0; p < (len / demand_paging_size); ++p) {
228 is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");124 memcpy(alias + (p * demand_paging_size),
229 125 guest_data_prototype, demand_paging_size);
230 /* In order to get minor faults, prefault via the alias. */
231 if (is_minor) {
232 size_t p;
233
234 expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
235
236 TEST_ASSERT(alias != NULL, "Alias required for minor faults");
237 for (p = 0; p < (len / demand_paging_size); ++p) {
238 memcpy(alias + (p * demand_paging_size),
239 guest_data_prototype, demand_paging_size);
240 }
241 }126 }
242
243 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
244 TEST_ASSERT(uffd >= 0, __KVM_SYSCALL_ERROR("userfaultfd()", uffd));
245
246 uffdio_api.api = UFFD_API;
247 uffdio_api.features = 0;
248 ret = ioctl(uffd, UFFDIO_API, &uffdio_api);
249 TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_API", ret));
250
251 uffdio_register.range.start = (uint64_t)hva;
252 uffdio_register.range.len = len;
253 uffdio_register.mode = uffd_mode;
254 ret = ioctl(uffd, UFFDIO_REGISTER, &uffdio_register);
255 TEST_ASSERT(ret != -1, __KVM_SYSCALL_ERROR("UFFDIO_REGISTER", ret));
256 TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
257 expected_ioctls, "missing userfaultfd ioctls");
258
259 uffd_args->uffd_mode = uffd_mode;
260 uffd_args->uffd = uffd;
261 uffd_args->pipefd = pipefd;
262 uffd_args->delay = uffd_delay;
263 pthread_create(uffd_handler_thread, NULL, uffd_handler_thread_fn,
264 uffd_args);
265
266 PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
267 hva, hva + len);
268}127}
269 128
270struct test_params {
271 int uffd_mode;
272 useconds_t uffd_delay;
273 enum vm_mem_backing_src_type src_type;
274 bool partition_vcpu_memory_access;
275};
276
277static void run_test(enum vm_guest_mode mode, void *arg)129static void run_test(enum vm_guest_mode mode, void *arg)
278{130{
279 struct test_params *p = arg;131 struct test_params *p = arg;
280 pthread_t *uffd_handler_threads = NULL;132 struct uffd_desc **uffd_descs = NULL;
281 struct uffd_handler_args *uffd_args = NULL;
282 struct timespec start;133 struct timespec start;
283 struct timespec ts_diff;134 struct timespec ts_diff;
284 int *pipefds = NULL;
285 struct kvm_vm *vm;135 struct kvm_vm *vm;
286 int r, i;136 int i;
287 137
288 vm = memstress_create_vm(mode, nr_vcpus, guest_percpu_mem_size, 1,138 vm = memstress_create_vm(mode, nr_vcpus, guest_percpu_mem_size, 1,
289 p->src_type, p->partition_vcpu_memory_access);139 p->src_type, p->partition_vcpu_memory_access);
@@ -296,15 +146,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
296 memset(guest_data_prototype, 0xAB, demand_paging_size);146 memset(guest_data_prototype, 0xAB, demand_paging_size);
297 147
298 if (p->uffd_mode) {148 if (p->uffd_mode) {
299 uffd_handler_threads =149 uffd_descs = malloc(nr_vcpus * sizeof(struct uffd_desc *));
300 malloc(nr_vcpus * sizeof(*uffd_handler_threads));150 TEST_ASSERT(uffd_descs, "Memory allocation failed");
301 TEST_ASSERT(uffd_handler_threads, "Memory allocation failed");
302
303 uffd_args = malloc(nr_vcpus * sizeof(*uffd_args));
304 TEST_ASSERT(uffd_args, "Memory allocation failed");
305
306 pipefds = malloc(sizeof(int) * nr_vcpus * 2);
307 TEST_ASSERT(pipefds, "Unable to allocate memory for pipefd");
308 151
309 for (i = 0; i < nr_vcpus; i++) {152 for (i = 0; i < nr_vcpus; i++) {
310 struct memstress_vcpu_args *vcpu_args;153 struct memstress_vcpu_args *vcpu_args;
@@ -317,19 +160,17 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
317 vcpu_hva = addr_gpa2hva(vm, vcpu_args->gpa);160 vcpu_hva = addr_gpa2hva(vm, vcpu_args->gpa);
318 vcpu_alias = addr_gpa2alias(vm, vcpu_args->gpa);161 vcpu_alias = addr_gpa2alias(vm, vcpu_args->gpa);
319 162
163 prefault_mem(vcpu_alias,
164 vcpu_args->pages * memstress_args.guest_page_size);
165
320 /*166 /*
321 * Set up user fault fd to handle demand paging167 * Set up user fault fd to handle demand paging
322 * requests.168 * requests.
323 */169 */
324 r = pipe2(&pipefds[i * 2],170 uffd_descs[i] = uffd_setup_demand_paging(
325 O_CLOEXEC | O_NONBLOCK);171 p->uffd_mode, p->uffd_delay, vcpu_hva,
326 TEST_ASSERT(!r, "Failed to set up pipefd");172 vcpu_args->pages * memstress_args.guest_page_size,
327 173 &handle_uffd_page_request);
328 setup_demand_paging(vm, &uffd_handler_threads[i],
329 pipefds[i * 2], p->uffd_mode,
330 p->uffd_delay, &uffd_args[i],
331 vcpu_hva, vcpu_alias,
332 vcpu_args->pages * memstress_args.guest_page_size);
333 }174 }
334 }175 }
335 176
@@ -344,15 +185,9 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 8 unmodified lines
344 pr_info("All vCPU threads joined\n");185 pr_info("All vCPU threads joined\n");
345 186
346 if (p->uffd_mode) {187 if (p->uffd_mode) {
347 char c;
348
349 /* Tell the user fault fd handler threads to quit */188 /* Tell the user fault fd handler threads to quit */
350 for (i = 0; i < nr_vcpus; i++) {189 for (i = 0; i < nr_vcpus; i++)
351 r = write(pipefds[i * 2 + 1], &c, 1);190 uffd_stop_demand_paging(uffd_descs[i]);
352 TEST_ASSERT(r == 1, "Unable to write to pipefd");
353
354 pthread_join(uffd_handler_threads[i], NULL);
355 }
356 }191 }
357 192
358 pr_info("Total guest execution time: %ld.%.9lds\n",193 pr_info("Total guest execution time: %ld.%.9lds\n",
@@ -364,11 +199,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 5 unmodified lines
364 memstress_destroy_vm(vm);199 memstress_destroy_vm(vm);
365 200
366 free(guest_data_prototype);201 free(guest_data_prototype);
367 if (p->uffd_mode) {202 if (p->uffd_mode)
368 free(uffd_handler_threads);203 free(uffd_descs);
369 free(uffd_args);
370 free(pipefds);
371 }
372}204}
373 205
374static void help(char *name)206static void help(char *name)
tools/testing/selftests/kvm/dirty_log_test.c+39−14
@@ -24,6 +24,9 @@ · 23 unmodified lines
24#include "guest_modes.h"24#include "guest_modes.h"
25#include "processor.h"25#include "processor.h"
26 26
27#define DIRTY_MEM_BITS 30 /* 1G */
28#define PAGE_SHIFT_4K 12
29
27/* The memory slot index to track dirty pages */30/* The memory slot index to track dirty pages */
28#define TEST_MEM_SLOT_INDEX 131#define TEST_MEM_SLOT_INDEX 1
29 32
@@ -226,13 +229,15 @@ static void clear_log_create_vm_done(struct kvm_vm *vm) · 196 unmodified lines
226}229}
227 230
228static void dirty_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,231static void dirty_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
229 void *bitmap, uint32_t num_pages)232 void *bitmap, uint32_t num_pages,
233 uint32_t *unused)
230{234{
231 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);235 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
232}236}
233 237
234static void clear_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,238static void clear_log_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
235 void *bitmap, uint32_t num_pages)239 void *bitmap, uint32_t num_pages,
240 uint32_t *unused)
236{241{
237 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);242 kvm_vm_get_dirty_log(vcpu->vm, slot, bitmap);
238 kvm_vm_clear_dirty_log(vcpu->vm, slot, bitmap, 0, num_pages);243 kvm_vm_clear_dirty_log(vcpu->vm, slot, bitmap, 0, num_pages);
@@ -271,6 +276,24 @@ static bool dirty_ring_supported(void) · 32 unmodified lines
271 276
272static void dirty_ring_create_vm_done(struct kvm_vm *vm)277static void dirty_ring_create_vm_done(struct kvm_vm *vm)
273{278{
279 uint64_t pages;
280 uint32_t limit;
281
282 /*
283 * We rely on vcpu exit due to full dirty ring state. Adjust
284 * the ring buffer size to ensure we're able to reach the
285 * full dirty ring state.
286 */
287 pages = (1ul << (DIRTY_MEM_BITS - vm->page_shift)) + 3;
288 pages = vm_adjust_num_guest_pages(vm->mode, pages);
289 if (vm->page_size < getpagesize())
290 pages = vm_num_host_pages(vm->mode, pages);
291
292 limit = 1 << (31 - __builtin_clz(pages));
293 test_dirty_ring_count = 1 << (31 - __builtin_clz(test_dirty_ring_count));
294 test_dirty_ring_count = min(limit, test_dirty_ring_count);
295 pr_info("dirty ring count: 0x%x\n", test_dirty_ring_count);
296
274 /*297 /*
275 * Switch to dirty ring mode after VM creation but before any298 * Switch to dirty ring mode after VM creation but before any
276 * of the vcpu creation.299 * of the vcpu creation.
@@ -329,10 +352,9 @@ static void dirty_ring_continue_vcpu(void) · 52 unmodified lines
329}352}
330 353
331static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,354static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
332 void *bitmap, uint32_t num_pages)355 void *bitmap, uint32_t num_pages,
356 uint32_t *ring_buf_idx)
333{357{
334 /* We only have one vcpu */
335 static uint32_t fetch_index = 0;
336 uint32_t count = 0, cleared;358 uint32_t count = 0, cleared;
337 bool continued_vcpu = false;359 bool continued_vcpu = false;
338 360
@@ -349,7 +371,8 @@ static void dirty_ring_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot, · 10 unmodified lines
349 371
350 /* Only have one vcpu */372 /* Only have one vcpu */
351 count = dirty_ring_collect_one(vcpu_map_dirty_ring(vcpu),373 count = dirty_ring_collect_one(vcpu_map_dirty_ring(vcpu),
352 slot, bitmap, num_pages, &fetch_index);374 slot, bitmap, num_pages,
375 ring_buf_idx);
353 376
354 cleared = kvm_vm_reset_dirty_ring(vcpu->vm);377 cleared = kvm_vm_reset_dirty_ring(vcpu->vm);
355 378
@@ -406,7 +429,8 @@ struct log_mode { · 50 unmodified lines
406 void (*create_vm_done)(struct kvm_vm *vm);429 void (*create_vm_done)(struct kvm_vm *vm);
407 /* Hook to collect the dirty pages into the bitmap provided */430 /* Hook to collect the dirty pages into the bitmap provided */
408 void (*collect_dirty_pages) (struct kvm_vcpu *vcpu, int slot,431 void (*collect_dirty_pages) (struct kvm_vcpu *vcpu, int slot,
409 void *bitmap, uint32_t num_pages);432 void *bitmap, uint32_t num_pages,
433 uint32_t *ring_buf_idx);
410 /* Hook to call when after each vcpu run */434 /* Hook to call when after each vcpu run */
411 void (*after_vcpu_run)(struct kvm_vcpu *vcpu, int ret, int err);435 void (*after_vcpu_run)(struct kvm_vcpu *vcpu, int ret, int err);
412 void (*before_vcpu_join) (void);436 void (*before_vcpu_join) (void);
@@ -471,13 +495,14 @@ static void log_mode_create_vm_done(struct kvm_vm *vm) · 58 unmodified lines
471}495}
472 496
473static void log_mode_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,497static void log_mode_collect_dirty_pages(struct kvm_vcpu *vcpu, int slot,
474 void *bitmap, uint32_t num_pages)498 void *bitmap, uint32_t num_pages,
499 uint32_t *ring_buf_idx)
475{500{
476 struct log_mode *mode = &log_modes[host_log_mode];501 struct log_mode *mode = &log_modes[host_log_mode];
477 502
478 TEST_ASSERT(mode->collect_dirty_pages != NULL,503 TEST_ASSERT(mode->collect_dirty_pages != NULL,
479 "collect_dirty_pages() is required for any log mode!");504 "collect_dirty_pages() is required for any log mode!");
480 mode->collect_dirty_pages(vcpu, slot, bitmap, num_pages);505 mode->collect_dirty_pages(vcpu, slot, bitmap, num_pages, ring_buf_idx);
481}506}
482 507
483static void log_mode_after_vcpu_run(struct kvm_vcpu *vcpu, int ret, int err)508static void log_mode_after_vcpu_run(struct kvm_vcpu *vcpu, int ret, int err)
@@ -681,9 +706,6 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu, · 197 unmodified lines
681 return vm;706 return vm;
682}707}
683 708
684#define DIRTY_MEM_BITS 30 /* 1G */
685#define PAGE_SHIFT_4K 12
686
687struct test_params {709struct test_params {
688 unsigned long iterations;710 unsigned long iterations;
689 unsigned long interval;711 unsigned long interval;
@@ -696,6 +718,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 6 unmodified lines
696 struct kvm_vcpu *vcpu;718 struct kvm_vcpu *vcpu;
697 struct kvm_vm *vm;719 struct kvm_vm *vm;
698 unsigned long *bmap;720 unsigned long *bmap;
721 uint32_t ring_buf_idx = 0;
699 722
700 if (!log_mode_supported()) {723 if (!log_mode_supported()) {
701 print_skip("Log mode '%s' not supported",724 print_skip("Log mode '%s' not supported",
@@ -769,6 +792,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 67 unmodified lines
769 host_dirty_count = 0;792 host_dirty_count = 0;
770 host_clear_count = 0;793 host_clear_count = 0;
771 host_track_next_count = 0;794 host_track_next_count = 0;
795 WRITE_ONCE(dirty_ring_vcpu_ring_full, false);
772 796
773 pthread_create(&vcpu_thread, NULL, vcpu_worker, vcpu);797 pthread_create(&vcpu_thread, NULL, vcpu_worker, vcpu);
774 798
@@ -776,7 +800,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 1 unmodified line
776 /* Give the vcpu thread some time to dirty some pages */800 /* Give the vcpu thread some time to dirty some pages */
777 usleep(p->interval * 1000);801 usleep(p->interval * 1000);
778 log_mode_collect_dirty_pages(vcpu, TEST_MEM_SLOT_INDEX,802 log_mode_collect_dirty_pages(vcpu, TEST_MEM_SLOT_INDEX,
779 bmap, host_num_pages);803 bmap, host_num_pages,
804 &ring_buf_idx);
780 805
781 /*806 /*
782 * See vcpu_sync_stop_requested definition for details on why807 * See vcpu_sync_stop_requested definition for details on why
@@ -820,7 +845,7 @@ static void help(char *name) · 37 unmodified lines
820 printf("usage: %s [-h] [-i iterations] [-I interval] "845 printf("usage: %s [-h] [-i iterations] [-I interval] "
821 "[-p offset] [-m mode]\n", name);846 "[-p offset] [-m mode]\n", name);
822 puts("");847 puts("");
823 printf(" -c: specify dirty ring size, in number of entries\n");848 printf(" -c: hint to dirty ring size, in number of entries\n");
824 printf(" (only useful for dirty-ring test; default: %"PRIu32")\n",849 printf(" (only useful for dirty-ring test; default: %"PRIu32")\n",
825 TEST_DIRTY_RING_COUNT);850 TEST_DIRTY_RING_COUNT);
826 printf(" -i: specify iteration counts (default: %"PRIu64")\n",851 printf(" -i: specify iteration counts (default: %"PRIu64")\n",
tools/testing/selftests/kvm/include/aarch64/processor.h+29−6
@@ -38,12 +38,25 @@ · 37 unmodified lines
38 * NORMAL 4 1111:111138 * NORMAL 4 1111:1111
39 * NORMAL_WT 5 1011:101139 * NORMAL_WT 5 1011:1011
40 */40 */
41#define DEFAULT_MAIR_EL1 ((0x00ul << (0 * 8)) | \41
42 (0x04ul << (1 * 8)) | \42/* Linux doesn't use these memory types, so let's define them. */
43 (0x0cul << (2 * 8)) | \43#define MAIR_ATTR_DEVICE_GRE UL(0x0c)
44 (0x44ul << (3 * 8)) | \44#define MAIR_ATTR_NORMAL_WT UL(0xbb)
45 (0xfful << (4 * 8)) | \45
46 (0xbbul << (5 * 8)))46#define MT_DEVICE_nGnRnE 0
47#define MT_DEVICE_nGnRE 1
48#define MT_DEVICE_GRE 2
49#define MT_NORMAL_NC 3
50#define MT_NORMAL 4
51#define MT_NORMAL_WT 5
52
53#define DEFAULT_MAIR_EL1 \
54 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
55 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \
56 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \
57 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \
58 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
59 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT))
47 60
48#define MPIDR_HWID_BITMASK (0xff00fffffful)61#define MPIDR_HWID_BITMASK (0xff00fffffful)
49 62
@@ -92,11 +105,19 @@ enum { · 42 unmodified lines
92#define ESR_EC_MASK (ESR_EC_NUM - 1)105#define ESR_EC_MASK (ESR_EC_NUM - 1)
93 106
94#define ESR_EC_SVC64 0x15107#define ESR_EC_SVC64 0x15
108#define ESR_EC_IABT 0x21
109#define ESR_EC_DABT 0x25
95#define ESR_EC_HW_BP_CURRENT 0x31110#define ESR_EC_HW_BP_CURRENT 0x31
96#define ESR_EC_SSTEP_CURRENT 0x33111#define ESR_EC_SSTEP_CURRENT 0x33
97#define ESR_EC_WP_CURRENT 0x35112#define ESR_EC_WP_CURRENT 0x35
98#define ESR_EC_BRK_INS 0x3c113#define ESR_EC_BRK_INS 0x3c
99 114
115/* Access flag */
116#define PTE_AF (1ULL << 10)
117
118/* Access flag update enable/disable */
119#define TCR_EL1_HA (1ULL << 39)
120
100void aarch64_get_supported_page_sizes(uint32_t ipa,121void aarch64_get_supported_page_sizes(uint32_t ipa,
101 bool *ps4k, bool *ps16k, bool *ps64k);122 bool *ps4k, bool *ps16k, bool *ps64k);
102 123
@@ -109,6 +130,8 @@ void vm_install_exception_handler(struct kvm_vm *vm, · 6 unmodified lines
109void vm_install_sync_handler(struct kvm_vm *vm,130void vm_install_sync_handler(struct kvm_vm *vm,
110 int vector, int ec, handler_fn handler);131 int vector, int ec, handler_fn handler);
111 132
133uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva);
134
112static inline void cpu_relax(void)135static inline void cpu_relax(void)
113{136{
114 asm volatile("yield" ::: "memory");137 asm volatile("yield" ::: "memory");
tools/testing/selftests/kvm/include/kvm_util_base.h+29−2
@@ -35,6 +35,7 @@ struct userspace_mem_region { · 34 unmodified lines
35 struct sparsebit *unused_phy_pages;35 struct sparsebit *unused_phy_pages;
36 int fd;36 int fd;
37 off_t offset;37 off_t offset;
38 enum vm_mem_backing_src_type backing_src_type;
38 void *host_mem;39 void *host_mem;
39 void *host_alias;40 void *host_alias;
40 void *mmap_start;41 void *mmap_start;
@@ -65,6 +66,14 @@ struct userspace_mem_regions { · 24 unmodified lines
65 DECLARE_HASHTABLE(slot_hash, 9);66 DECLARE_HASHTABLE(slot_hash, 9);
66};67};
67 68
69enum kvm_mem_region_type {
70 MEM_REGION_CODE,
71 MEM_REGION_DATA,
72 MEM_REGION_PT,
73 MEM_REGION_TEST_DATA,
74 NR_MEM_REGIONS,
75};
76
68struct kvm_vm {77struct kvm_vm {
69 int mode;78 int mode;
70 unsigned long type;79 unsigned long type;
@@ -94,6 +103,13 @@ struct kvm_vm { · 23 unmodified lines
94 int stats_fd;103 int stats_fd;
95 struct kvm_stats_header stats_header;104 struct kvm_stats_header stats_header;
96 struct kvm_stats_desc *stats_desc;105 struct kvm_stats_desc *stats_desc;
106
107 /*
108 * KVM region slots. These are the default memslots used by page
109 * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
110 * memslot.
111 */
112 uint32_t memslots[NR_MEM_REGIONS];
97};113};
98 114
99 115
@@ -106,6 +122,13 @@ struct kvm_vm { · 6 unmodified lines
106struct userspace_mem_region *122struct userspace_mem_region *
107memslot2region(struct kvm_vm *vm, uint32_t memslot);123memslot2region(struct kvm_vm *vm, uint32_t memslot);
108 124
125static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
126 enum kvm_mem_region_type type)
127{
128 assert(type < NR_MEM_REGIONS);
129 return memslot2region(vm, vm->memslots[type]);
130}
131
109/* Minimum allocated guest virtual and physical addresses */132/* Minimum allocated guest virtual and physical addresses */
110#define KVM_UTIL_MIN_VADDR 0x2000133#define KVM_UTIL_MIN_VADDR 0x2000
111#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000134#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000
@@ -387,7 +410,11 @@ void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); · 275 unmodified lines
387struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);410struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
388vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);411vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
389vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);412vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
413vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
414 enum kvm_mem_region_type type);
390vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);415vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
416vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
417 enum kvm_mem_region_type type);
391vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);418vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
392 419
393void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,420void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
@@ -649,13 +676,13 @@ vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); · 255 unmodified lines
649 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to676 * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
650 * calculate the amount of memory needed for per-vCPU data, e.g. stacks.677 * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
651 */678 */
652struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages);679struct kvm_vm *____vm_create(enum vm_guest_mode mode);
653struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,680struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
654 uint64_t nr_extra_pages);681 uint64_t nr_extra_pages);
655 682
656static inline struct kvm_vm *vm_create_barebones(void)683static inline struct kvm_vm *vm_create_barebones(void)
657{684{
658 return ____vm_create(VM_MODE_DEFAULT, 0);685 return ____vm_create(VM_MODE_DEFAULT);
659}686}
660 687
661static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)688static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
tools/testing/selftests/kvm/include/memstress.h+3−0
@@ -47,6 +47,9 @@ struct memstress_args { · 46 unmodified lines
47 /* The vCPU=>pCPU pinning map. Only valid if pin_vcpus is true. */47 /* The vCPU=>pCPU pinning map. Only valid if pin_vcpus is true. */
48 uint32_t vcpu_to_pcpu[KVM_MAX_VCPUS];48 uint32_t vcpu_to_pcpu[KVM_MAX_VCPUS];
49 49
50 /* Test is done, stop running vCPUs. */
51 bool stop_vcpus;
52
50 struct memstress_vcpu_args vcpu_args[KVM_MAX_VCPUS];53 struct memstress_vcpu_args vcpu_args[KVM_MAX_VCPUS];
51};54};
52 55
tools/testing/selftests/kvm/include/userfaultfd_util.h+45−0
@@ -0,0 +1,45 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * KVM userfaultfd util
4 *
5 * Copyright (C) 2018, Red Hat, Inc.
6 * Copyright (C) 2019-2022 Google LLC
7 */
8
9#define _GNU_SOURCE /* for pipe2 */
10
11#include <inttypes.h>
12#include <time.h>
13#include <pthread.h>
14#include <linux/userfaultfd.h>
15
16#include "test_util.h"
17
18typedef int (*uffd_handler_t)(int uffd_mode, int uffd, struct uffd_msg *msg);
19
20struct uffd_desc {
21 int uffd_mode;
22 int uffd;
23 int pipefds[2];
24 useconds_t delay;
25 uffd_handler_t handler;
26 pthread_t thread;
27};
28
29struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
30 void *hva, uint64_t len,
31 uffd_handler_t handler);
32
33void uffd_stop_demand_paging(struct uffd_desc *uffd);
34
35#ifdef PRINT_PER_PAGE_UPDATES
36#define PER_PAGE_DEBUG(...) printf(__VA_ARGS__)
37#else
38#define PER_PAGE_DEBUG(...) _no_printf(__VA_ARGS__)
39#endif
40
41#ifdef PRINT_PER_VCPU_UPDATES
42#define PER_VCPU_DEBUG(...) printf(__VA_ARGS__)
43#else
44#define PER_VCPU_DEBUG(...) _no_printf(__VA_ARGS__)
45#endif
tools/testing/selftests/kvm/lib/aarch64/processor.c+34−21
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include "guest_modes.h"11#include "guest_modes.h"
12#include "kvm_util.h"12#include "kvm_util.h"
13#include "processor.h"13#include "processor.h"
14#include <linux/bitfield.h>
14 15
15#define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac000016#define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000
16 17
@@ -76,13 +77,15 @@ static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm) · 59 unmodified lines
76 77
77void virt_arch_pgd_alloc(struct kvm_vm *vm)78void virt_arch_pgd_alloc(struct kvm_vm *vm)
78{79{
79 if (!vm->pgd_created) {80 size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size;
80 vm_paddr_t paddr = vm_phy_pages_alloc(vm,81
81 page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size,82 if (vm->pgd_created)
82 KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);83 return;
83 vm->pgd = paddr;84
84 vm->pgd_created = true;85 vm->pgd = vm_phy_pages_alloc(vm, nr_pages,
85 }86 KVM_GUEST_PAGE_TABLE_MIN_PADDR,
87 vm->memslots[MEM_REGION_PT]);
88 vm->pgd_created = true;
86}89}
87 90
88static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,91static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
@@ -133,12 +136,12 @@ static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, · 44 unmodified lines
133 136
134void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)137void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
135{138{
136 uint64_t attr_idx = 4; /* NORMAL (See DEFAULT_MAIR_EL1) */139 uint64_t attr_idx = MT_NORMAL;
137 140
138 _virt_pg_map(vm, vaddr, paddr, attr_idx);141 _virt_pg_map(vm, vaddr, paddr, attr_idx);
139}142}
140 143
141vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)144uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva)
142{145{
143 uint64_t *ptep;146 uint64_t *ptep;
144 147
@@ -169,11 +172,18 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) · 24 unmodified lines
169 TEST_FAIL("Page table levels must be 2, 3, or 4");172 TEST_FAIL("Page table levels must be 2, 3, or 4");
170 }173 }
171 174
172 return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));175 return ptep;
173 176
174unmapped_gva:177unmapped_gva:
175 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);178 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
176 exit(1);179 exit(EXIT_FAILURE);
180}
181
182vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
183{
184 uint64_t *ptep = virt_get_pte_hva(vm, gva);
185
186 return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
177}187}
178 188
179static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)189static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
@@ -318,13 +328,16 @@ void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) · 138 unmodified lines
318struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,328struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
319 struct kvm_vcpu_init *init, void *guest_code)329 struct kvm_vcpu_init *init, void *guest_code)
320{330{
321 size_t stack_size = vm->page_size == 4096 ?331 size_t stack_size;
322 DEFAULT_STACK_PGS * vm->page_size :332 uint64_t stack_vaddr;
323 vm->page_size;
324 uint64_t stack_vaddr = vm_vaddr_alloc(vm, stack_size,
325 DEFAULT_ARM64_GUEST_STACK_VADDR_MIN);
326 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);333 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
327 334
335 stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
336 vm->page_size;
337 stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
338 DEFAULT_ARM64_GUEST_STACK_VADDR_MIN,
339 MEM_REGION_DATA);
340
328 aarch64_vcpu_setup(vcpu, init);341 aarch64_vcpu_setup(vcpu, init);
329 342
330 vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);343 vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
@@ -428,8 +441,8 @@ unexpected_exception: · 97 unmodified lines
428 441
429void vm_init_descriptor_tables(struct kvm_vm *vm)442void vm_init_descriptor_tables(struct kvm_vm *vm)
430{443{
431 vm->handlers = vm_vaddr_alloc(vm, sizeof(struct handlers),444 vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
432 vm->page_size);445 vm->page_size, MEM_REGION_DATA);
433 446
434 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;447 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
435}448}
@@ -486,9 +499,9 @@ void aarch64_get_supported_page_sizes(uint32_t ipa, · 50 unmodified lines
486 err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);499 err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);
487 TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd));500 TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd));
488 501
489 *ps4k = ((val >> 28) & 0xf) != 0xf;502 *ps4k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN4), val) != 0xf;
490 *ps64k = ((val >> 24) & 0xf) == 0;503 *ps64k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN64), val) == 0;
491 *ps16k = ((val >> 20) & 0xf) != 0;504 *ps16k = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_TGRAN16), val) != 0;
492 505
493 close(vcpu_fd);506 close(vcpu_fd);
494 close(vm_fd);507 close(vm_fd);
tools/testing/selftests/kvm/lib/elf.c+2−1
@@ -161,7 +161,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) · 160 unmodified lines
161 seg_vend |= vm->page_size - 1;161 seg_vend |= vm->page_size - 1;
162 size_t seg_size = seg_vend - seg_vstart + 1;162 size_t seg_size = seg_vend - seg_vstart + 1;
163 163
164 vm_vaddr_t vaddr = vm_vaddr_alloc(vm, seg_size, seg_vstart);164 vm_vaddr_t vaddr = __vm_vaddr_alloc(vm, seg_size, seg_vstart,
165 MEM_REGION_CODE);
165 TEST_ASSERT(vaddr == seg_vstart, "Unable to allocate "166 TEST_ASSERT(vaddr == seg_vstart, "Unable to allocate "
166 "virtual memory for segment at requested min addr,\n"167 "virtual memory for segment at requested min addr,\n"
167 " segment idx: %u\n"168 " segment idx: %u\n"
tools/testing/selftests/kvm/lib/kvm_util.c+53−31
@@ -186,13 +186,10 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { · 185 unmodified lines
186_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,186_Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
187 "Missing new mode params?");187 "Missing new mode params?");
188 188
189struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages)189struct kvm_vm *____vm_create(enum vm_guest_mode mode)
190{190{
191 struct kvm_vm *vm;191 struct kvm_vm *vm;
192 192
193 pr_debug("%s: mode='%s' pages='%ld'\n", __func__,
194 vm_guest_mode_string(mode), nr_pages);
195
196 vm = calloc(1, sizeof(*vm));193 vm = calloc(1, sizeof(*vm));
197 TEST_ASSERT(vm != NULL, "Insufficient Memory");194 TEST_ASSERT(vm != NULL, "Insufficient Memory");
198 195
@@ -288,9 +285,6 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode, uint64_t nr_pages) · 89 unmodified lines
288 285
289 /* Allocate and setup memory for guest. */286 /* Allocate and setup memory for guest. */
290 vm->vpages_mapped = sparsebit_alloc();287 vm->vpages_mapped = sparsebit_alloc();
291 if (nr_pages != 0)
292 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
293 0, 0, nr_pages, 0);
294 288
295 return vm;289 return vm;
296}290}
@@ -337,8 +331,16 @@ struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus, · 40 unmodified lines
337 nr_extra_pages);331 nr_extra_pages);
338 struct userspace_mem_region *slot0;332 struct userspace_mem_region *slot0;
339 struct kvm_vm *vm;333 struct kvm_vm *vm;
334 int i;
335
336 pr_debug("%s: mode='%s' pages='%ld'\n", __func__,
337 vm_guest_mode_string(mode), nr_pages);
340 338
341 vm = ____vm_create(mode, nr_pages);339 vm = ____vm_create(mode);
340
341 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);
342 for (i = 0; i < NR_MEM_REGIONS; i++)
343 vm->memslots[i] = 0;
342 344
343 kvm_vm_elf_load(vm, program_invocation_name);345 kvm_vm_elf_load(vm, program_invocation_name);
344 346
@@ -649,6 +651,12 @@ static void __vm_mem_region_delete(struct kvm_vm *vm, · 304 unmodified lines
649 sparsebit_free(&region->unused_phy_pages);651 sparsebit_free(&region->unused_phy_pages);
650 ret = munmap(region->mmap_start, region->mmap_size);652 ret = munmap(region->mmap_start, region->mmap_size);
651 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));653 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
654 if (region->fd >= 0) {
655 /* There's an extra map when using shared memory. */
656 ret = munmap(region->mmap_alias, region->mmap_size);
657 TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
658 close(region->fd);
659 }
652 660
653 free(region);661 free(region);
654}662}
@@ -986,6 +994,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, · 331 unmodified lines
986 vm_mem_backing_src_alias(src_type)->name);994 vm_mem_backing_src_alias(src_type)->name);
987 }995 }
988 996
997 region->backing_src_type = src_type;
989 region->unused_phy_pages = sparsebit_alloc();998 region->unused_phy_pages = sparsebit_alloc();
990 sparsebit_set_num(region->unused_phy_pages,999 sparsebit_set_num(region->unused_phy_pages,
991 guest_paddr >> vm->page_shift, npages);1000 guest_paddr >> vm->page_shift, npages);
@@ -1280,32 +1289,15 @@ va_found: · 288 unmodified lines
1280 return pgidx_start * vm->page_size;1289 return pgidx_start * vm->page_size;
1281}1290}
1282 1291
1283/*1292vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1284 * VM Virtual Address Allocate1293 enum kvm_mem_region_type type)
1285 *
1286 * Input Args:
1287 * vm - Virtual Machine
1288 * sz - Size in bytes
1289 * vaddr_min - Minimum starting virtual address
1290 *
1291 * Output Args: None
1292 *
1293 * Return:
1294 * Starting guest virtual address
1295 *
1296 * Allocates at least sz bytes within the virtual address space of the vm
1297 * given by vm. The allocated bytes are mapped to a virtual address >=
1298 * the address given by vaddr_min. Note that each allocation uses a
1299 * a unique set of pages, with the minimum real allocation being at least
1300 * a page.
1301 */
1302vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1303{1294{
1304 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);1295 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1305 1296
1306 virt_pgd_alloc(vm);1297 virt_pgd_alloc(vm);
1307 vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,1298 vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,
1308 KVM_UTIL_MIN_PFN * vm->page_size, 0);1299 KVM_UTIL_MIN_PFN * vm->page_size,
1300 vm->memslots[type]);
1309 1301
1310 /*1302 /*
1311 * Find an unused range of virtual page addresses of at least1303 * Find an unused range of virtual page addresses of at least
@@ -1325,6 +1317,30 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min) · 13 unmodified lines
1325 return vaddr_start;1317 return vaddr_start;
1326}1318}
1327 1319
1320/*
1321 * VM Virtual Address Allocate
1322 *
1323 * Input Args:
1324 * vm - Virtual Machine
1325 * sz - Size in bytes
1326 * vaddr_min - Minimum starting virtual address
1327 *
1328 * Output Args: None
1329 *
1330 * Return:
1331 * Starting guest virtual address
1332 *
1333 * Allocates at least sz bytes within the virtual address space of the vm
1334 * given by vm. The allocated bytes are mapped to a virtual address >=
1335 * the address given by vaddr_min. Note that each allocation uses a
1336 * a unique set of pages, with the minimum real allocation being at least
1337 * a page. The allocated physical space comes from the TEST_DATA memory region.
1338 */
1339vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min)
1340{
1341 return __vm_vaddr_alloc(vm, sz, vaddr_min, MEM_REGION_TEST_DATA);
1342}
1343
1328/*1344/*
1329 * VM Virtual Address Allocate Pages1345 * VM Virtual Address Allocate Pages
1330 *1346 *
@@ -1344,6 +1360,11 @@ vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages) · 13 unmodified lines
1344 return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);1360 return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR);
1345}1361}
1346 1362
1363vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm, enum kvm_mem_region_type type)
1364{
1365 return __vm_vaddr_alloc(vm, getpagesize(), KVM_UTIL_MIN_VADDR, type);
1366}
1367
1347/*1368/*
1348 * VM Virtual Address Allocate Page1369 * VM Virtual Address Allocate Page
1349 *1370 *
@@ -1570,7 +1591,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu) · 220 unmodified lines
1570 1591
1571void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)1592void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu)
1572{1593{
1573 uint32_t page_size = vcpu->vm->page_size;1594 uint32_t page_size = getpagesize();
1574 uint32_t size = vcpu->vm->dirty_ring_size;1595 uint32_t size = vcpu->vm->dirty_ring_size;
1575 1596
1576 TEST_ASSERT(size > 0, "Should enable dirty ring first");1597 TEST_ASSERT(size > 0, "Should enable dirty ring first");
@@ -1911,7 +1932,8 @@ vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, · 334 unmodified lines
1911 1932
1912vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)1933vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm)
1913{1934{
1914 return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);1935 return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR,
1936 vm->memslots[MEM_REGION_PT]);
1915}1937}
1916 1938
1917/*1939/*
tools/testing/selftests/kvm/lib/memstress.c+3−0
@@ -292,6 +292,7 @@ void memstress_start_vcpu_threads(int nr_vcpus, · 291 unmodified lines
292 292
293 vcpu_thread_fn = vcpu_fn;293 vcpu_thread_fn = vcpu_fn;
294 WRITE_ONCE(all_vcpu_threads_running, false);294 WRITE_ONCE(all_vcpu_threads_running, false);
295 WRITE_ONCE(memstress_args.stop_vcpus, false);
295 296
296 for (i = 0; i < nr_vcpus; i++) {297 for (i = 0; i < nr_vcpus; i++) {
297 struct vcpu_thread *vcpu = &vcpu_threads[i];298 struct vcpu_thread *vcpu = &vcpu_threads[i];
@@ -314,6 +315,8 @@ void memstress_join_vcpu_threads(int nr_vcpus) · 16 unmodified lines
314{315{
315 int i;316 int i;
316 317
318 WRITE_ONCE(memstress_args.stop_vcpus, true);
319
317 for (i = 0; i < nr_vcpus; i++)320 for (i = 0; i < nr_vcpus; i++)
318 pthread_join(vcpu_threads[i].thread, NULL);321 pthread_join(vcpu_threads[i].thread, NULL);
319}322}
tools/testing/selftests/kvm/lib/riscv/processor.c+17−12
@@ -55,13 +55,15 @@ static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level) · 54 unmodified lines
55 55
56void virt_arch_pgd_alloc(struct kvm_vm *vm)56void virt_arch_pgd_alloc(struct kvm_vm *vm)
57{57{
58 if (!vm->pgd_created) {58 size_t nr_pages = page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size;
59 vm_paddr_t paddr = vm_phy_pages_alloc(vm,59
60 page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size,60 if (vm->pgd_created)
61 KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);61 return;
62 vm->pgd = paddr;62
63 vm->pgd_created = true;63 vm->pgd = vm_phy_pages_alloc(vm, nr_pages,
64 }64 KVM_GUEST_PAGE_TABLE_MIN_PADDR,
65 vm->memslots[MEM_REGION_PT]);
66 vm->pgd_created = true;
65}67}
66 68
67void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)69void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
@@ -279,15 +281,18 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 211 unmodified lines
279 void *guest_code)281 void *guest_code)
280{282{
281 int r;283 int r;
282 size_t stack_size = vm->page_size == 4096 ?284 size_t stack_size;
283 DEFAULT_STACK_PGS * vm->page_size :285 unsigned long stack_vaddr;
284 vm->page_size;
285 unsigned long stack_vaddr = vm_vaddr_alloc(vm, stack_size,
286 DEFAULT_RISCV_GUEST_STACK_VADDR_MIN);
287 unsigned long current_gp = 0;286 unsigned long current_gp = 0;
288 struct kvm_mp_state mps;287 struct kvm_mp_state mps;
289 struct kvm_vcpu *vcpu;288 struct kvm_vcpu *vcpu;
290 289
290 stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
291 vm->page_size;
292 stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
293 DEFAULT_RISCV_GUEST_STACK_VADDR_MIN,
294 MEM_REGION_DATA);
295
291 vcpu = __vm_vcpu_add(vm, vcpu_id);296 vcpu = __vm_vcpu_add(vm, vcpu_id);
292 riscv_vcpu_mmu_setup(vcpu);297 riscv_vcpu_mmu_setup(vcpu);
293 298
tools/testing/selftests/kvm/lib/s390x/processor.c+5−3
@@ -21,7 +21,8 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) · 20 unmodified lines
21 return;21 return;
22 22
23 paddr = vm_phy_pages_alloc(vm, PAGES_PER_REGION,23 paddr = vm_phy_pages_alloc(vm, PAGES_PER_REGION,
24 KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);24 KVM_GUEST_PAGE_TABLE_MIN_PADDR,
25 vm->memslots[MEM_REGION_PT]);
25 memset(addr_gpa2hva(vm, paddr), 0xff, PAGES_PER_REGION * vm->page_size);26 memset(addr_gpa2hva(vm, paddr), 0xff, PAGES_PER_REGION * vm->page_size);
26 27
27 vm->pgd = paddr;28 vm->pgd = paddr;
@@ -167,8 +168,9 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 139 unmodified lines
167 TEST_ASSERT(vm->page_size == 4096, "Unsupported page size: 0x%x",168 TEST_ASSERT(vm->page_size == 4096, "Unsupported page size: 0x%x",
168 vm->page_size);169 vm->page_size);
169 170
170 stack_vaddr = vm_vaddr_alloc(vm, stack_size,171 stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
171 DEFAULT_GUEST_STACK_VADDR_MIN);172 DEFAULT_GUEST_STACK_VADDR_MIN,
173 MEM_REGION_DATA);
172 174
173 vcpu = __vm_vcpu_add(vm, vcpu_id);175 vcpu = __vm_vcpu_add(vm, vcpu_id);
174 176
tools/testing/selftests/kvm/lib/userfaultfd_util.c+186−0
@@ -0,0 +1,186 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * KVM userfaultfd util
4 * Adapted from demand_paging_test.c
5 *
6 * Copyright (C) 2018, Red Hat, Inc.
7 * Copyright (C) 2019-2022 Google LLC
8 */
9
10#define _GNU_SOURCE /* for pipe2 */
11
12#include <inttypes.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <time.h>
16#include <poll.h>
17#include <pthread.h>
18#include <linux/userfaultfd.h>
19#include <sys/syscall.h>
20
21#include "kvm_util.h"
22#include "test_util.h"
23#include "memstress.h"
24#include "userfaultfd_util.h"
25
26#ifdef __NR_userfaultfd
27
28static void *uffd_handler_thread_fn(void *arg)
29{
30 struct uffd_desc *uffd_desc = (struct uffd_desc *)arg;
31 int uffd = uffd_desc->uffd;
32 int pipefd = uffd_desc->pipefds[0];
33 useconds_t delay = uffd_desc->delay;
34 int64_t pages = 0;
35 struct timespec start;
36 struct timespec ts_diff;
37
38 clock_gettime(CLOCK_MONOTONIC, &start);
39 while (1) {
40 struct uffd_msg msg;
41 struct pollfd pollfd[2];
42 char tmp_chr;
43 int r;
44
45 pollfd[0].fd = uffd;
46 pollfd[0].events = POLLIN;
47 pollfd[1].fd = pipefd;
48 pollfd[1].events = POLLIN;
49
50 r = poll(pollfd, 2, -1);
51 switch (r) {
52 case -1:
53 pr_info("poll err");
54 continue;
55 case 0:
56 continue;
57 case 1:
58 break;
59 default:
60 pr_info("Polling uffd returned %d", r);
61 return NULL;
62 }
63
64 if (pollfd[0].revents & POLLERR) {
65 pr_info("uffd revents has POLLERR");
66 return NULL;
67 }
68
69 if (pollfd[1].revents & POLLIN) {
70 r = read(pollfd[1].fd, &tmp_chr, 1);
71 TEST_ASSERT(r == 1,
72 "Error reading pipefd in UFFD thread\n");
73 return NULL;
74 }
75
76 if (!(pollfd[0].revents & POLLIN))
77 continue;
78
79 r = read(uffd, &msg, sizeof(msg));
80 if (r == -1) {
81 if (errno == EAGAIN)
82 continue;
83 pr_info("Read of uffd got errno %d\n", errno);
84 return NULL;
85 }
86
87 if (r != sizeof(msg)) {
88 pr_info("Read on uffd returned unexpected size: %d bytes", r);
89 return NULL;
90 }
91
92 if (!(msg.event & UFFD_EVENT_PAGEFAULT))
93 continue;
94
95 if (delay)
96 usleep(delay);
97 r = uffd_desc->handler(uffd_desc->uffd_mode, uffd, &msg);
98 if (r < 0)
99 return NULL;
100 pages++;
101 }
102
103 ts_diff = timespec_elapsed(start);
104 PER_VCPU_DEBUG("userfaulted %ld pages over %ld.%.9lds. (%f/sec)\n",
105 pages, ts_diff.tv_sec, ts_diff.tv_nsec,
106 pages / ((double)ts_diff.tv_sec + (double)ts_diff.tv_nsec / 100000000.0));
107
108 return NULL;
109}
110
111struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay,
112 void *hva, uint64_t len,
113 uffd_handler_t handler)
114{
115 struct uffd_desc *uffd_desc;
116 bool is_minor = (uffd_mode == UFFDIO_REGISTER_MODE_MINOR);
117 int uffd;
118 struct uffdio_api uffdio_api;
119 struct uffdio_register uffdio_register;
120 uint64_t expected_ioctls = ((uint64_t) 1) << _UFFDIO_COPY;
121 int ret;
122
123 PER_PAGE_DEBUG("Userfaultfd %s mode, faults resolved with %s\n",
124 is_minor ? "MINOR" : "MISSING",
125 is_minor ? "UFFDIO_CONINUE" : "UFFDIO_COPY");
126
127 uffd_desc = malloc(sizeof(struct uffd_desc));
128 TEST_ASSERT(uffd_desc, "malloc failed");
129
130 /* In order to get minor faults, prefault via the alias. */
131 if (is_minor)
132 expected_ioctls = ((uint64_t) 1) << _UFFDIO_CONTINUE;
133
134 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
135 TEST_ASSERT(uffd >= 0, "uffd creation failed, errno: %d", errno);
136
137 uffdio_api.api = UFFD_API;
138 uffdio_api.features = 0;
139 TEST_ASSERT(ioctl(uffd, UFFDIO_API, &uffdio_api) != -1,
140 "ioctl UFFDIO_API failed: %" PRIu64,
141 (uint64_t)uffdio_api.api);
142
143 uffdio_register.range.start = (uint64_t)hva;
144 uffdio_register.range.len = len;
145 uffdio_register.mode = uffd_mode;
146 TEST_ASSERT(ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) != -1,
147 "ioctl UFFDIO_REGISTER failed");
148 TEST_ASSERT((uffdio_register.ioctls & expected_ioctls) ==
149 expected_ioctls, "missing userfaultfd ioctls");
150
151 ret = pipe2(uffd_desc->pipefds, O_CLOEXEC | O_NONBLOCK);
152 TEST_ASSERT(!ret, "Failed to set up pipefd");
153
154 uffd_desc->uffd_mode = uffd_mode;
155 uffd_desc->uffd = uffd;
156 uffd_desc->delay = delay;
157 uffd_desc->handler = handler;
158 pthread_create(&uffd_desc->thread, NULL, uffd_handler_thread_fn,
159 uffd_desc);
160
161 PER_VCPU_DEBUG("Created uffd thread for HVA range [%p, %p)\n",
162 hva, hva + len);
163
164 return uffd_desc;
165}
166
167void uffd_stop_demand_paging(struct uffd_desc *uffd)
168{
169 char c = 0;
170 int ret;
171
172 ret = write(uffd->pipefds[1], &c, 1);
173 TEST_ASSERT(ret == 1, "Unable to write to pipefd");
174
175 ret = pthread_join(uffd->thread, NULL);
176 TEST_ASSERT(ret == 0, "Pthread_join failed.");
177
178 close(uffd->uffd);
179
180 close(uffd->pipefds[1]);
181 close(uffd->pipefds[0]);
182
183 free(uffd);
184}
185
186#endif /* __NR_userfaultfd */
tools/testing/selftests/kvm/lib/x86_64/processor.c+7−6
@@ -499,7 +499,7 @@ vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) · 498 unmodified lines
499static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)499static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
500{500{
501 if (!vm->gdt)501 if (!vm->gdt)
502 vm->gdt = vm_vaddr_alloc_page(vm);502 vm->gdt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
503 503
504 dt->base = vm->gdt;504 dt->base = vm->gdt;
505 dt->limit = getpagesize();505 dt->limit = getpagesize();
@@ -509,7 +509,7 @@ static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp, · 3 unmodified lines
509 int selector)509 int selector)
510{510{
511 if (!vm->tss)511 if (!vm->tss)
512 vm->tss = vm_vaddr_alloc_page(vm);512 vm->tss = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
513 513
514 memset(segp, 0, sizeof(*segp));514 memset(segp, 0, sizeof(*segp));
515 segp->base = vm->tss;515 segp->base = vm->tss;
@@ -599,8 +599,9 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, · 83 unmodified lines
599 vm_vaddr_t stack_vaddr;599 vm_vaddr_t stack_vaddr;
600 struct kvm_vcpu *vcpu;600 struct kvm_vcpu *vcpu;
601 601
602 stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),602 stack_vaddr = __vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
603 DEFAULT_GUEST_STACK_VADDR_MIN);603 DEFAULT_GUEST_STACK_VADDR_MIN,
604 MEM_REGION_DATA);
604 605
605 vcpu = __vm_vcpu_add(vm, vcpu_id);606 vcpu = __vm_vcpu_add(vm, vcpu_id);
606 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());607 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
@@ -1093,8 +1094,8 @@ void vm_init_descriptor_tables(struct kvm_vm *vm) · 486 unmodified lines
1093 extern void *idt_handlers;1094 extern void *idt_handlers;
1094 int i;1095 int i;
1095 1096
1096 vm->idt = vm_vaddr_alloc_page(vm);1097 vm->idt = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1097 vm->handlers = vm_vaddr_alloc_page(vm);1098 vm->handlers = __vm_vaddr_alloc_page(vm, MEM_REGION_DATA);
1098 /* Handlers have the same address in both address spaces.*/1099 /* Handlers have the same address in both address spaces.*/
1099 for (i = 0; i < NUM_INTERRUPTS; i++)1100 for (i = 0; i < NUM_INTERRUPTS; i++)
1100 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,1101 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
tools/testing/selftests/kvm/memslot_modification_stress_test.c+1−5
@@ -34,8 +34,6 @@ · 33 unmodified lines
34static int nr_vcpus = 1;34static int nr_vcpus = 1;
35static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;35static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
36 36
37static bool run_vcpus = true;
38
39static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)37static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
40{38{
41 struct kvm_vcpu *vcpu = vcpu_args->vcpu;39 struct kvm_vcpu *vcpu = vcpu_args->vcpu;
@@ -45,7 +43,7 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) · 3 unmodified lines
45 run = vcpu->run;43 run = vcpu->run;
46 44
47 /* Let the guest access its memory until a stop signal is received */45 /* Let the guest access its memory until a stop signal is received */
48 while (READ_ONCE(run_vcpus)) {46 while (!READ_ONCE(memstress_args.stop_vcpus)) {
49 ret = _vcpu_run(vcpu);47 ret = _vcpu_run(vcpu);
50 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);48 TEST_ASSERT(ret == 0, "vcpu_run failed: %d\n", ret);
51 49
@@ -109,8 +107,6 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 57 unmodified lines
109 107
110 add_remove_memslot(vm, p->delay, p->nr_iterations);108 add_remove_memslot(vm, p->delay, p->nr_iterations);
111 109
112 run_vcpus = false;
113
114 memstress_join_vcpu_threads(nr_vcpus);110 memstress_join_vcpu_threads(nr_vcpus);
115 pr_info("All vCPU threads joined\n");111 pr_info("All vCPU threads joined\n");
116 112
tools/testing/selftests/kvm/memslot_perf_test.c+207−106
@@ -20,20 +20,20 @@ · 19 unmodified lines
20#include <unistd.h>20#include <unistd.h>
21 21
22#include <linux/compiler.h>22#include <linux/compiler.h>
23#include <linux/sizes.h>
23 24
24#include <test_util.h>25#include <test_util.h>
25#include <kvm_util.h>26#include <kvm_util.h>
26#include <processor.h>27#include <processor.h>
27 28
28#define MEM_SIZE ((512U << 20) + 4096)29#define MEM_EXTRA_SIZE SZ_64K
29#define MEM_SIZE_PAGES (MEM_SIZE / 4096)30
30#define MEM_GPA 0x10000000UL31#define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE)
32#define MEM_GPA SZ_256M
31#define MEM_AUX_GPA MEM_GPA33#define MEM_AUX_GPA MEM_GPA
32#define MEM_SYNC_GPA MEM_AUX_GPA34#define MEM_SYNC_GPA MEM_AUX_GPA
33#define MEM_TEST_GPA (MEM_AUX_GPA + 4096)35#define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE)
34#define MEM_TEST_SIZE (MEM_SIZE - 4096)36#define MEM_TEST_SIZE (MEM_SIZE - MEM_EXTRA_SIZE)
35static_assert(MEM_SIZE % 4096 == 0, "invalid mem size");
36static_assert(MEM_TEST_SIZE % 4096 == 0, "invalid mem test size");
37 37
38/*38/*
39 * 32 MiB is max size that gets well over 100 iterations on 509 slots.39 * 32 MiB is max size that gets well over 100 iterations on 509 slots.
@@ -41,44 +41,38 @@ static_assert(MEM_TEST_SIZE % 4096 == 0, "invalid mem test size"); · 1 unmodified line
41 * 8194 slots in use can then be tested (although with slightly41 * 8194 slots in use can then be tested (although with slightly
42 * limited resolution).42 * limited resolution).
43 */43 */
44#define MEM_SIZE_MAP ((32U << 20) + 4096)44#define MEM_SIZE_MAP (SZ_32M + MEM_EXTRA_SIZE)
45#define MEM_SIZE_MAP_PAGES (MEM_SIZE_MAP / 4096)45#define MEM_TEST_MAP_SIZE (MEM_SIZE_MAP - MEM_EXTRA_SIZE)
46#define MEM_TEST_MAP_SIZE (MEM_SIZE_MAP - 4096)
47#define MEM_TEST_MAP_SIZE_PAGES (MEM_TEST_MAP_SIZE / 4096)
48static_assert(MEM_SIZE_MAP % 4096 == 0, "invalid map test region size");
49static_assert(MEM_TEST_MAP_SIZE % 4096 == 0, "invalid map test region size");
50static_assert(MEM_TEST_MAP_SIZE_PAGES % 2 == 0, "invalid map test region size");
51static_assert(MEM_TEST_MAP_SIZE_PAGES > 2, "invalid map test region size");
52 46
53/*47/*
54 * 128 MiB is min size that fills 32k slots with at least one page in each48 * 128 MiB is min size that fills 32k slots with at least one page in each
55 * while at the same time gets 100+ iterations in such test49 * while at the same time gets 100+ iterations in such test
50 *
51 * 2 MiB chunk size like a typical huge page
56 */52 */
57#define MEM_TEST_UNMAP_SIZE (128U << 20)53#define MEM_TEST_UNMAP_SIZE SZ_128M
58#define MEM_TEST_UNMAP_SIZE_PAGES (MEM_TEST_UNMAP_SIZE / 4096)54#define MEM_TEST_UNMAP_CHUNK_SIZE SZ_2M
59/* 2 MiB chunk size like a typical huge page */
60#define MEM_TEST_UNMAP_CHUNK_PAGES (2U << (20 - 12))
61static_assert(MEM_TEST_UNMAP_SIZE <= MEM_TEST_SIZE,
62 "invalid unmap test region size");
63static_assert(MEM_TEST_UNMAP_SIZE % 4096 == 0,
64 "invalid unmap test region size");
65static_assert(MEM_TEST_UNMAP_SIZE_PAGES %
66 (2 * MEM_TEST_UNMAP_CHUNK_PAGES) == 0,
67 "invalid unmap test region size");
68 55
69/*56/*
70 * For the move active test the middle of the test area is placed on57 * For the move active test the middle of the test area is placed on
71 * a memslot boundary: half lies in the memslot being moved, half in58 * a memslot boundary: half lies in the memslot being moved, half in
72 * other memslot(s).59 * other memslot(s).
73 *60 *
74 * When running this test with 32k memslots (32764, really) each memslot61 * We have different number of memory slots, excluding the reserved
75 * contains 4 pages.62 * memory slot 0, on various architectures and configurations. The
76 * The last one additionally contains the remaining 21 pages of memory,63 * memory size in this test is calculated by picking the maximal
77 * for the total size of 25 pages.64 * last memory slot's memory size, with alignment to the largest
78 * Hence, the maximum size here is 50 pages.65 * supported page size (64KB). In this way, the selected memory
66 * size for this test is compatible with test_memslot_move_prepare().
67 *
68 * architecture slots memory-per-slot memory-on-last-slot
69 * --------------------------------------------------------------
70 * x86-4KB 32763 16KB 160KB
71 * arm64-4KB 32766 16KB 112KB
72 * arm64-16KB 32766 16KB 112KB
73 * arm64-64KB 8192 64KB 128KB
79 */74 */
80#define MEM_TEST_MOVE_SIZE_PAGES (50)75#define MEM_TEST_MOVE_SIZE (3 * SZ_64K)
81#define MEM_TEST_MOVE_SIZE (MEM_TEST_MOVE_SIZE_PAGES * 4096)
82#define MEM_TEST_MOVE_GPA_DEST (MEM_GPA + MEM_SIZE)76#define MEM_TEST_MOVE_GPA_DEST (MEM_GPA + MEM_SIZE)
83static_assert(MEM_TEST_MOVE_SIZE <= MEM_TEST_SIZE,77static_assert(MEM_TEST_MOVE_SIZE <= MEM_TEST_SIZE,
84 "invalid move test region size");78 "invalid move test region size");
@@ -100,6 +94,7 @@ struct vm_data { · 15 unmodified lines
100};94};
101 95
102struct sync_area {96struct sync_area {
97 uint32_t guest_page_size;
103 atomic_bool start_flag;98 atomic_bool start_flag;
104 atomic_bool exit_flag;99 atomic_bool exit_flag;
105 atomic_bool sync_flag;100 atomic_bool sync_flag;
@@ -192,14 +187,15 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages) · 86 unmodified lines
192 uint64_t gpage, pgoffs;187 uint64_t gpage, pgoffs;
193 uint32_t slot, slotoffs;188 uint32_t slot, slotoffs;
194 void *base;189 void *base;
190 uint32_t guest_page_size = data->vm->page_size;
195 191
196 TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");192 TEST_ASSERT(gpa >= MEM_GPA, "Too low gpa to translate");
197 TEST_ASSERT(gpa < MEM_GPA + data->npages * 4096,193 TEST_ASSERT(gpa < MEM_GPA + data->npages * guest_page_size,
198 "Too high gpa to translate");194 "Too high gpa to translate");
199 gpa -= MEM_GPA;195 gpa -= MEM_GPA;
200 196
201 gpage = gpa / 4096;197 gpage = gpa / guest_page_size;
202 pgoffs = gpa % 4096;198 pgoffs = gpa % guest_page_size;
203 slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);199 slot = min(gpage / data->pages_per_slot, (uint64_t)data->nslots - 1);
204 slotoffs = gpage - (slot * data->pages_per_slot);200 slotoffs = gpage - (slot * data->pages_per_slot);
205 201
@@ -217,14 +213,16 @@ static void *vm_gpa2hva(struct vm_data *data, uint64_t gpa, uint64_t *rempages) · 11 unmodified lines
217 }213 }
218 214
219 base = data->hva_slots[slot];215 base = data->hva_slots[slot];
220 return (uint8_t *)base + slotoffs * 4096 + pgoffs;216 return (uint8_t *)base + slotoffs * guest_page_size + pgoffs;
221}217}
222 218
223static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)219static uint64_t vm_slot2gpa(struct vm_data *data, uint32_t slot)
224{220{
221 uint32_t guest_page_size = data->vm->page_size;
222
225 TEST_ASSERT(slot < data->nslots, "Too high slot number");223 TEST_ASSERT(slot < data->nslots, "Too high slot number");
226 224
227 return MEM_GPA + slot * data->pages_per_slot * 4096;225 return MEM_GPA + slot * data->pages_per_slot * guest_page_size;
228}226}
229 227
230static struct vm_data *alloc_vm(void)228static struct vm_data *alloc_vm(void)
@@ -241,81 +239,111 @@ static struct vm_data *alloc_vm(void) · 10 unmodified lines
241 return data;239 return data;
242}240}
243 241
242static bool check_slot_pages(uint32_t host_page_size, uint32_t guest_page_size,
243 uint64_t pages_per_slot, uint64_t rempages)
244{
245 if (!pages_per_slot)
246 return false;
247
248 if ((pages_per_slot * guest_page_size) % host_page_size)
249 return false;
250
251 if ((rempages * guest_page_size) % host_page_size)
252 return false;
253
254 return true;
255}
256
257
258static uint64_t get_max_slots(struct vm_data *data, uint32_t host_page_size)
259{
260 uint32_t guest_page_size = data->vm->page_size;
261 uint64_t mempages, pages_per_slot, rempages;
262 uint64_t slots;
263
264 mempages = data->npages;
265 slots = data->nslots;
266 while (--slots > 1) {
267 pages_per_slot = mempages / slots;
268 rempages = mempages % pages_per_slot;
269 if (check_slot_pages(host_page_size, guest_page_size,
270 pages_per_slot, rempages))
271 return slots + 1; /* slot 0 is reserved */
272 }
273
274 return 0;
275}
276
244static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,277static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots,
245 void *guest_code, uint64_t mempages,278 void *guest_code, uint64_t mem_size,
246 struct timespec *slot_runtime)279 struct timespec *slot_runtime)
247{280{
248 uint32_t max_mem_slots;281 uint64_t mempages, rempages;
249 uint64_t rempages;
250 uint64_t guest_addr;282 uint64_t guest_addr;
251 uint32_t slot;283 uint32_t slot, host_page_size, guest_page_size;
252 struct timespec tstart;284 struct timespec tstart;
253 struct sync_area *sync;285 struct sync_area *sync;
254 286
255 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);287 host_page_size = getpagesize();
256 TEST_ASSERT(max_mem_slots > 1,288 guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
257 "KVM_CAP_NR_MEMSLOTS should be greater than 1");289 mempages = mem_size / guest_page_size;
258 TEST_ASSERT(nslots > 1 || nslots == -1,
259 "Slot count cap should be greater than 1");
260 if (nslots != -1)
261 max_mem_slots = min(max_mem_slots, (uint32_t)nslots);
262 pr_info_v("Allowed number of memory slots: %"PRIu32"\n", max_mem_slots);
263 290
264 TEST_ASSERT(mempages > 1,291 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
265 "Can't test without any memory");292 TEST_ASSERT(data->vm->page_size == guest_page_size, "Invalid VM page size");
266 293
267 data->npages = mempages;294 data->npages = mempages;
268 data->nslots = max_mem_slots - 1;295 TEST_ASSERT(data->npages > 1, "Can't test without any memory");
269 data->pages_per_slot = mempages / data->nslots;296 data->nslots = nslots;
270 if (!data->pages_per_slot) {297 data->pages_per_slot = data->npages / data->nslots;
271 *maxslots = mempages + 1;298 rempages = data->npages % data->nslots;
299 if (!check_slot_pages(host_page_size, guest_page_size,
300 data->pages_per_slot, rempages)) {
301 *maxslots = get_max_slots(data, host_page_size);
272 return false;302 return false;
273 }303 }
274 304
275 rempages = mempages % data->nslots;
276 data->hva_slots = malloc(sizeof(*data->hva_slots) * data->nslots);305 data->hva_slots = malloc(sizeof(*data->hva_slots) * data->nslots);
277 TEST_ASSERT(data->hva_slots, "malloc() fail");306 TEST_ASSERT(data->hva_slots, "malloc() fail");
278 307
279 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);308 data->vm = __vm_create_with_one_vcpu(&data->vcpu, mempages, guest_code);
280 309
281 pr_info_v("Adding slots 1..%i, each slot with %"PRIu64" pages + %"PRIu64" extra pages last\n",310 pr_info_v("Adding slots 1..%i, each slot with %"PRIu64" pages + %"PRIu64" extra pages last\n",
282 max_mem_slots - 1, data->pages_per_slot, rempages);311 data->nslots, data->pages_per_slot, rempages);
283 312
284 clock_gettime(CLOCK_MONOTONIC, &tstart);313 clock_gettime(CLOCK_MONOTONIC, &tstart);
285 for (slot = 1, guest_addr = MEM_GPA; slot < max_mem_slots; slot++) {314 for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
286 uint64_t npages;315 uint64_t npages;
287 316
288 npages = data->pages_per_slot;317 npages = data->pages_per_slot;
289 if (slot == max_mem_slots - 1)318 if (slot == data->nslots)
290 npages += rempages;319 npages += rempages;
291 320
292 vm_userspace_mem_region_add(data->vm, VM_MEM_SRC_ANONYMOUS,321 vm_userspace_mem_region_add(data->vm, VM_MEM_SRC_ANONYMOUS,
293 guest_addr, slot, npages,322 guest_addr, slot, npages,
294 0);323 0);
295 guest_addr += npages * 4096;324 guest_addr += npages * guest_page_size;
296 }325 }
297 *slot_runtime = timespec_elapsed(tstart);326 *slot_runtime = timespec_elapsed(tstart);
298 327
299 for (slot = 0, guest_addr = MEM_GPA; slot < max_mem_slots - 1; slot++) {328 for (slot = 1, guest_addr = MEM_GPA; slot <= data->nslots; slot++) {
300 uint64_t npages;329 uint64_t npages;
301 uint64_t gpa;330 uint64_t gpa;
302 331
303 npages = data->pages_per_slot;332 npages = data->pages_per_slot;
304 if (slot == max_mem_slots - 2)333 if (slot == data->nslots)
305 npages += rempages;334 npages += rempages;
306 335
307 gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr,336 gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot);
308 slot + 1);
309 TEST_ASSERT(gpa == guest_addr,337 TEST_ASSERT(gpa == guest_addr,
310 "vm_phy_pages_alloc() failed\n");338 "vm_phy_pages_alloc() failed\n");
311 339
312 data->hva_slots[slot] = addr_gpa2hva(data->vm, guest_addr);340 data->hva_slots[slot - 1] = addr_gpa2hva(data->vm, guest_addr);
313 memset(data->hva_slots[slot], 0, npages * 4096);341 memset(data->hva_slots[slot - 1], 0, npages * guest_page_size);
314 342
315 guest_addr += npages * 4096;343 guest_addr += npages * guest_page_size;
316 }344 }
317 345
318 virt_map(data->vm, MEM_GPA, MEM_GPA, mempages);346 virt_map(data->vm, MEM_GPA, MEM_GPA, data->npages);
319 347
320 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);348 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
321 atomic_init(&sync->start_flag, false);349 atomic_init(&sync->start_flag, false);
@@ -414,6 +442,7 @@ static bool guest_perform_sync(void) · 92 unmodified lines
414static void guest_code_test_memslot_move(void)442static void guest_code_test_memslot_move(void)
415{443{
416 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;444 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
445 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
417 uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);446 uintptr_t base = (typeof(base))READ_ONCE(sync->move_area_ptr);
418 447
419 GUEST_SYNC(0);448 GUEST_SYNC(0);
@@ -424,7 +453,7 @@ static void guest_code_test_memslot_move(void) · 4 unmodified lines
424 uintptr_t ptr;453 uintptr_t ptr;
425 454
426 for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;455 for (ptr = base; ptr < base + MEM_TEST_MOVE_SIZE;
427 ptr += 4096)456 ptr += page_size)
428 *(uint64_t *)ptr = MEM_TEST_VAL_1;457 *(uint64_t *)ptr = MEM_TEST_VAL_1;
429 458
430 /*459 /*
@@ -442,6 +471,7 @@ static void guest_code_test_memslot_move(void) · 11 unmodified lines
442static void guest_code_test_memslot_map(void)471static void guest_code_test_memslot_map(void)
443{472{
444 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;473 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
474 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
445 475
446 GUEST_SYNC(0);476 GUEST_SYNC(0);
447 477
@@ -451,14 +481,16 @@ static void guest_code_test_memslot_map(void) · 3 unmodified lines
451 uintptr_t ptr;481 uintptr_t ptr;
452 482
453 for (ptr = MEM_TEST_GPA;483 for (ptr = MEM_TEST_GPA;
454 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2; ptr += 4096)484 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
485 ptr += page_size)
455 *(uint64_t *)ptr = MEM_TEST_VAL_1;486 *(uint64_t *)ptr = MEM_TEST_VAL_1;
456 487
457 if (!guest_perform_sync())488 if (!guest_perform_sync())
458 break;489 break;
459 490
460 for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;491 for (ptr = MEM_TEST_GPA + MEM_TEST_MAP_SIZE / 2;
461 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE; ptr += 4096)492 ptr < MEM_TEST_GPA + MEM_TEST_MAP_SIZE;
493 ptr += page_size)
462 *(uint64_t *)ptr = MEM_TEST_VAL_2;494 *(uint64_t *)ptr = MEM_TEST_VAL_2;
463 495
464 if (!guest_perform_sync())496 if (!guest_perform_sync())
@@ -505,6 +537,9 @@ static void guest_code_test_memslot_unmap(void) · 40 unmodified lines
505 537
506static void guest_code_test_memslot_rw(void)538static void guest_code_test_memslot_rw(void)
507{539{
540 struct sync_area *sync = (typeof(sync))MEM_SYNC_GPA;
541 uint32_t page_size = (typeof(page_size))READ_ONCE(sync->guest_page_size);
542
508 GUEST_SYNC(0);543 GUEST_SYNC(0);
509 544
510 guest_spin_until_start();545 guest_spin_until_start();
@@ -513,14 +548,14 @@ static void guest_code_test_memslot_rw(void) · 2 unmodified lines
513 uintptr_t ptr;548 uintptr_t ptr;
514 549
515 for (ptr = MEM_TEST_GPA;550 for (ptr = MEM_TEST_GPA;
516 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += 4096)551 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size)
517 *(uint64_t *)ptr = MEM_TEST_VAL_1;552 *(uint64_t *)ptr = MEM_TEST_VAL_1;
518 553
519 if (!guest_perform_sync())554 if (!guest_perform_sync())
520 break;555 break;
521 556
522 for (ptr = MEM_TEST_GPA + 4096 / 2;557 for (ptr = MEM_TEST_GPA + page_size / 2;
523 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += 4096) {558 ptr < MEM_TEST_GPA + MEM_TEST_SIZE; ptr += page_size) {
524 uint64_t val = *(uint64_t *)ptr;559 uint64_t val = *(uint64_t *)ptr;
525 560
526 GUEST_ASSERT_1(val == MEM_TEST_VAL_2, val);561 GUEST_ASSERT_1(val == MEM_TEST_VAL_2, val);
@@ -538,6 +573,7 @@ static bool test_memslot_move_prepare(struct vm_data *data, · 11 unmodified lines
538 struct sync_area *sync,573 struct sync_area *sync,
539 uint64_t *maxslots, bool isactive)574 uint64_t *maxslots, bool isactive)
540{575{
576 uint32_t guest_page_size = data->vm->page_size;
541 uint64_t movesrcgpa, movetestgpa;577 uint64_t movesrcgpa, movetestgpa;
542 578
543 movesrcgpa = vm_slot2gpa(data, data->nslots - 1);579 movesrcgpa = vm_slot2gpa(data, data->nslots - 1);
@@ -546,7 +582,7 @@ static bool test_memslot_move_prepare(struct vm_data *data, · 2 unmodified lines
546 uint64_t lastpages;582 uint64_t lastpages;
547 583
548 vm_gpa2hva(data, movesrcgpa, &lastpages);584 vm_gpa2hva(data, movesrcgpa, &lastpages);
549 if (lastpages < MEM_TEST_MOVE_SIZE_PAGES / 2) {585 if (lastpages * guest_page_size < MEM_TEST_MOVE_SIZE / 2) {
550 *maxslots = 0;586 *maxslots = 0;
551 return false;587 return false;
552 }588 }
@@ -592,8 +628,9 @@ static void test_memslot_do_unmap(struct vm_data *data, · 39 unmodified lines
592 uint64_t offsp, uint64_t count)628 uint64_t offsp, uint64_t count)
593{629{
594 uint64_t gpa, ctr;630 uint64_t gpa, ctr;
631 uint32_t guest_page_size = data->vm->page_size;
595 632
596 for (gpa = MEM_TEST_GPA + offsp * 4096, ctr = 0; ctr < count; ) {633 for (gpa = MEM_TEST_GPA + offsp * guest_page_size, ctr = 0; ctr < count; ) {
597 uint64_t npages;634 uint64_t npages;
598 void *hva;635 void *hva;
599 int ret;636 int ret;
@@ -601,12 +638,12 @@ static void test_memslot_do_unmap(struct vm_data *data, · 1 unmodified line
601 hva = vm_gpa2hva(data, gpa, &npages);638 hva = vm_gpa2hva(data, gpa, &npages);
602 TEST_ASSERT(npages, "Empty memory slot at gptr 0x%"PRIx64, gpa);639 TEST_ASSERT(npages, "Empty memory slot at gptr 0x%"PRIx64, gpa);
603 npages = min(npages, count - ctr);640 npages = min(npages, count - ctr);
604 ret = madvise(hva, npages * 4096, MADV_DONTNEED);641 ret = madvise(hva, npages * guest_page_size, MADV_DONTNEED);
605 TEST_ASSERT(!ret,642 TEST_ASSERT(!ret,
606 "madvise(%p, MADV_DONTNEED) on VM memory should not fail for gptr 0x%"PRIx64,643 "madvise(%p, MADV_DONTNEED) on VM memory should not fail for gptr 0x%"PRIx64,
607 hva, gpa);644 hva, gpa);
608 ctr += npages;645 ctr += npages;
609 gpa += npages * 4096;646 gpa += npages * guest_page_size;
610 }647 }
611 TEST_ASSERT(ctr == count,648 TEST_ASSERT(ctr == count,
612 "madvise(MADV_DONTNEED) should exactly cover all of the requested area");649 "madvise(MADV_DONTNEED) should exactly cover all of the requested area");
@@ -617,11 +654,12 @@ static void test_memslot_map_unmap_check(struct vm_data *data, · 4 unmodified lines
617{654{
618 uint64_t gpa;655 uint64_t gpa;
619 uint64_t *val;656 uint64_t *val;
657 uint32_t guest_page_size = data->vm->page_size;
620 658
621 if (!map_unmap_verify)659 if (!map_unmap_verify)
622 return;660 return;
623 661
624 gpa = MEM_TEST_GPA + offsp * 4096;662 gpa = MEM_TEST_GPA + offsp * guest_page_size;
625 val = (typeof(val))vm_gpa2hva(data, gpa, NULL);663 val = (typeof(val))vm_gpa2hva(data, gpa, NULL);
626 TEST_ASSERT(*val == valexp,664 TEST_ASSERT(*val == valexp,
627 "Guest written values should read back correctly before unmap (%"PRIu64" vs %"PRIu64" @ %"PRIx64")",665 "Guest written values should read back correctly before unmap (%"PRIu64" vs %"PRIu64" @ %"PRIx64")",
@@ -631,12 +669,14 @@ static void test_memslot_map_unmap_check(struct vm_data *data, · 3 unmodified lines
631 669
632static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)670static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync)
633{671{
672 uint32_t guest_page_size = data->vm->page_size;
673 uint64_t guest_pages = MEM_TEST_MAP_SIZE / guest_page_size;
674
634 /*675 /*
635 * Unmap the second half of the test area while guest writes to (maps)676 * Unmap the second half of the test area while guest writes to (maps)
636 * the first half.677 * the first half.
637 */678 */
638 test_memslot_do_unmap(data, MEM_TEST_MAP_SIZE_PAGES / 2,679 test_memslot_do_unmap(data, guest_pages / 2, guest_pages / 2);
639 MEM_TEST_MAP_SIZE_PAGES / 2);
640 680
641 /*681 /*
642 * Wait for the guest to finish writing the first half of the test682 * Wait for the guest to finish writing the first half of the test
@@ -647,10 +687,8 @@ static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync) · 4 unmodified lines
647 */687 */
648 host_perform_sync(sync);688 host_perform_sync(sync);
649 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);689 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
650 test_memslot_map_unmap_check(data,690 test_memslot_map_unmap_check(data, guest_pages / 2 - 1, MEM_TEST_VAL_1);
651 MEM_TEST_MAP_SIZE_PAGES / 2 - 1,691 test_memslot_do_unmap(data, 0, guest_pages / 2);
652 MEM_TEST_VAL_1);
653 test_memslot_do_unmap(data, 0, MEM_TEST_MAP_SIZE_PAGES / 2);
654 692
655 693
656 /*694 /*
@@ -663,16 +701,16 @@ static void test_memslot_map_loop(struct vm_data *data, struct sync_area *sync) · 6 unmodified lines
663 * the test area.701 * the test area.
664 */702 */
665 host_perform_sync(sync);703 host_perform_sync(sync);
666 test_memslot_map_unmap_check(data, MEM_TEST_MAP_SIZE_PAGES / 2,704 test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
667 MEM_TEST_VAL_2);705 test_memslot_map_unmap_check(data, guest_pages - 1, MEM_TEST_VAL_2);
668 test_memslot_map_unmap_check(data, MEM_TEST_MAP_SIZE_PAGES - 1,
669 MEM_TEST_VAL_2);
670}706}
671 707
672static void test_memslot_unmap_loop_common(struct vm_data *data,708static void test_memslot_unmap_loop_common(struct vm_data *data,
673 struct sync_area *sync,709 struct sync_area *sync,
674 uint64_t chunk)710 uint64_t chunk)
675{711{
712 uint32_t guest_page_size = data->vm->page_size;
713 uint64_t guest_pages = MEM_TEST_UNMAP_SIZE / guest_page_size;
676 uint64_t ctr;714 uint64_t ctr;
677 715
678 /*716 /*
@@ -684,42 +722,49 @@ static void test_memslot_unmap_loop_common(struct vm_data *data, · 5 unmodified lines
684 */722 */
685 host_perform_sync(sync);723 host_perform_sync(sync);
686 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);724 test_memslot_map_unmap_check(data, 0, MEM_TEST_VAL_1);
687 for (ctr = 0; ctr < MEM_TEST_UNMAP_SIZE_PAGES / 2; ctr += chunk)725 for (ctr = 0; ctr < guest_pages / 2; ctr += chunk)
688 test_memslot_do_unmap(data, ctr, chunk);726 test_memslot_do_unmap(data, ctr, chunk);
689 727
690 /* Likewise, but for the opposite host / guest areas */728 /* Likewise, but for the opposite host / guest areas */
691 host_perform_sync(sync);729 host_perform_sync(sync);
692 test_memslot_map_unmap_check(data, MEM_TEST_UNMAP_SIZE_PAGES / 2,730 test_memslot_map_unmap_check(data, guest_pages / 2, MEM_TEST_VAL_2);
693 MEM_TEST_VAL_2);731 for (ctr = guest_pages / 2; ctr < guest_pages; ctr += chunk)
694 for (ctr = MEM_TEST_UNMAP_SIZE_PAGES / 2;
695 ctr < MEM_TEST_UNMAP_SIZE_PAGES; ctr += chunk)
696 test_memslot_do_unmap(data, ctr, chunk);732 test_memslot_do_unmap(data, ctr, chunk);
697}733}
698 734
699static void test_memslot_unmap_loop(struct vm_data *data,735static void test_memslot_unmap_loop(struct vm_data *data,
700 struct sync_area *sync)736 struct sync_area *sync)
701{737{
702 test_memslot_unmap_loop_common(data, sync, 1);738 uint32_t host_page_size = getpagesize();
739 uint32_t guest_page_size = data->vm->page_size;
740 uint64_t guest_chunk_pages = guest_page_size >= host_page_size ?
741 1 : host_page_size / guest_page_size;
742
743 test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
703}744}
704 745
705static void test_memslot_unmap_loop_chunked(struct vm_data *data,746static void test_memslot_unmap_loop_chunked(struct vm_data *data,
706 struct sync_area *sync)747 struct sync_area *sync)
707{748{
708 test_memslot_unmap_loop_common(data, sync, MEM_TEST_UNMAP_CHUNK_PAGES);749 uint32_t guest_page_size = data->vm->page_size;
750 uint64_t guest_chunk_pages = MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size;
751
752 test_memslot_unmap_loop_common(data, sync, guest_chunk_pages);
709}753}
710 754
711static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)755static void test_memslot_rw_loop(struct vm_data *data, struct sync_area *sync)
712{756{
713 uint64_t gptr;757 uint64_t gptr;
758 uint32_t guest_page_size = data->vm->page_size;
714 759
715 for (gptr = MEM_TEST_GPA + 4096 / 2;760 for (gptr = MEM_TEST_GPA + guest_page_size / 2;
716 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += 4096)761 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size)
717 *(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;762 *(uint64_t *)vm_gpa2hva(data, gptr, NULL) = MEM_TEST_VAL_2;
718 763
719 host_perform_sync(sync);764 host_perform_sync(sync);
720 765
721 for (gptr = MEM_TEST_GPA;766 for (gptr = MEM_TEST_GPA;
722 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += 4096) {767 gptr < MEM_TEST_GPA + MEM_TEST_SIZE; gptr += guest_page_size) {
723 uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);768 uint64_t *vptr = (typeof(vptr))vm_gpa2hva(data, gptr, NULL);
724 uint64_t val = *vptr;769 uint64_t val = *vptr;
725 770
@@ -748,7 +793,7 @@ static bool test_execute(int nslots, uint64_t *maxslots, · 22 unmodified lines
748 struct timespec *slot_runtime,793 struct timespec *slot_runtime,
749 struct timespec *guest_runtime)794 struct timespec *guest_runtime)
750{795{
751 uint64_t mem_size = tdata->mem_size ? : MEM_SIZE_PAGES;796 uint64_t mem_size = tdata->mem_size ? : MEM_SIZE;
752 struct vm_data *data;797 struct vm_data *data;
753 struct sync_area *sync;798 struct sync_area *sync;
754 struct timespec tstart;799 struct timespec tstart;
@@ -763,6 +808,7 @@ static bool test_execute(int nslots, uint64_t *maxslots, · 8 unmodified lines
763 808
764 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);809 sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL);
765 810
811 sync->guest_page_size = data->vm->page_size;
766 if (tdata->prepare &&812 if (tdata->prepare &&
767 !tdata->prepare(data, sync, maxslots)) {813 !tdata->prepare(data, sync, maxslots)) {
768 ret = false;814 ret = false;
@@ -796,19 +842,19 @@ exit_free: · 27 unmodified lines
796static const struct test_data tests[] = {842static const struct test_data tests[] = {
797 {843 {
798 .name = "map",844 .name = "map",
799 .mem_size = MEM_SIZE_MAP_PAGES,845 .mem_size = MEM_SIZE_MAP,
800 .guest_code = guest_code_test_memslot_map,846 .guest_code = guest_code_test_memslot_map,
801 .loop = test_memslot_map_loop,847 .loop = test_memslot_map_loop,
802 },848 },
803 {849 {
804 .name = "unmap",850 .name = "unmap",
805 .mem_size = MEM_TEST_UNMAP_SIZE_PAGES + 1,851 .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
806 .guest_code = guest_code_test_memslot_unmap,852 .guest_code = guest_code_test_memslot_unmap,
807 .loop = test_memslot_unmap_loop,853 .loop = test_memslot_unmap_loop,
808 },854 },
809 {855 {
810 .name = "unmap chunked",856 .name = "unmap chunked",
811 .mem_size = MEM_TEST_UNMAP_SIZE_PAGES + 1,857 .mem_size = MEM_TEST_UNMAP_SIZE + MEM_EXTRA_SIZE,
812 .guest_code = guest_code_test_memslot_unmap,858 .guest_code = guest_code_test_memslot_unmap,
813 .loop = test_memslot_unmap_loop_chunked,859 .loop = test_memslot_unmap_loop_chunked,
814 },860 },
@@ -866,9 +912,46 @@ static void help(char *name, struct test_args *targs) · 51 unmodified lines
866 pr_info("%d: %s\n", ctr, tests[ctr].name);912 pr_info("%d: %s\n", ctr, tests[ctr].name);
867}913}
868 914
915static bool check_memory_sizes(void)
916{
917 uint32_t host_page_size = getpagesize();
918 uint32_t guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size;
919
920 if (host_page_size > SZ_64K || guest_page_size > SZ_64K) {
921 pr_info("Unsupported page size on host (0x%x) or guest (0x%x)\n",
922 host_page_size, guest_page_size);
923 return false;
924 }
925
926 if (MEM_SIZE % guest_page_size ||
927 MEM_TEST_SIZE % guest_page_size) {
928 pr_info("invalid MEM_SIZE or MEM_TEST_SIZE\n");
929 return false;
930 }
931
932 if (MEM_SIZE_MAP % guest_page_size ||
933 MEM_TEST_MAP_SIZE % guest_page_size ||
934 (MEM_TEST_MAP_SIZE / guest_page_size) <= 2 ||
935 (MEM_TEST_MAP_SIZE / guest_page_size) % 2) {
936 pr_info("invalid MEM_SIZE_MAP or MEM_TEST_MAP_SIZE\n");
937 return false;
938 }
939
940 if (MEM_TEST_UNMAP_SIZE > MEM_TEST_SIZE ||
941 MEM_TEST_UNMAP_SIZE % guest_page_size ||
942 (MEM_TEST_UNMAP_SIZE / guest_page_size) %
943 (2 * MEM_TEST_UNMAP_CHUNK_SIZE / guest_page_size)) {
944 pr_info("invalid MEM_TEST_UNMAP_SIZE or MEM_TEST_UNMAP_CHUNK_SIZE\n");
945 return false;
946 }
947
948 return true;
949}
950
869static bool parse_args(int argc, char *argv[],951static bool parse_args(int argc, char *argv[],
870 struct test_args *targs)952 struct test_args *targs)
871{953{
954 uint32_t max_mem_slots;
872 int opt;955 int opt;
873 956
874 while ((opt = getopt(argc, argv, "hvds:f:e:l:r:")) != -1) {957 while ((opt = getopt(argc, argv, "hvds:f:e:l:r:")) != -1) {
@@ -885,8 +968,8 @@ static bool parse_args(int argc, char *argv[], · 10 unmodified lines
885 break;968 break;
886 case 's':969 case 's':
887 targs->nslots = atoi_paranoid(optarg);970 targs->nslots = atoi_paranoid(optarg);
888 if (targs->nslots <= 0 && targs->nslots != -1) {971 if (targs->nslots <= 1 && targs->nslots != -1) {
889 pr_info("Slot count cap has to be positive or -1 for no cap\n");972 pr_info("Slot count cap must be larger than 1 or -1 for no cap\n");
890 return false;973 return false;
891 }974 }
892 break;975 break;
@@ -920,6 +1003,21 @@ static bool parse_args(int argc, char *argv[], · 27 unmodified lines
920 return false;1003 return false;
921 }1004 }
922 1005
1006 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
1007 if (max_mem_slots <= 1) {
1008 pr_info("KVM_CAP_NR_MEMSLOTS should be greater than 1\n");
1009 return false;
1010 }
1011
1012 /* Memory slot 0 is reserved */
1013 if (targs->nslots == -1)
1014 targs->nslots = max_mem_slots - 1;
1015 else
1016 targs->nslots = min_t(int, targs->nslots, max_mem_slots) - 1;
1017
1018 pr_info_v("Allowed Number of memory slots: %"PRIu32"\n",
1019 targs->nslots + 1);
1020
923 return true;1021 return true;
924}1022}
925 1023
@@ -994,6 +1092,9 @@ int main(int argc, char *argv[]) · 68 unmodified lines
994 struct test_result rbestslottime;1092 struct test_result rbestslottime;
995 int tctr;1093 int tctr;
996 1094
1095 if (!check_memory_sizes())
1096 return -1;
1097
997 if (!parse_args(argc, argv, &targs))1098 if (!parse_args(argc, argv, &targs))
998 return -1;1099 return -1;
999 1100
virt/kvm/Kconfig+6−0
@@ -33,6 +33,12 @@ config HAVE_KVM_DIRTY_RING_ACQ_REL · 32 unmodified lines
33 bool33 bool
34 select HAVE_KVM_DIRTY_RING34 select HAVE_KVM_DIRTY_RING
35 35
36# Allow enabling both the dirty bitmap and dirty ring. Only architectures
37# that need to dirty memory outside of a vCPU context should select this.
38config NEED_KVM_DIRTY_RING_WITH_BITMAP
39 bool
40 depends on HAVE_KVM_DIRTY_RING
41
36config HAVE_KVM_EVENTFD42config HAVE_KVM_EVENTFD
37 bool43 bool
38 select EVENTFD44 select EVENTFD
virt/kvm/dirty_ring.c+44−2
@@ -21,12 +21,26 @@ u32 kvm_dirty_ring_get_rsvd_entries(void) · 20 unmodified lines
21 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();21 return KVM_DIRTY_RING_RSVD_ENTRIES + kvm_cpu_dirty_log_size();
22}22}
23 23
24bool kvm_use_dirty_bitmap(struct kvm *kvm)
25{
26 lockdep_assert_held(&kvm->slots_lock);
27
28 return !kvm->dirty_ring_size || kvm->dirty_ring_with_bitmap;
29}
30
31#ifndef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
32bool kvm_arch_allow_write_without_running_vcpu(struct kvm *kvm)
33{
34 return false;
35}
36#endif
37
24static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)38static u32 kvm_dirty_ring_used(struct kvm_dirty_ring *ring)
25{39{
26 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);40 return READ_ONCE(ring->dirty_index) - READ_ONCE(ring->reset_index);
27}41}
28 42
29bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)43static bool kvm_dirty_ring_soft_full(struct kvm_dirty_ring *ring)
30{44{
31 return kvm_dirty_ring_used(ring) >= ring->soft_limit;45 return kvm_dirty_ring_used(ring) >= ring->soft_limit;
32}46}
@@ -142,13 +156,19 @@ int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring) · 109 unmodified lines
142 156
143 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);157 kvm_reset_dirty_gfn(kvm, cur_slot, cur_offset, mask);
144 158
159 /*
160 * The request KVM_REQ_DIRTY_RING_SOFT_FULL will be cleared
161 * by the VCPU thread next time when it enters the guest.
162 */
163
145 trace_kvm_dirty_ring_reset(ring);164 trace_kvm_dirty_ring_reset(ring);
146 165
147 return count;166 return count;
148}167}
149 168
150void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset)169void kvm_dirty_ring_push(struct kvm_vcpu *vcpu, u32 slot, u64 offset)
151{170{
171 struct kvm_dirty_ring *ring = &vcpu->dirty_ring;
152 struct kvm_dirty_gfn *entry;172 struct kvm_dirty_gfn *entry;
153 173
154 /* It should never get full */174 /* It should never get full */
@@ -166,6 +186,28 @@ void kvm_dirty_ring_push(struct kvm_dirty_ring *ring, u32 slot, u64 offset) · 11 unmodified lines
166 kvm_dirty_gfn_set_dirtied(entry);186 kvm_dirty_gfn_set_dirtied(entry);
167 ring->dirty_index++;187 ring->dirty_index++;
168 trace_kvm_dirty_ring_push(ring, slot, offset);188 trace_kvm_dirty_ring_push(ring, slot, offset);
189
190 if (kvm_dirty_ring_soft_full(ring))
191 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
192}
193
194bool kvm_dirty_ring_check_request(struct kvm_vcpu *vcpu)
195{
196 /*
197 * The VCPU isn't runnable when the dirty ring becomes soft full.
198 * The KVM_REQ_DIRTY_RING_SOFT_FULL event is always set to prevent
199 * the VCPU from running until the dirty pages are harvested and
200 * the dirty ring is reset by userspace.
201 */
202 if (kvm_check_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu) &&
203 kvm_dirty_ring_soft_full(&vcpu->dirty_ring)) {
204 kvm_make_request(KVM_REQ_DIRTY_RING_SOFT_FULL, vcpu);
205 vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
206 trace_kvm_dirty_ring_exit(vcpu);
207 return true;
208 }
209
210 return false;
169}211}
170 212
171struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)213struct page *kvm_dirty_ring_get_page(struct kvm_dirty_ring *ring, u32 offset)
virt/kvm/kvm_main.c+53−12
@@ -1617,7 +1617,7 @@ static int kvm_prepare_memory_region(struct kvm *kvm, · 1616 unmodified lines
1617 new->dirty_bitmap = NULL;1617 new->dirty_bitmap = NULL;
1618 else if (old && old->dirty_bitmap)1618 else if (old && old->dirty_bitmap)
1619 new->dirty_bitmap = old->dirty_bitmap;1619 new->dirty_bitmap = old->dirty_bitmap;
1620 else if (!kvm->dirty_ring_size) {1620 else if (kvm_use_dirty_bitmap(kvm)) {
1621 r = kvm_alloc_dirty_bitmap(new);1621 r = kvm_alloc_dirty_bitmap(new);
1622 if (r)1622 if (r)
1623 return r;1623 return r;
@@ -2068,8 +2068,8 @@ int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, · 444 unmodified lines
2068 unsigned long n;2068 unsigned long n;
2069 unsigned long any = 0;2069 unsigned long any = 0;
2070 2070
2071 /* Dirty ring tracking is exclusive to dirty log tracking */2071 /* Dirty ring tracking may be exclusive to dirty log tracking */
2072 if (kvm->dirty_ring_size)2072 if (!kvm_use_dirty_bitmap(kvm))
2073 return -ENXIO;2073 return -ENXIO;
2074 2074
2075 *memslot = NULL;2075 *memslot = NULL;
@@ -2133,8 +2133,8 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) · 57 unmodified lines
2133 unsigned long *dirty_bitmap_buffer;2133 unsigned long *dirty_bitmap_buffer;
2134 bool flush;2134 bool flush;
2135 2135
2136 /* Dirty ring tracking is exclusive to dirty log tracking */2136 /* Dirty ring tracking may be exclusive to dirty log tracking */
2137 if (kvm->dirty_ring_size)2137 if (!kvm_use_dirty_bitmap(kvm))
2138 return -ENXIO;2138 return -ENXIO;
2139 2139
2140 as_id = log->slot >> 16;2140 as_id = log->slot >> 16;
@@ -2245,8 +2245,8 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, · 104 unmodified lines
2245 unsigned long *dirty_bitmap_buffer;2245 unsigned long *dirty_bitmap_buffer;
2246 bool flush;2246 bool flush;
2247 2247
2248 /* Dirty ring tracking is exclusive to dirty log tracking */2248 /* Dirty ring tracking may be exclusive to dirty log tracking */
2249 if (kvm->dirty_ring_size)2249 if (!kvm_use_dirty_bitmap(kvm))
2250 return -ENXIO;2250 return -ENXIO;
2251 2251
2252 as_id = log->slot >> 16;2252 as_id = log->slot >> 16;
@@ -3321,18 +3321,19 @@ void mark_page_dirty_in_slot(struct kvm *kvm, · 1068 unmodified lines
3321 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();3321 struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
3322 3322
3323#ifdef CONFIG_HAVE_KVM_DIRTY_RING3323#ifdef CONFIG_HAVE_KVM_DIRTY_RING
3324 if (WARN_ON_ONCE(!vcpu) || WARN_ON_ONCE(vcpu->kvm != kvm))3324 if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
3325 return;3325 return;
3326
3327 WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
3326#endif3328#endif
3327 3329
3328 if (memslot && kvm_slot_dirty_track_enabled(memslot)) {3330 if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
3329 unsigned long rel_gfn = gfn - memslot->base_gfn;3331 unsigned long rel_gfn = gfn - memslot->base_gfn;
3330 u32 slot = (memslot->as_id << 16) | memslot->id;3332 u32 slot = (memslot->as_id << 16) | memslot->id;
3331 3333
3332 if (kvm->dirty_ring_size)3334 if (kvm->dirty_ring_size && vcpu)
3333 kvm_dirty_ring_push(&vcpu->dirty_ring,3335 kvm_dirty_ring_push(vcpu, slot, rel_gfn);
3334 slot, rel_gfn);3336 else if (memslot->dirty_bitmap)
3335 else
3336 set_bit_le(rel_gfn, memslot->dirty_bitmap);3337 set_bit_le(rel_gfn, memslot->dirty_bitmap);
3337 }3338 }
3338}3339}
@@ -4499,6 +4500,9 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) · 1160 unmodified lines
4499 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);4500 return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
4500#else4501#else
4501 return 0;4502 return 0;
4503#endif
4504#ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
4505 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
4502#endif4506#endif
4503 case KVM_CAP_BINARY_STATS_FD:4507 case KVM_CAP_BINARY_STATS_FD:
4504 case KVM_CAP_SYSTEM_EVENT_DATA:4508 case KVM_CAP_SYSTEM_EVENT_DATA:
@@ -4575,6 +4579,20 @@ int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm, · 70 unmodified lines
4575 return -EINVAL;4579 return -EINVAL;
4576}4580}
4577 4581
4582static bool kvm_are_all_memslots_empty(struct kvm *kvm)
4583{
4584 int i;
4585
4586 lockdep_assert_held(&kvm->slots_lock);
4587
4588 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
4589 if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
4590 return false;
4591 }
4592
4593 return true;
4594}
4595
4578static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,4596static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
4579 struct kvm_enable_cap *cap)4597 struct kvm_enable_cap *cap)
4580{4598{
@@ -4605,6 +4623,29 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, · 24 unmodified lines
4605 return -EINVAL;4623 return -EINVAL;
4606 4624
4607 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);4625 return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
4626 case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
4627 int r = -EINVAL;
4628
4629 if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
4630 !kvm->dirty_ring_size || cap->flags)
4631 return r;
4632
4633 mutex_lock(&kvm->slots_lock);
4634
4635 /*
4636 * For simplicity, allow enabling ring+bitmap if and only if
4637 * there are no memslots, e.g. to ensure all memslots allocate
4638 * a bitmap after the capability is enabled.
4639 */
4640 if (kvm_are_all_memslots_empty(kvm)) {
4641 kvm->dirty_ring_with_bitmap = true;
4642 r = 0;
4643 }
4644
4645 mutex_unlock(&kvm->slots_lock);
4646
4647 return r;
4648 }
4608 default:4649 default:
4609 return kvm_vm_ioctl_enable_cap(kvm, cap);4650 return kvm_vm_ioctl_enable_cap(kvm, cap);
4610 }4651 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t