History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull kvm fixes from Paolo Bonzini: "ARM: - Fix EL2 Stage-1 MMIO mappings where a random address was used - Fix SMCCC function number comparison when the SVE hint is set RISC-V: - Fix KVM_GET_REG_LIST API for ISA_EXT registers - Fix reading ISA_EXT register of a missing extension - Fix ISA_EXT register handling in get-reg-list test - Fix filtering of AIA registers in get-reg-list test x86: - Fixes for TSC_AUX virtualization - Stop zapping page tables asynchronously, since we don't zap them as often as before" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: KVM: SVM: Do not use user return MSR support for virtualized TSC_AUX KVM: SVM: Fix TSC_AUX virtualization setup KVM: SVM: INTERCEPT_RDTSCP is never intercepted anyway KVM: x86/mmu: Stop zapping invalidated TDP MMU roots asynchronously KVM: x86/mmu: Do not filter address spaces in for_each_tdp_mmu_root_yield_safe() KVM: x86/mmu: Open code leaf invalidation from mmu_notifier KVM: riscv: selftests: Selectively filter-out AIA registers KVM: riscv: selftests: Fix ISA_EXT register handling in get-reg-list RISC-V: KVM: Fix riscv_vcpu_get_isa_ext_single() for missing extensions RISC-V: KVM: Fix KVM_GET_REG_LIST API for ISA_EXT registers KVM: selftests: Assert that vasprintf() is successful KVM: arm64: nvhe: Ignore SVE hint in SMCCC function ID KVM: arm64: Properly return allocated EL2 VA from hyp_alloc_private_va_range()

Linus Torvalds committed 3 years ago

Commit 8a511e72 parents 5edc6bb 5804c1920 files · +209 −161Browse files

Unified · Split

20 files
  1. arch/arm64/include/asm/kvm_hyp.h +1 −1
  2. arch/arm64/kvm/hyp/include/nvhe/ffa.h +1 −1
  3. arch/arm64/kvm/hyp/nvhe/ffa.c +1 −2
  4. arch/arm64/kvm/hyp/nvhe/hyp-init.S +1 −0
  5. arch/arm64/kvm/hyp/nvhe/hyp-main.c +6 −2
  6. arch/arm64/kvm/hyp/nvhe/psci-relay.c +1 −2
  7. arch/arm64/kvm/mmu.c +3 −0
  8. arch/riscv/kvm/vcpu_onereg.c +5 −2
  9. arch/x86/include/asm/kvm_host.h +1 −2
  10. arch/x86/kvm/mmu/mmu.c +5 −16
  11. arch/x86/kvm/mmu/mmu_internal.h +7 −8
  12. arch/x86/kvm/mmu/tdp_mmu.c +67 −85
  13. arch/x86/kvm/mmu/tdp_mmu.h +2 −3
  14. arch/x86/kvm/svm/sev.c +26 −8
  15. arch/x86/kvm/svm/svm.c +35 −8
  16. arch/x86/kvm/svm/svm.h +1 −0
  17. arch/x86/kvm/x86.c +1 −4
  18. include/linux/arm-smccc.h +2 −0
  19. tools/testing/selftests/kvm/lib/test_util.c +1 −1
  20. tools/testing/selftests/kvm/riscv/get-reg-list.c +42 −16
arch/arm64/include/asm/kvm_hyp.h+1−1
@@ -118,7 +118,7 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu); · 117 unmodified lines
118 118
119u64 __guest_enter(struct kvm_vcpu *vcpu);119u64 __guest_enter(struct kvm_vcpu *vcpu);
120 120
121bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt);121bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id);
122 122
123#ifdef __KVM_NVHE_HYPERVISOR__123#ifdef __KVM_NVHE_HYPERVISOR__
124void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,124void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
arch/arm64/kvm/hyp/include/nvhe/ffa.h+1−1
@@ -12,6 +12,6 @@ · 11 unmodified lines
12#define FFA_MAX_FUNC_NUM 0x7F12#define FFA_MAX_FUNC_NUM 0x7F
13 13
14int hyp_ffa_init(void *pages);14int hyp_ffa_init(void *pages);
15bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt);15bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id);
16 16
17#endif /* __KVM_HYP_FFA_H */17#endif /* __KVM_HYP_FFA_H */
arch/arm64/kvm/hyp/nvhe/ffa.c+1−2
@@ -634,9 +634,8 @@ out_handled: · 633 unmodified lines
634 return true;634 return true;
635}635}
636 636
637bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt)637bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
638{638{
639 DECLARE_REG(u64, func_id, host_ctxt, 0);
640 struct arm_smccc_res res;639 struct arm_smccc_res res;
641 640
642 /*641 /*
arch/arm64/kvm/hyp/nvhe/hyp-init.S+1−0
@@ -57,6 +57,7 @@ __do_hyp_init: · 56 unmodified lines
57 cmp x0, #HVC_STUB_HCALL_NR57 cmp x0, #HVC_STUB_HCALL_NR
58 b.lo __kvm_handle_stub_hvc58 b.lo __kvm_handle_stub_hvc
59 59
60 bic x0, x0, #ARM_SMCCC_CALL_HINTS
60 mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)61 mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
61 cmp x0, x362 cmp x0, x3
62 b.eq 1f63 b.eq 1f
arch/arm64/kvm/hyp/nvhe/hyp-main.c+6−2
@@ -368,6 +368,7 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) · 367 unmodified lines
368 if (static_branch_unlikely(&kvm_protected_mode_initialized))368 if (static_branch_unlikely(&kvm_protected_mode_initialized))
369 hcall_min = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize;369 hcall_min = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize;
370 370
371 id &= ~ARM_SMCCC_CALL_HINTS;
371 id -= KVM_HOST_SMCCC_ID(0);372 id -= KVM_HOST_SMCCC_ID(0);
372 373
373 if (unlikely(id < hcall_min || id >= ARRAY_SIZE(host_hcall)))374 if (unlikely(id < hcall_min || id >= ARRAY_SIZE(host_hcall)))
@@ -392,11 +393,14 @@ static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt) · 18 unmodified lines
392 393
393static void handle_host_smc(struct kvm_cpu_context *host_ctxt)394static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
394{395{
396 DECLARE_REG(u64, func_id, host_ctxt, 0);
395 bool handled;397 bool handled;
396 398
397 handled = kvm_host_psci_handler(host_ctxt);399 func_id &= ~ARM_SMCCC_CALL_HINTS;
400
401 handled = kvm_host_psci_handler(host_ctxt, func_id);
398 if (!handled)402 if (!handled)
399 handled = kvm_host_ffa_handler(host_ctxt);403 handled = kvm_host_ffa_handler(host_ctxt, func_id);
400 if (!handled)404 if (!handled)
401 default_host_smc_handler(host_ctxt);405 default_host_smc_handler(host_ctxt);
402 406
arch/arm64/kvm/hyp/nvhe/psci-relay.c+1−2
@@ -273,9 +273,8 @@ static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ · 272 unmodified lines
273 }273 }
274}274}
275 275
276bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt)276bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
277{277{
278 DECLARE_REG(u64, func_id, host_ctxt, 0);
279 unsigned long ret;278 unsigned long ret;
280 279
281 switch (kvm_host_psci_config.version) {280 switch (kvm_host_psci_config.version) {
arch/arm64/kvm/mmu.c+3−0
@@ -652,6 +652,9 @@ int hyp_alloc_private_va_range(size_t size, unsigned long *haddr) · 651 unmodified lines
652 652
653 mutex_unlock(&kvm_hyp_pgd_mutex);653 mutex_unlock(&kvm_hyp_pgd_mutex);
654 654
655 if (!ret)
656 *haddr = base;
657
655 return ret;658 return ret;
656}659}
657 660
arch/riscv/kvm/vcpu_onereg.c+5−2
@@ -460,8 +460,11 @@ static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, · 459 unmodified lines
460 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))460 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
461 return -ENOENT;461 return -ENOENT;
462 462
463 *reg_val = 0;
464 host_isa_ext = kvm_isa_ext_arr[reg_num];463 host_isa_ext = kvm_isa_ext_arr[reg_num];
464 if (!__riscv_isa_extension_available(NULL, host_isa_ext))
465 return -ENOENT;
466
467 *reg_val = 0;
465 if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))468 if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
466 *reg_val = 1; /* Mark the given extension as available */469 *reg_val = 1; /* Mark the given extension as available */
467 470
@@ -842,7 +845,7 @@ static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, · 374 unmodified lines
842 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i;845 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i;
843 846
844 isa_ext = kvm_isa_ext_arr[i];847 isa_ext = kvm_isa_ext_arr[i];
845 if (!__riscv_isa_extension_available(vcpu->arch.isa, isa_ext))848 if (!__riscv_isa_extension_available(NULL, isa_ext))
846 continue;849 continue;
847 850
848 if (uindices) {851 if (uindices) {
arch/x86/include/asm/kvm_host.h+1−2
@@ -1419,7 +1419,6 @@ struct kvm_arch { · 1418 unmodified lines
1419 * the thread holds the MMU lock in write mode.1419 * the thread holds the MMU lock in write mode.
1420 */1420 */
1421 spinlock_t tdp_mmu_pages_lock;1421 spinlock_t tdp_mmu_pages_lock;
1422 struct workqueue_struct *tdp_mmu_zap_wq;
1423#endif /* CONFIG_X86_64 */1422#endif /* CONFIG_X86_64 */
1424 1423
1425 /*1424 /*
@@ -1835,7 +1834,7 @@ void kvm_mmu_vendor_module_exit(void); · 409 unmodified lines
1835 1834
1836void kvm_mmu_destroy(struct kvm_vcpu *vcpu);1835void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1837int kvm_mmu_create(struct kvm_vcpu *vcpu);1836int kvm_mmu_create(struct kvm_vcpu *vcpu);
1838int kvm_mmu_init_vm(struct kvm *kvm);1837void kvm_mmu_init_vm(struct kvm *kvm);
1839void kvm_mmu_uninit_vm(struct kvm *kvm);1838void kvm_mmu_uninit_vm(struct kvm *kvm);
1840 1839
1841void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);1840void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
arch/x86/kvm/mmu/mmu.c+5−16
@@ -6167,20 +6167,15 @@ static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) · 6166 unmodified lines
6167 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));6167 return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
6168}6168}
6169 6169
6170int kvm_mmu_init_vm(struct kvm *kvm)6170void kvm_mmu_init_vm(struct kvm *kvm)
6171{6171{
6172 int r;
6173
6174 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);6172 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
6175 INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);6173 INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
6176 INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages);6174 INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages);
6177 spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);6175 spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
6178 6176
6179 if (tdp_mmu_enabled) {6177 if (tdp_mmu_enabled)
6180 r = kvm_mmu_init_tdp_mmu(kvm);6178 kvm_mmu_init_tdp_mmu(kvm);
6181 if (r < 0)
6182 return r;
6183 }
6184 6179
6185 kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;6180 kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;
6186 kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;6181 kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;
@@ -6189,8 +6184,6 @@ int kvm_mmu_init_vm(struct kvm *kvm) · 2 unmodified lines
6189 6184
6190 kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;6185 kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;
6191 kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;6186 kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;
6192
6193 return 0;
6194}6187}
6195 6188
6196static void mmu_free_vm_memory_caches(struct kvm *kvm)6189static void mmu_free_vm_memory_caches(struct kvm *kvm)
@@ -6246,7 +6239,6 @@ static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_e · 49 unmodified lines
6246void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)6239void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
6247{6240{
6248 bool flush;6241 bool flush;
6249 int i;
6250 6242
6251 if (WARN_ON_ONCE(gfn_end <= gfn_start))6243 if (WARN_ON_ONCE(gfn_end <= gfn_start))
6252 return;6244 return;
@@ -6257,11 +6249,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 4 unmodified lines
6257 6249
6258 flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);6250 flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
6259 6251
6260 if (tdp_mmu_enabled) {6252 if (tdp_mmu_enabled)
6261 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)6253 flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
6262 flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
6263 gfn_end, true, flush);
6264 }
6265 6254
6266 if (flush)6255 if (flush)
6267 kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);6256 kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
arch/x86/kvm/mmu/mmu_internal.h+7−8
@@ -58,7 +58,12 @@ struct kvm_mmu_page { · 57 unmodified lines
58 58
59 bool tdp_mmu_page;59 bool tdp_mmu_page;
60 bool unsync;60 bool unsync;
61 u8 mmu_valid_gen;61 union {
62 u8 mmu_valid_gen;
63
64 /* Only accessed under slots_lock. */
65 bool tdp_mmu_scheduled_root_to_zap;
66 };
62 67
63 /*68 /*
64 * The shadow page can't be replaced by an equivalent huge page69 * The shadow page can't be replaced by an equivalent huge page
@@ -100,13 +105,7 @@ struct kvm_mmu_page { · 35 unmodified lines
100 struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */105 struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
101 tdp_ptep_t ptep;106 tdp_ptep_t ptep;
102 };107 };
103 union {108 DECLARE_BITMAP(unsync_child_bitmap, 512);
104 DECLARE_BITMAP(unsync_child_bitmap, 512);
105 struct {
106 struct work_struct tdp_mmu_async_work;
107 void *tdp_mmu_async_data;
108 };
109 };
110 109
111 /*110 /*
112 * Tracks shadow pages that, if zapped, would allow KVM to create an NX111 * Tracks shadow pages that, if zapped, would allow KVM to create an NX
arch/x86/kvm/mmu/tdp_mmu.c+67−85
@@ -12,18 +12,10 @@ · 11 unmodified lines
12#include <trace/events/kvm.h>12#include <trace/events/kvm.h>
13 13
14/* Initializes the TDP MMU for the VM, if enabled. */14/* Initializes the TDP MMU for the VM, if enabled. */
15int kvm_mmu_init_tdp_mmu(struct kvm *kvm)15void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
16{16{
17 struct workqueue_struct *wq;
18
19 wq = alloc_workqueue("kvm", WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 0);
20 if (!wq)
21 return -ENOMEM;
22
23 INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);17 INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
24 spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);18 spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
25 kvm->arch.tdp_mmu_zap_wq = wq;
26 return 1;
27}19}
28 20
29/* Arbitrarily returns true so that this may be used in if statements. */21/* Arbitrarily returns true so that this may be used in if statements. */
@@ -46,20 +38,15 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) · 16 unmodified lines
46 * ultimately frees all roots.38 * ultimately frees all roots.
47 */39 */
48 kvm_tdp_mmu_invalidate_all_roots(kvm);40 kvm_tdp_mmu_invalidate_all_roots(kvm);
49 41 kvm_tdp_mmu_zap_invalidated_roots(kvm);
50 /*
51 * Destroying a workqueue also first flushes the workqueue, i.e. no
52 * need to invoke kvm_tdp_mmu_zap_invalidated_roots().
53 */
54 destroy_workqueue(kvm->arch.tdp_mmu_zap_wq);
55 42
56 WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));43 WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));
57 WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));44 WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
58 45
59 /*46 /*
60 * Ensure that all the outstanding RCU callbacks to free shadow pages47 * Ensure that all the outstanding RCU callbacks to free shadow pages
61 * can run before the VM is torn down. Work items on tdp_mmu_zap_wq48 * can run before the VM is torn down. Putting the last reference to
62 * can call kvm_tdp_mmu_put_root and create new callbacks.49 * zapped roots will create new callbacks.
63 */50 */
64 rcu_barrier();51 rcu_barrier();
65}52}
@@ -86,46 +73,6 @@ static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) · 20 unmodified lines
86 tdp_mmu_free_sp(sp);73 tdp_mmu_free_sp(sp);
87}74}
88 75
89static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
90 bool shared);
91
92static void tdp_mmu_zap_root_work(struct work_struct *work)
93{
94 struct kvm_mmu_page *root = container_of(work, struct kvm_mmu_page,
95 tdp_mmu_async_work);
96 struct kvm *kvm = root->tdp_mmu_async_data;
97
98 read_lock(&kvm->mmu_lock);
99
100 /*
101 * A TLB flush is not necessary as KVM performs a local TLB flush when
102 * allocating a new root (see kvm_mmu_load()), and when migrating vCPU
103 * to a different pCPU. Note, the local TLB flush on reuse also
104 * invalidates any paging-structure-cache entries, i.e. TLB entries for
105 * intermediate paging structures, that may be zapped, as such entries
106 * are associated with the ASID on both VMX and SVM.
107 */
108 tdp_mmu_zap_root(kvm, root, true);
109
110 /*
111 * Drop the refcount using kvm_tdp_mmu_put_root() to test its logic for
112 * avoiding an infinite loop. By design, the root is reachable while
113 * it's being asynchronously zapped, thus a different task can put its
114 * last reference, i.e. flowing through kvm_tdp_mmu_put_root() for an
115 * asynchronously zapped root is unavoidable.
116 */
117 kvm_tdp_mmu_put_root(kvm, root, true);
118
119 read_unlock(&kvm->mmu_lock);
120}
121
122static void tdp_mmu_schedule_zap_root(struct kvm *kvm, struct kvm_mmu_page *root)
123{
124 root->tdp_mmu_async_data = kvm;
125 INIT_WORK(&root->tdp_mmu_async_work, tdp_mmu_zap_root_work);
126 queue_work(kvm->arch.tdp_mmu_zap_wq, &root->tdp_mmu_async_work);
127}
128
129void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,76void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
130 bool shared)77 bool shared)
131{78{
@@ -211,8 +158,12 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm, · 79 unmodified lines
211#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \158#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
212 __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)159 __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)
213 160
214#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \161#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _shared) \
215 __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, false, false)162 for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, false); \
163 _root; \
164 _root = tdp_mmu_next_root(_kvm, _root, _shared, false)) \
165 if (!kvm_lockdep_assert_mmu_lock_held(_kvm, _shared)) { \
166 } else
216 167
217/*168/*
218 * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write,169 * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write,
@@ -292,7 +243,7 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) · 73 unmodified lines
292 * by a memslot update or by the destruction of the VM. Initialize the243 * by a memslot update or by the destruction of the VM. Initialize the
293 * refcount to two; one reference for the vCPU, and one reference for244 * refcount to two; one reference for the vCPU, and one reference for
294 * the TDP MMU itself, which is held until the root is invalidated and245 * the TDP MMU itself, which is held until the root is invalidated and
295 * is ultimately put by tdp_mmu_zap_root_work().246 * is ultimately put by kvm_tdp_mmu_zap_invalidated_roots().
296 */247 */
297 refcount_set(&root->tdp_mmu_root_count, 2);248 refcount_set(&root->tdp_mmu_root_count, 2);
298 249
@@ -877,13 +828,12 @@ static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root, · 578 unmodified lines
877 * true if a TLB flush is needed before releasing the MMU lock, i.e. if one or828 * true if a TLB flush is needed before releasing the MMU lock, i.e. if one or
878 * more SPTEs were zapped since the MMU lock was last acquired.829 * more SPTEs were zapped since the MMU lock was last acquired.
879 */830 */
880bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,831bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
881 bool can_yield, bool flush)
882{832{
883 struct kvm_mmu_page *root;833 struct kvm_mmu_page *root;
884 834
885 for_each_tdp_mmu_root_yield_safe(kvm, root, as_id)835 for_each_tdp_mmu_root_yield_safe(kvm, root, false)
886 flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush);836 flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
887 837
888 return flush;838 return flush;
889}839}
@@ -891,7 +841,6 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, · 1 unmodified line
891void kvm_tdp_mmu_zap_all(struct kvm *kvm)841void kvm_tdp_mmu_zap_all(struct kvm *kvm)
892{842{
893 struct kvm_mmu_page *root;843 struct kvm_mmu_page *root;
894 int i;
895 844
896 /*845 /*
897 * Zap all roots, including invalid roots, as all SPTEs must be dropped846 * Zap all roots, including invalid roots, as all SPTEs must be dropped
@@ -905,10 +854,8 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) · 7 unmodified lines
905 * is being destroyed or the userspace VMM has exited. In both cases,854 * is being destroyed or the userspace VMM has exited. In both cases,
906 * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.855 * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.
907 */856 */
908 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {857 for_each_tdp_mmu_root_yield_safe(kvm, root, false)
909 for_each_tdp_mmu_root_yield_safe(kvm, root, i)858 tdp_mmu_zap_root(kvm, root, false);
910 tdp_mmu_zap_root(kvm, root, false);
911 }
912}859}
913 860
914/*861/*
@@ -917,18 +864,47 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) · 2 unmodified lines
917 */864 */
918void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)865void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)
919{866{
920 flush_workqueue(kvm->arch.tdp_mmu_zap_wq);867 struct kvm_mmu_page *root;
868
869 read_lock(&kvm->mmu_lock);
870
871 for_each_tdp_mmu_root_yield_safe(kvm, root, true) {
872 if (!root->tdp_mmu_scheduled_root_to_zap)
873 continue;
874
875 root->tdp_mmu_scheduled_root_to_zap = false;
876 KVM_BUG_ON(!root->role.invalid, kvm);
877
878 /*
879 * A TLB flush is not necessary as KVM performs a local TLB
880 * flush when allocating a new root (see kvm_mmu_load()), and
881 * when migrating a vCPU to a different pCPU. Note, the local
882 * TLB flush on reuse also invalidates paging-structure-cache
883 * entries, i.e. TLB entries for intermediate paging structures,
884 * that may be zapped, as such entries are associated with the
885 * ASID on both VMX and SVM.
886 */
887 tdp_mmu_zap_root(kvm, root, true);
888
889 /*
890 * The referenced needs to be put *after* zapping the root, as
891 * the root must be reachable by mmu_notifiers while it's being
892 * zapped
893 */
894 kvm_tdp_mmu_put_root(kvm, root, true);
895 }
896
897 read_unlock(&kvm->mmu_lock);
921}898}
922 899
923/*900/*
924 * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that901 * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that
925 * is about to be zapped, e.g. in response to a memslots update. The actual902 * is about to be zapped, e.g. in response to a memslots update. The actual
926 * zapping is performed asynchronously. Using a separate workqueue makes it903 * zapping is done separately so that it happens with mmu_lock with read,
927 * easy to ensure that the destruction is performed before the "fast zap"904 * whereas invalidating roots must be done with mmu_lock held for write (unless
928 * completes, without keeping a separate list of invalidated roots; the list is905 * the VM is being destroyed).
929 * effectively the list of work items in the workqueue.
930 *906 *
931 * Note, the asynchronous worker is gifted the TDP MMU's reference.907 * Note, kvm_tdp_mmu_zap_invalidated_roots() is gifted the TDP MMU's reference.
932 * See kvm_tdp_mmu_get_vcpu_root_hpa().908 * See kvm_tdp_mmu_get_vcpu_root_hpa().
933 */909 */
934void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm)910void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm)
@@ -953,19 +929,20 @@ void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) · 18 unmodified lines
953 /*929 /*
954 * As above, mmu_lock isn't held when destroying the VM! There can't930 * As above, mmu_lock isn't held when destroying the VM! There can't
955 * be other references to @kvm, i.e. nothing else can invalidate roots931 * be other references to @kvm, i.e. nothing else can invalidate roots
956 * or be consuming roots, but walking the list of roots does need to be932 * or get/put references to roots.
957 * guarded against roots being deleted by the asynchronous zap worker.
958 */933 */
959 rcu_read_lock();934 list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
960 935 /*
961 list_for_each_entry_rcu(root, &kvm->arch.tdp_mmu_roots, link) {936 * Note, invalid roots can outlive a memslot update! Invalid
937 * roots must be *zapped* before the memslot update completes,
938 * but a different task can acquire a reference and keep the
939 * root alive after its been zapped.
940 */
962 if (!root->role.invalid) {941 if (!root->role.invalid) {
942 root->tdp_mmu_scheduled_root_to_zap = true;
963 root->role.invalid = true;943 root->role.invalid = true;
964 tdp_mmu_schedule_zap_root(kvm, root);
965 }944 }
966 }945 }
967
968 rcu_read_unlock();
969}946}
970 947
971/*948/*
@@ -1146,8 +1123,13 @@ retry: · 174 unmodified lines
1146bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,1123bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
1147 bool flush)1124 bool flush)
1148{1125{
1149 return kvm_tdp_mmu_zap_leafs(kvm, range->slot->as_id, range->start,1126 struct kvm_mmu_page *root;
1150 range->end, range->may_block, flush);1127
1128 __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false)
1129 flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
1130 range->may_block, flush);
1131
1132 return flush;
1151}1133}
1152 1134
1153typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter,1135typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter,
arch/x86/kvm/mmu/tdp_mmu.h+2−3
@@ -7,7 +7,7 @@ · 6 unmodified lines
7 7
8#include "spte.h"8#include "spte.h"
9 9
10int kvm_mmu_init_tdp_mmu(struct kvm *kvm);10void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
11void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);11void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
12 12
13hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);13hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);
@@ -20,8 +20,7 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root) · 6 unmodified lines
20void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,20void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
21 bool shared);21 bool shared);
22 22
23bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start,23bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
24 gfn_t end, bool can_yield, bool flush);
25bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);24bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);
26void kvm_tdp_mmu_zap_all(struct kvm *kvm);25void kvm_tdp_mmu_zap_all(struct kvm *kvm);
27void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);26void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);
arch/x86/kvm/svm/sev.c+26−8
@@ -2962,6 +2962,32 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) · 2961 unmodified lines
2962 count, in);2962 count, in);
2963}2963}
2964 2964
2965static void sev_es_vcpu_after_set_cpuid(struct vcpu_svm *svm)
2966{
2967 struct kvm_vcpu *vcpu = &svm->vcpu;
2968
2969 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
2970 bool v_tsc_aux = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) ||
2971 guest_cpuid_has(vcpu, X86_FEATURE_RDPID);
2972
2973 set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, v_tsc_aux, v_tsc_aux);
2974 }
2975}
2976
2977void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm)
2978{
2979 struct kvm_vcpu *vcpu = &svm->vcpu;
2980 struct kvm_cpuid_entry2 *best;
2981
2982 /* For sev guests, the memory encryption bit is not reserved in CR3. */
2983 best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
2984 if (best)
2985 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
2986
2987 if (sev_es_guest(svm->vcpu.kvm))
2988 sev_es_vcpu_after_set_cpuid(svm);
2989}
2990
2965static void sev_es_init_vmcb(struct vcpu_svm *svm)2991static void sev_es_init_vmcb(struct vcpu_svm *svm)
2966{2992{
2967 struct vmcb *vmcb = svm->vmcb01.ptr;2993 struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -3024,14 +3050,6 @@ static void sev_es_init_vmcb(struct vcpu_svm *svm) · 56 unmodified lines
3024 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);3050 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
3025 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);3051 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
3026 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);3052 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
3027
3028 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) &&
3029 (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP) ||
3030 guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDPID))) {
3031 set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, 1, 1);
3032 if (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP))
3033 svm_clr_intercept(svm, INTERCEPT_RDTSCP);
3034 }
3035}3053}
3036 3054
3037void sev_init_vmcb(struct vcpu_svm *svm)3055void sev_init_vmcb(struct vcpu_svm *svm)
arch/x86/kvm/svm/svm.c+35−8
@@ -683,6 +683,21 @@ static int svm_hardware_enable(void) · 682 unmodified lines
683 683
684 amd_pmu_enable_virt();684 amd_pmu_enable_virt();
685 685
686 /*
687 * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
688 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
689 * Since Linux does not change the value of TSC_AUX once set, prime the
690 * TSC_AUX field now to avoid a RDMSR on every vCPU run.
691 */
692 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
693 struct sev_es_save_area *hostsa;
694 u32 msr_hi;
695
696 hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
697
698 rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi);
699 }
700
686 return 0;701 return 0;
687}702}
688 703
@@ -1532,7 +1547,14 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 843 unmodified lines
1532 if (tsc_scaling)1547 if (tsc_scaling)
1533 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);1548 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1534 1549
1535 if (likely(tsc_aux_uret_slot >= 0))1550 /*
1551 * TSC_AUX is always virtualized for SEV-ES guests when the feature is
1552 * available. The user return MSR support is not required in this case
1553 * because TSC_AUX is restored on #VMEXIT from the host save area
1554 * (which has been initialized in svm_hardware_enable()).
1555 */
1556 if (likely(tsc_aux_uret_slot >= 0) &&
1557 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1536 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);1558 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1537 1559
1538 svm->guest_state_loaded = true;1560 svm->guest_state_loaded = true;
@@ -3086,6 +3108,16 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) · 1547 unmodified lines
3086 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;3108 svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3087 break;3109 break;
3088 case MSR_TSC_AUX:3110 case MSR_TSC_AUX:
3111 /*
3112 * TSC_AUX is always virtualized for SEV-ES guests when the
3113 * feature is available. The user return MSR support is not
3114 * required in this case because TSC_AUX is restored on #VMEXIT
3115 * from the host save area (which has been initialized in
3116 * svm_hardware_enable()).
3117 */
3118 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
3119 break;
3120
3089 /*3121 /*
3090 * TSC_AUX is usually changed only during boot and never read3122 * TSC_AUX is usually changed only during boot and never read
3091 * directly. Intercept TSC_AUX instead of exposing it to the3123 * directly. Intercept TSC_AUX instead of exposing it to the
@@ -4284,7 +4316,6 @@ static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) · 1192 unmodified lines
4284static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)4316static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4285{4317{
4286 struct vcpu_svm *svm = to_svm(vcpu);4318 struct vcpu_svm *svm = to_svm(vcpu);
4287 struct kvm_cpuid_entry2 *best;
4288 4319
4289 /*4320 /*
4290 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM4321 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
@@ -4328,12 +4359,8 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 37 unmodified lines
4328 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,4359 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4329 !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));4360 !!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4330 4361
4331 /* For sev guests, the memory encryption bit is not reserved in CR3. */4362 if (sev_guest(vcpu->kvm))
4332 if (sev_guest(vcpu->kvm)) {4363 sev_vcpu_after_set_cpuid(svm);
4333 best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
4334 if (best)
4335 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4336 }
4337 4364
4338 init_vmcb_after_set_cpuid(vcpu);4365 init_vmcb_after_set_cpuid(vcpu);
4339}4366}
arch/x86/kvm/svm/svm.h+1−0
@@ -684,6 +684,7 @@ void __init sev_hardware_setup(void); · 683 unmodified lines
684void sev_hardware_unsetup(void);684void sev_hardware_unsetup(void);
685int sev_cpu_init(struct svm_cpu_data *sd);685int sev_cpu_init(struct svm_cpu_data *sd);
686void sev_init_vmcb(struct vcpu_svm *svm);686void sev_init_vmcb(struct vcpu_svm *svm);
687void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm);
687void sev_free_vcpu(struct kvm_vcpu *vcpu);688void sev_free_vcpu(struct kvm_vcpu *vcpu);
688int sev_handle_vmgexit(struct kvm_vcpu *vcpu);689int sev_handle_vmgexit(struct kvm_vcpu *vcpu);
689int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);690int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);
arch/x86/kvm/x86.c+1−4
@@ -12308,9 +12308,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 12307 unmodified lines
12308 if (ret)12308 if (ret)
12309 goto out;12309 goto out;
12310 12310
12311 ret = kvm_mmu_init_vm(kvm);12311 kvm_mmu_init_vm(kvm);
12312 if (ret)
12313 goto out_page_track;
12314 12312
12315 ret = static_call(kvm_x86_vm_init)(kvm);12313 ret = static_call(kvm_x86_vm_init)(kvm);
12316 if (ret)12314 if (ret)
@@ -12355,7 +12353,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 38 unmodified lines
12355 12353
12356out_uninit_mmu:12354out_uninit_mmu:
12357 kvm_mmu_uninit_vm(kvm);12355 kvm_mmu_uninit_vm(kvm);
12358out_page_track:
12359 kvm_page_track_cleanup(kvm);12356 kvm_page_track_cleanup(kvm);
12360out:12357out:
12361 return ret;12358 return ret;
include/linux/arm-smccc.h+2−0
@@ -67,6 +67,8 @@ · 66 unmodified lines
67#define ARM_SMCCC_VERSION_1_3 0x1000367#define ARM_SMCCC_VERSION_1_3 0x10003
68 68
69#define ARM_SMCCC_1_3_SVE_HINT 0x1000069#define ARM_SMCCC_1_3_SVE_HINT 0x10000
70#define ARM_SMCCC_CALL_HINTS ARM_SMCCC_1_3_SVE_HINT
71
70 72
71#define ARM_SMCCC_VERSION_FUNC_ID \73#define ARM_SMCCC_VERSION_FUNC_ID \
72 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \74 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
tools/testing/selftests/kvm/lib/test_util.c+1−1
@@ -387,7 +387,7 @@ char *strdup_printf(const char *fmt, ...) · 386 unmodified lines
387 char *str;387 char *str;
388 388
389 va_start(ap, fmt);389 va_start(ap, fmt);
390 vasprintf(&str, fmt, ap);390 TEST_ASSERT(vasprintf(&str, fmt, ap) >= 0, "vasprintf() failed");
391 va_end(ap);391 va_end(ap);
392 392
393 return str;393 return str;
tools/testing/selftests/kvm/riscv/get-reg-list.c+42−16
@@ -12,19 +12,37 @@ · 11 unmodified lines
12 12
13#define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK)13#define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK)
14 14
15static bool isa_ext_cant_disable[KVM_RISCV_ISA_EXT_MAX];
16
15bool filter_reg(__u64 reg)17bool filter_reg(__u64 reg)
16{18{
19 switch (reg & ~REG_MASK) {
17 /*20 /*
18 * Some ISA extensions are optional and not present on all host,21 * Same set of ISA_EXT registers are not present on all host because
19 * but they can't be disabled through ISA_EXT registers when present.22 * ISA_EXT registers are visible to the KVM user space based on the
20 * So, to make life easy, just filtering out these kind of registers.23 * ISA extensions available on the host. Also, disabling an ISA
24 * extension using corresponding ISA_EXT register does not affect
25 * the visibility of the ISA_EXT register itself.
26 *
27 * Based on above, we should filter-out all ISA_EXT registers.
21 */28 */
22 switch (reg & ~REG_MASK) {29 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_A:
30 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_C:
31 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_D:
32 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_F:
33 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_H:
34 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_I:
35 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_M:
36 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVPBMT:
23 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSTC:37 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSTC:
24 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVINVAL:38 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVINVAL:
25 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHINTPAUSE:39 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
40 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOM:
41 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICBOZ:
26 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBB:42 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBB:
27 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSAIA:43 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SSAIA:
44 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_V:
45 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_SVNAPOT:
28 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBA:46 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBA:
29 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBS:47 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZBS:
30 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICNTR:48 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZICNTR:
@@ -32,6 +50,15 @@ bool filter_reg(__u64 reg) · 1 unmodified line
32 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIFENCEI:50 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIFENCEI:
33 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHPM:51 case KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_ZIHPM:
34 return true;52 return true;
53 /* AIA registers are always available when Ssaia can't be disabled */
54 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siselect):
55 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio1):
56 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio2):
57 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(sieh):
58 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(siph):
59 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio1h):
60 case KVM_REG_RISCV_CSR | KVM_REG_RISCV_CSR_AIA | KVM_REG_RISCV_CSR_AIA_REG(iprio2h):
61 return isa_ext_cant_disable[KVM_RISCV_ISA_EXT_SSAIA];
35 default:62 default:
36 break;63 break;
37 }64 }
@@ -50,24 +77,27 @@ static inline bool vcpu_has_ext(struct kvm_vcpu *vcpu, int ext) · 12 unmodified lines
50 unsigned long value;77 unsigned long value;
51 78
52 ret = __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(ext), &value);79 ret = __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(ext), &value);
53 if (ret) {80 return (ret) ? false : !!value;
54 printf("Failed to get ext %d", ext);
55 return false;
56 }
57
58 return !!value;
59}81}
60 82
61void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)83void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)
62{84{
85 unsigned long isa_ext_state[KVM_RISCV_ISA_EXT_MAX] = { 0 };
63 struct vcpu_reg_sublist *s;86 struct vcpu_reg_sublist *s;
87 int rc;
88
89 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++)
90 __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(i), &isa_ext_state[i]);
64 91
65 /*92 /*
66 * Disable all extensions which were enabled by default93 * Disable all extensions which were enabled by default
67 * if they were available in the risc-v host.94 * if they were available in the risc-v host.
68 */95 */
69 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++)96 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
70 __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(i), 0);97 rc = __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(i), 0);
98 if (rc && isa_ext_state[i])
99 isa_ext_cant_disable[i] = true;
100 }
71 101
72 for_each_sublist(c, s) {102 for_each_sublist(c, s) {
73 if (!s->feature)103 if (!s->feature)
@@ -506,10 +536,6 @@ static __u64 base_regs[] = { · 432 unmodified lines
506 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(time),536 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(time),
507 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(compare),537 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(compare),
508 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(state),538 KVM_REG_RISCV | KVM_REG_SIZE_U64 | KVM_REG_RISCV_TIMER | KVM_REG_RISCV_TIMER_REG(state),
509 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_A,
510 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_C,
511 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_I,
512 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_ISA_EXT | KVM_RISCV_ISA_EXT_M,
513 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_V01,539 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_V01,
514 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_TIME,540 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_TIME,
515 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_IPI,541 KVM_REG_RISCV | KVM_REG_SIZE_ULONG | KVM_REG_RISCV_SBI_EXT | KVM_REG_RISCV_SBI_SINGLE | KVM_RISCV_SBI_EXT_IPI,