Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull kvm fixes from Paolo Bonzini: "ARM: - Fix EL2 Stage-1 MMIO mappings where a random address was used - Fix SMCCC function number comparison when the SVE hint is set RISC-V: - Fix KVM_GET_REG_LIST API for ISA_EXT registers - Fix reading ISA_EXT register of a missing extension - Fix ISA_EXT register handling in get-reg-list test - Fix filtering of AIA registers in get-reg-list test x86: - Fixes for TSC_AUX virtualization - Stop zapping page tables asynchronously, since we don't zap them as often as before" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: KVM: SVM: Do not use user return MSR support for virtualized TSC_AUX KVM: SVM: Fix TSC_AUX virtualization setup KVM: SVM: INTERCEPT_RDTSCP is never intercepted anyway KVM: x86/mmu: Stop zapping invalidated TDP MMU roots asynchronously KVM: x86/mmu: Do not filter address spaces in for_each_tdp_mmu_root_yield_safe() KVM: x86/mmu: Open code leaf invalidation from mmu_notifier KVM: riscv: selftests: Selectively filter-out AIA registers KVM: riscv: selftests: Fix ISA_EXT register handling in get-reg-list RISC-V: KVM: Fix riscv_vcpu_get_isa_ext_single() for missing extensions RISC-V: KVM: Fix KVM_GET_REG_LIST API for ISA_EXT registers KVM: selftests: Assert that vasprintf() is successful KVM: arm64: nvhe: Ignore SVE hint in SMCCC function ID KVM: arm64: Properly return allocated EL2 VA from hyp_alloc_private_va_range()

Linus Torvalds committed 3 years ago

Commit 8a511e72 parents 5edc6bb 5804c1920 files · +209 −161Browse files

Unified · Split

20 files

arch/arm64/include/asm/kvm_hyp.h +1 −1
arch/arm64/kvm/hyp/include/nvhe/ffa.h +1 −1
arch/arm64/kvm/hyp/nvhe/ffa.c +1 −2
arch/arm64/kvm/hyp/nvhe/hyp-init.S +1 −0
arch/arm64/kvm/hyp/nvhe/hyp-main.c +6 −2
arch/arm64/kvm/hyp/nvhe/psci-relay.c +1 −2
arch/arm64/kvm/mmu.c +3 −0
arch/riscv/kvm/vcpu_onereg.c +5 −2
arch/x86/include/asm/kvm_host.h +1 −2
arch/x86/kvm/mmu/mmu.c +5 −16
arch/x86/kvm/mmu/mmu_internal.h +7 −8
arch/x86/kvm/mmu/tdp_mmu.c +67 −85
arch/x86/kvm/mmu/tdp_mmu.h +2 −3
arch/x86/kvm/svm/sev.c +26 −8
arch/x86/kvm/svm/svm.c +35 −8
arch/x86/kvm/svm/svm.h +1 −0
arch/x86/kvm/x86.c +1 −4
include/linux/arm-smccc.h +2 −0
tools/testing/selftests/kvm/lib/test_util.c +1 −1
tools/testing/selftests/kvm/riscv/get-reg-list.c +42 −16

arch/arm64/include/asm/kvm_hyp.h+1−1

@@ -118,7 +118,7 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu); · 117 unmodified lines
118	118
119	119	u64 __guest_enter(struct kvm_vcpu *vcpu);
120	120
121		−bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt);
	121	+bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id);
122	122
123	123	#ifdef __KVM_NVHE_HYPERVISOR__
124	124	void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,

arch/arm64/kvm/hyp/include/nvhe/ffa.h+1−1

@@ -12,6 +12,6 @@ · 11 unmodified lines
12	12	#define FFA_MAX_FUNC_NUM 0x7F
13	13
14	14	int hyp_ffa_init(void *pages);
15		−bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt);
	15	+bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id);
16	16
17	17	#endif /* __KVM_HYP_FFA_H */

arch/arm64/kvm/hyp/nvhe/ffa.c+1−2

@@ -634,9 +634,8 @@ out_handled: · 633 unmodified lines
634	634	return true;
635	635	}
636	636
637		−bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt)
	637	+bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
638	638	{
639		− DECLARE_REG(u64, func_id, host_ctxt, 0);
640	639	struct arm_smccc_res res;
641	640
642	641	/*

arch/arm64/kvm/hyp/nvhe/hyp-init.S+1−0

@@ -57,6 +57,7 @@ __do_hyp_init: · 56 unmodified lines
57	57	cmp x0, #HVC_STUB_HCALL_NR
58	58	b.lo __kvm_handle_stub_hvc
59	59
	60	+ bic x0, x0, #ARM_SMCCC_CALL_HINTS
60	61	mov x3, #KVM_HOST_SMCCC_FUNC(__kvm_hyp_init)
61	62	cmp x0, x3
62	63	b.eq 1f

arch/arm64/kvm/hyp/nvhe/hyp-main.c+6−2

@@ -368,6 +368,7 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) · 367 unmodified lines
368	368	if (static_branch_unlikely(&kvm_protected_mode_initialized))
369	369	hcall_min = __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize;
370	370
	371	+ id &= ~ARM_SMCCC_CALL_HINTS;
371	372	id -= KVM_HOST_SMCCC_ID(0);
372	373
373	374	if (unlikely(id < hcall_min \|\| id >= ARRAY_SIZE(host_hcall)))
@@ -392,11 +393,14 @@ static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt) · 18 unmodified lines
392	393
393	394	static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
394	395	{
	396	+ DECLARE_REG(u64, func_id, host_ctxt, 0);
395	397	bool handled;
396	398
397		− handled = kvm_host_psci_handler(host_ctxt);
	399	+ func_id &= ~ARM_SMCCC_CALL_HINTS;
	400	+
	401	+ handled = kvm_host_psci_handler(host_ctxt, func_id);
398	402	if (!handled)
399		− handled = kvm_host_ffa_handler(host_ctxt);
	403	+ handled = kvm_host_ffa_handler(host_ctxt, func_id);
400	404	if (!handled)
401	405	default_host_smc_handler(host_ctxt);
402	406

arch/arm64/kvm/hyp/nvhe/psci-relay.c+1−2

@@ -273,9 +273,8 @@ static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ · 272 unmodified lines
273	273	}
274	274	}
275	275
276		−bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt)
	276	+bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
277	277	{
278		− DECLARE_REG(u64, func_id, host_ctxt, 0);
279	278	unsigned long ret;
280	279
281	280	switch (kvm_host_psci_config.version) {

arch/arm64/kvm/mmu.c+3−0

@@ -652,6 +652,9 @@ int hyp_alloc_private_va_range(size_t size, unsigned long *haddr) · 651 unmodified lines
652	652
653	653	mutex_unlock(&kvm_hyp_pgd_mutex);
654	654
	655	+ if (!ret)
	656	+ *haddr = base;
	657	+
655	658	return ret;
656	659	}
657	660

arch/riscv/kvm/vcpu_onereg.c+5−2

@@ -460,8 +460,11 @@ static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, · 459 unmodified lines
460	460	reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
461	461	return -ENOENT;
462	462
463		− *reg_val = 0;
464	463	host_isa_ext = kvm_isa_ext_arr[reg_num];
	464	+ if (!__riscv_isa_extension_available(NULL, host_isa_ext))
	465	+ return -ENOENT;
	466	+
	467	+ *reg_val = 0;
465	468	if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
466	469	reg_val = 1; / Mark the given extension as available */
467	470
@@ -842,7 +845,7 @@ static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, · 374 unmodified lines
842	845	u64 reg = KVM_REG_RISCV \| size \| KVM_REG_RISCV_ISA_EXT \| i;
843	846
844	847	isa_ext = kvm_isa_ext_arr[i];
845		− if (!__riscv_isa_extension_available(vcpu->arch.isa, isa_ext))
	848	+ if (!__riscv_isa_extension_available(NULL, isa_ext))
846	849	continue;
847	850
848	851	if (uindices) {

arch/x86/include/asm/kvm_host.h+1−2

@@ -1419,7 +1419,6 @@ struct kvm_arch { · 1418 unmodified lines
1419	1419	* the thread holds the MMU lock in write mode.
1420	1420	*/
1421	1421	spinlock_t tdp_mmu_pages_lock;
1422		− struct workqueue_struct *tdp_mmu_zap_wq;
1423	1422	#endif /* CONFIG_X86_64 */
1424	1423
1425	1424	/*
@@ -1835,7 +1834,7 @@ void kvm_mmu_vendor_module_exit(void); · 409 unmodified lines
1835	1834
1836	1835	void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
1837	1836	int kvm_mmu_create(struct kvm_vcpu *vcpu);
1838		−int kvm_mmu_init_vm(struct kvm *kvm);
	1837	+void kvm_mmu_init_vm(struct kvm *kvm);
1839	1838	void kvm_mmu_uninit_vm(struct kvm *kvm);
1840	1839
1841	1840	void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);

arch/x86/kvm/mmu/mmu.c+5−16

@@ -6167,20 +6167,15 @@ static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) · 6166 unmodified lines
6167	6167	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
6168	6168	}
6169	6169
6170		−int kvm_mmu_init_vm(struct kvm *kvm)
	6170	+void kvm_mmu_init_vm(struct kvm *kvm)
6171	6171	{
6172		− int r;
6173		−
6174	6172	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
6175	6173	INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages);
6176	6174	INIT_LIST_HEAD(&kvm->arch.possible_nx_huge_pages);
6177	6175	spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);
6178	6176
6179		− if (tdp_mmu_enabled) {
6180		− r = kvm_mmu_init_tdp_mmu(kvm);
6181		− if (r < 0)
6182		− return r;
6183		− }
	6177	+ if (tdp_mmu_enabled)
	6178	+ kvm_mmu_init_tdp_mmu(kvm);
6184	6179
6185	6180	kvm->arch.split_page_header_cache.kmem_cache = mmu_page_header_cache;
6186	6181	kvm->arch.split_page_header_cache.gfp_zero = __GFP_ZERO;
@@ -6189,8 +6184,6 @@ int kvm_mmu_init_vm(struct kvm *kvm) · 2 unmodified lines
6189	6184
6190	6185	kvm->arch.split_desc_cache.kmem_cache = pte_list_desc_cache;
6191	6186	kvm->arch.split_desc_cache.gfp_zero = __GFP_ZERO;
6192		−
6193		− return 0;
6194	6187	}
6195	6188
6196	6189	static void mmu_free_vm_memory_caches(struct kvm *kvm)
@@ -6246,7 +6239,6 @@ static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_e · 49 unmodified lines
6246	6239	void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
6247	6240	{
6248	6241	bool flush;
6249		− int i;
6250	6242
6251	6243	if (WARN_ON_ONCE(gfn_end <= gfn_start))
6252	6244	return;
@@ -6257,11 +6249,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) · 4 unmodified lines
6257	6249
6258	6250	flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
6259	6251
6260		− if (tdp_mmu_enabled) {
6261		− for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
6262		− flush = kvm_tdp_mmu_zap_leafs(kvm, i, gfn_start,
6263		− gfn_end, true, flush);
6264		− }
	6252	+ if (tdp_mmu_enabled)
	6253	+ flush = kvm_tdp_mmu_zap_leafs(kvm, gfn_start, gfn_end, flush);
6265	6254
6266	6255	if (flush)
6267	6256	kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);

arch/x86/kvm/mmu/mmu_internal.h+7−8

@@ -58,7 +58,12 @@ struct kvm_mmu_page { · 57 unmodified lines
58	58
59	59	bool tdp_mmu_page;
60	60	bool unsync;
61		− u8 mmu_valid_gen;
	61	+ union {
	62	+ u8 mmu_valid_gen;
	63	+
	64	+ /* Only accessed under slots_lock. */
	65	+ bool tdp_mmu_scheduled_root_to_zap;
	66	+ };
62	67
63	68	/*
64	69	* The shadow page can't be replaced by an equivalent huge page
@@ -100,13 +105,7 @@ struct kvm_mmu_page { · 35 unmodified lines
100	105	struct kvm_rmap_head parent_ptes; /* rmap pointers to parent sptes */
101	106	tdp_ptep_t ptep;
102	107	};
103		− union {
104		− DECLARE_BITMAP(unsync_child_bitmap, 512);
105		− struct {
106		− struct work_struct tdp_mmu_async_work;
107		− void *tdp_mmu_async_data;
108		− };
109		− };
	108	+ DECLARE_BITMAP(unsync_child_bitmap, 512);
110	109
111	110	/*
112	111	* Tracks shadow pages that, if zapped, would allow KVM to create an NX

arch/x86/kvm/mmu/tdp_mmu.c+67−85

@@ -12,18 +12,10 @@ · 11 unmodified lines
12	12	#include <trace/events/kvm.h>
13	13
14	14	/* Initializes the TDP MMU for the VM, if enabled. */
15		−int kvm_mmu_init_tdp_mmu(struct kvm *kvm)
	15	+void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
16	16	{
17		− struct workqueue_struct *wq;
18		−
19		− wq = alloc_workqueue("kvm", WQ_UNBOUND\|WQ_MEM_RECLAIM\|WQ_CPU_INTENSIVE, 0);
20		− if (!wq)
21		− return -ENOMEM;
22		−
23	17	INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
24	18	spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
25		− kvm->arch.tdp_mmu_zap_wq = wq;
26		− return 1;
27	19	}
28	20
29	21	/* Arbitrarily returns true so that this may be used in if statements. */
@@ -46,20 +38,15 @@ void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) · 16 unmodified lines
46	38	* ultimately frees all roots.
47	39	*/
48	40	kvm_tdp_mmu_invalidate_all_roots(kvm);
49		−
50		− /*
51		− * Destroying a workqueue also first flushes the workqueue, i.e. no
52		− * need to invoke kvm_tdp_mmu_zap_invalidated_roots().
53		− */
54		− destroy_workqueue(kvm->arch.tdp_mmu_zap_wq);
	41	+ kvm_tdp_mmu_zap_invalidated_roots(kvm);
55	42
56	43	WARN_ON(atomic64_read(&kvm->arch.tdp_mmu_pages));
57	44	WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
58	45
59	46	/*
60	47	* Ensure that all the outstanding RCU callbacks to free shadow pages
61		− * can run before the VM is torn down. Work items on tdp_mmu_zap_wq
62		− * can call kvm_tdp_mmu_put_root and create new callbacks.
	48	+ * can run before the VM is torn down. Putting the last reference to
	49	+ * zapped roots will create new callbacks.
63	50	*/
64	51	rcu_barrier();
65	52	}
@@ -86,46 +73,6 @@ static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head) · 20 unmodified lines
86	73	tdp_mmu_free_sp(sp);
87	74	}
88	75
89		−static void tdp_mmu_zap_root(struct kvm kvm, struct kvm_mmu_page root,
90		− bool shared);
91		−
92		−static void tdp_mmu_zap_root_work(struct work_struct *work)
93		−{
94		− struct kvm_mmu_page *root = container_of(work, struct kvm_mmu_page,
95		− tdp_mmu_async_work);
96		− struct kvm *kvm = root->tdp_mmu_async_data;
97		−
98		− read_lock(&kvm->mmu_lock);
99		−
100		− /*
101		− * A TLB flush is not necessary as KVM performs a local TLB flush when
102		− * allocating a new root (see kvm_mmu_load()), and when migrating vCPU
103		− * to a different pCPU. Note, the local TLB flush on reuse also
104		− * invalidates any paging-structure-cache entries, i.e. TLB entries for
105		− * intermediate paging structures, that may be zapped, as such entries
106		− * are associated with the ASID on both VMX and SVM.
107		− */
108		− tdp_mmu_zap_root(kvm, root, true);
109		−
110		− /*
111		− * Drop the refcount using kvm_tdp_mmu_put_root() to test its logic for
112		− * avoiding an infinite loop. By design, the root is reachable while
113		− * it's being asynchronously zapped, thus a different task can put its
114		− * last reference, i.e. flowing through kvm_tdp_mmu_put_root() for an
115		− * asynchronously zapped root is unavoidable.
116		− */
117		− kvm_tdp_mmu_put_root(kvm, root, true);
118		−
119		− read_unlock(&kvm->mmu_lock);
120		−}
121		−
122		−static void tdp_mmu_schedule_zap_root(struct kvm kvm, struct kvm_mmu_page root)
123		−{
124		− root->tdp_mmu_async_data = kvm;
125		− INIT_WORK(&root->tdp_mmu_async_work, tdp_mmu_zap_root_work);
126		− queue_work(kvm->arch.tdp_mmu_zap_wq, &root->tdp_mmu_async_work);
127		−}
128		−
129	76	void kvm_tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root,
130	77	bool shared)
131	78	{
@@ -211,8 +158,12 @@ static struct kvm_mmu_page tdp_mmu_next_root(struct kvm kvm, · 79 unmodified lines
211	158	#define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
212	159	__for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)
213	160
214		−#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id) \
215		− __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, false, false)
	161	+#define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _shared) \
	162	+ for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, false); \
	163	+ _root; \
	164	+ _root = tdp_mmu_next_root(_kvm, _root, _shared, false)) \
	165	+ if (!kvm_lockdep_assert_mmu_lock_held(_kvm, _shared)) { \
	166	+ } else
216	167
217	168	/*
218	169	* Iterate over all TDP MMU roots. Requires that mmu_lock be held for write,
@@ -292,7 +243,7 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) · 73 unmodified lines
292	243	* by a memslot update or by the destruction of the VM. Initialize the
293	244	* refcount to two; one reference for the vCPU, and one reference for
294	245	* the TDP MMU itself, which is held until the root is invalidated and
295		− * is ultimately put by tdp_mmu_zap_root_work().
	246	+ * is ultimately put by kvm_tdp_mmu_zap_invalidated_roots().
296	247	*/
297	248	refcount_set(&root->tdp_mmu_root_count, 2);
298	249
@@ -877,13 +828,12 @@ static bool tdp_mmu_zap_leafs(struct kvm kvm, struct kvm_mmu_page root, · 578 unmodified lines
877	828	* true if a TLB flush is needed before releasing the MMU lock, i.e. if one or
878	829	* more SPTEs were zapped since the MMU lock was last acquired.
879	830	*/
880		−bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end,
881		− bool can_yield, bool flush)
	831	+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
882	832	{
883	833	struct kvm_mmu_page *root;
884	834
885		− for_each_tdp_mmu_root_yield_safe(kvm, root, as_id)
886		− flush = tdp_mmu_zap_leafs(kvm, root, start, end, can_yield, flush);
	835	+ for_each_tdp_mmu_root_yield_safe(kvm, root, false)
	836	+ flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
887	837
888	838	return flush;
889	839	}
@@ -891,7 +841,6 @@ bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start, gfn_t end, · 1 unmodified line
891	841	void kvm_tdp_mmu_zap_all(struct kvm *kvm)
892	842	{
893	843	struct kvm_mmu_page *root;
894		− int i;
895	844
896	845	/*
897	846	* Zap all roots, including invalid roots, as all SPTEs must be dropped
@@ -905,10 +854,8 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) · 7 unmodified lines
905	854	* is being destroyed or the userspace VMM has exited. In both cases,
906	855	* KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.
907	856	*/
908		− for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
909		− for_each_tdp_mmu_root_yield_safe(kvm, root, i)
910		− tdp_mmu_zap_root(kvm, root, false);
911		− }
	857	+ for_each_tdp_mmu_root_yield_safe(kvm, root, false)
	858	+ tdp_mmu_zap_root(kvm, root, false);
912	859	}
913	860
914	861	/*
@@ -917,18 +864,47 @@ void kvm_tdp_mmu_zap_all(struct kvm *kvm) · 2 unmodified lines
917	864	*/
918	865	void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)
919	866	{
920		− flush_workqueue(kvm->arch.tdp_mmu_zap_wq);
	867	+ struct kvm_mmu_page *root;
	868	+
	869	+ read_lock(&kvm->mmu_lock);
	870	+
	871	+ for_each_tdp_mmu_root_yield_safe(kvm, root, true) {
	872	+ if (!root->tdp_mmu_scheduled_root_to_zap)
	873	+ continue;
	874	+
	875	+ root->tdp_mmu_scheduled_root_to_zap = false;
	876	+ KVM_BUG_ON(!root->role.invalid, kvm);
	877	+
	878	+ /*
	879	+ * A TLB flush is not necessary as KVM performs a local TLB
	880	+ * flush when allocating a new root (see kvm_mmu_load()), and
	881	+ * when migrating a vCPU to a different pCPU. Note, the local
	882	+ * TLB flush on reuse also invalidates paging-structure-cache
	883	+ * entries, i.e. TLB entries for intermediate paging structures,
	884	+ * that may be zapped, as such entries are associated with the
	885	+ * ASID on both VMX and SVM.
	886	+ */
	887	+ tdp_mmu_zap_root(kvm, root, true);
	888	+
	889	+ /*
	890	+ * The referenced needs to be put after zapping the root, as
	891	+ * the root must be reachable by mmu_notifiers while it's being
	892	+ * zapped
	893	+ */
	894	+ kvm_tdp_mmu_put_root(kvm, root, true);
	895	+ }
	896	+
	897	+ read_unlock(&kvm->mmu_lock);
921	898	}
922	899
923	900	/*
924	901	* Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that
925	902	* is about to be zapped, e.g. in response to a memslots update. The actual
926		− * zapping is performed asynchronously. Using a separate workqueue makes it
927		− * easy to ensure that the destruction is performed before the "fast zap"
928		− * completes, without keeping a separate list of invalidated roots; the list is
929		− * effectively the list of work items in the workqueue.
	903	+ * zapping is done separately so that it happens with mmu_lock with read,
	904	+ * whereas invalidating roots must be done with mmu_lock held for write (unless
	905	+ * the VM is being destroyed).
930	906	*
931		− * Note, the asynchronous worker is gifted the TDP MMU's reference.
	907	+ * Note, kvm_tdp_mmu_zap_invalidated_roots() is gifted the TDP MMU's reference.
932	908	* See kvm_tdp_mmu_get_vcpu_root_hpa().
933	909	*/
934	910	void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm)
@@ -953,19 +929,20 @@ void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm) · 18 unmodified lines
953	929	/*
954	930	* As above, mmu_lock isn't held when destroying the VM! There can't
955	931	* be other references to @kvm, i.e. nothing else can invalidate roots
956		− * or be consuming roots, but walking the list of roots does need to be
957		− * guarded against roots being deleted by the asynchronous zap worker.
	932	+ * or get/put references to roots.
958	933	*/
959		− rcu_read_lock();
960		−
961		− list_for_each_entry_rcu(root, &kvm->arch.tdp_mmu_roots, link) {
	934	+ list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
	935	+ /*
	936	+ * Note, invalid roots can outlive a memslot update! Invalid
	937	+ * roots must be zapped before the memslot update completes,
	938	+ * but a different task can acquire a reference and keep the
	939	+ * root alive after its been zapped.
	940	+ */
962	941	if (!root->role.invalid) {
	942	+ root->tdp_mmu_scheduled_root_to_zap = true;
963	943	root->role.invalid = true;
964		− tdp_mmu_schedule_zap_root(kvm, root);
965	944	}
966	945	}
967		−
968		− rcu_read_unlock();
969	946	}
970	947
971	948	/*
@@ -1146,8 +1123,13 @@ retry: · 174 unmodified lines
1146	1123	bool kvm_tdp_mmu_unmap_gfn_range(struct kvm kvm, struct kvm_gfn_range range,
1147	1124	bool flush)
1148	1125	{
1149		− return kvm_tdp_mmu_zap_leafs(kvm, range->slot->as_id, range->start,
1150		− range->end, range->may_block, flush);
	1126	+ struct kvm_mmu_page *root;
	1127	+
	1128	+ __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false)
	1129	+ flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
	1130	+ range->may_block, flush);
	1131	+
	1132	+ return flush;
1151	1133	}
1152	1134
1153	1135	typedef bool (tdp_handler_t)(struct kvm kvm, struct tdp_iter *iter,

arch/x86/kvm/mmu/tdp_mmu.h+2−3

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	7
8	8	#include "spte.h"
9	9
10		−int kvm_mmu_init_tdp_mmu(struct kvm *kvm);
	10	+void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
11	11	void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
12	12
13	13	hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu);
@@ -20,8 +20,7 @@ __must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root) · 6 unmodified lines
20	20	void kvm_tdp_mmu_put_root(struct kvm kvm, struct kvm_mmu_page root,
21	21	bool shared);
22	22
23		−bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, int as_id, gfn_t start,
24		− gfn_t end, bool can_yield, bool flush);
	23	+bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
25	24	bool kvm_tdp_mmu_zap_sp(struct kvm kvm, struct kvm_mmu_page sp);
26	25	void kvm_tdp_mmu_zap_all(struct kvm *kvm);
27	26	void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm);

arch/x86/kvm/svm/sev.c+26−8

@@ -2962,6 +2962,32 @@ int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) · 2961 unmodified lines
2962	2962	count, in);
2963	2963	}
2964	2964
	2965	+static void sev_es_vcpu_after_set_cpuid(struct vcpu_svm *svm)
	2966	+{
	2967	+ struct kvm_vcpu *vcpu = &svm->vcpu;
	2968	+
	2969	+ if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
	2970	+ bool v_tsc_aux = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) \|\|
	2971	+ guest_cpuid_has(vcpu, X86_FEATURE_RDPID);
	2972	+
	2973	+ set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, v_tsc_aux, v_tsc_aux);
	2974	+ }
	2975	+}
	2976	+
	2977	+void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm)
	2978	+{
	2979	+ struct kvm_vcpu *vcpu = &svm->vcpu;
	2980	+ struct kvm_cpuid_entry2 *best;
	2981	+
	2982	+ /* For sev guests, the memory encryption bit is not reserved in CR3. */
	2983	+ best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
	2984	+ if (best)
	2985	+ vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
	2986	+
	2987	+ if (sev_es_guest(svm->vcpu.kvm))
	2988	+ sev_es_vcpu_after_set_cpuid(svm);
	2989	+}
	2990	+
2965	2991	static void sev_es_init_vmcb(struct vcpu_svm *svm)
2966	2992	{
2967	2993	struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -3024,14 +3050,6 @@ static void sev_es_init_vmcb(struct vcpu_svm *svm) · 56 unmodified lines
3024	3050	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
3025	3051	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
3026	3052	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
3027		−
3028		− if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) &&
3029		− (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP) \|\|
3030		− guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDPID))) {
3031		− set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, 1, 1);
3032		− if (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP))
3033		− svm_clr_intercept(svm, INTERCEPT_RDTSCP);
3034		− }
3035	3053	}
3036	3054
3037	3055	void sev_init_vmcb(struct vcpu_svm *svm)

arch/x86/kvm/svm/svm.c+35−8

@@ -683,6 +683,21 @@ static int svm_hardware_enable(void) · 682 unmodified lines
683	683
684	684	amd_pmu_enable_virt();
685	685
	686	+ /*
	687	+ * If TSC_AUX virtualization is supported, TSC_AUX becomes a swap type
	688	+ * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests.
	689	+ * Since Linux does not change the value of TSC_AUX once set, prime the
	690	+ * TSC_AUX field now to avoid a RDMSR on every vCPU run.
	691	+ */
	692	+ if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) {
	693	+ struct sev_es_save_area *hostsa;
	694	+ u32 msr_hi;
	695	+
	696	+ hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
	697	+
	698	+ rdmsr(MSR_TSC_AUX, hostsa->tsc_aux, msr_hi);
	699	+ }
	700	+
686	701	return 0;
687	702	}
688	703
@@ -1532,7 +1547,14 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 843 unmodified lines
1532	1547	if (tsc_scaling)
1533	1548	__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1534	1549
1535		− if (likely(tsc_aux_uret_slot >= 0))
	1550	+ /*
	1551	+ * TSC_AUX is always virtualized for SEV-ES guests when the feature is
	1552	+ * available. The user return MSR support is not required in this case
	1553	+ * because TSC_AUX is restored on #VMEXIT from the host save area
	1554	+ * (which has been initialized in svm_hardware_enable()).
	1555	+ */
	1556	+ if (likely(tsc_aux_uret_slot >= 0) &&
	1557	+ (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) \|\| !sev_es_guest(vcpu->kvm)))
1536	1558	kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1537	1559
1538	1560	svm->guest_state_loaded = true;
@@ -3086,6 +3108,16 @@ static int svm_set_msr(struct kvm_vcpu vcpu, struct msr_data msr) · 1547 unmodified lines
3086	3108	svm->sysenter_esp_hi = guest_cpuid_is_intel(vcpu) ? (data >> 32) : 0;
3087	3109	break;
3088	3110	case MSR_TSC_AUX:
	3111	+ /*
	3112	+ * TSC_AUX is always virtualized for SEV-ES guests when the
	3113	+ * feature is available. The user return MSR support is not
	3114	+ * required in this case because TSC_AUX is restored on #VMEXIT
	3115	+ * from the host save area (which has been initialized in
	3116	+ * svm_hardware_enable()).
	3117	+ */
	3118	+ if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
	3119	+ break;
	3120	+
3089	3121	/*
3090	3122	* TSC_AUX is usually changed only during boot and never read
3091	3123	* directly. Intercept TSC_AUX instead of exposing it to the
@@ -4284,7 +4316,6 @@ static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) · 1192 unmodified lines
4284	4316	static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4285	4317	{
4286	4318	struct vcpu_svm *svm = to_svm(vcpu);
4287		− struct kvm_cpuid_entry2 *best;
4288	4319
4289	4320	/*
4290	4321	* SVM doesn't provide a way to disable just XSAVES in the guest, KVM
@@ -4328,12 +4359,8 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) · 37 unmodified lines
4328	4359	set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0,
4329	4360	!!guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D));
4330	4361
4331		− /* For sev guests, the memory encryption bit is not reserved in CR3. */
4332		− if (sev_guest(vcpu->kvm)) {
4333		− best = kvm_find_cpuid_entry(vcpu, 0x8000001F);
4334		− if (best)
4335		− vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f));
4336		− }
	4362	+ if (sev_guest(vcpu->kvm))
	4363	+ sev_vcpu_after_set_cpuid(svm);
4337	4364
4338	4365	init_vmcb_after_set_cpuid(vcpu);
4339	4366	}

arch/x86/kvm/svm/svm.h+1−0

@@ -684,6 +684,7 @@ void __init sev_hardware_setup(void); · 683 unmodified lines
684	684	void sev_hardware_unsetup(void);
685	685	int sev_cpu_init(struct svm_cpu_data *sd);
686	686	void sev_init_vmcb(struct vcpu_svm *svm);
	687	+void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm);
687	688	void sev_free_vcpu(struct kvm_vcpu *vcpu);
688	689	int sev_handle_vmgexit(struct kvm_vcpu *vcpu);
689	690	int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in);

arch/x86/kvm/x86.c+1−4

@@ -12308,9 +12308,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 12307 unmodified lines
12308	12308	if (ret)
12309	12309	goto out;
12310	12310
12311		− ret = kvm_mmu_init_vm(kvm);
12312		− if (ret)
12313		− goto out_page_track;
	12311	+ kvm_mmu_init_vm(kvm);
12314	12312
12315	12313	ret = static_call(kvm_x86_vm_init)(kvm);
12316	12314	if (ret)
@@ -12355,7 +12353,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 38 unmodified lines
12355	12353
12356	12354	out_uninit_mmu:
12357	12355	kvm_mmu_uninit_vm(kvm);
12358		−out_page_track:
12359	12356	kvm_page_track_cleanup(kvm);
12360	12357	out:
12361	12358	return ret;

include/linux/arm-smccc.h+2−0

@@ -67,6 +67,8 @@ · 66 unmodified lines
67	67	#define ARM_SMCCC_VERSION_1_3 0x10003
68	68
69	69	#define ARM_SMCCC_1_3_SVE_HINT 0x10000
	70	+#define ARM_SMCCC_CALL_HINTS ARM_SMCCC_1_3_SVE_HINT
	71	+
70	72
71	73	#define ARM_SMCCC_VERSION_FUNC_ID \
72	74	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \

tools/testing/selftests/kvm/lib/test_util.c+1−1

@@ -387,7 +387,7 @@ char strdup_printf(const char fmt, ...) · 386 unmodified lines
387	387	char *str;
388	388
389	389	va_start(ap, fmt);
390		− vasprintf(&str, fmt, ap);
	390	+ TEST_ASSERT(vasprintf(&str, fmt, ap) >= 0, "vasprintf() failed");
391	391	va_end(ap);
392	392
393	393	return str;

tools/testing/selftests/kvm/riscv/get-reg-list.c+42−16

@@ -12,19 +12,37 @@ · 11 unmodified lines
12	12
13	13	#define REG_MASK (KVM_REG_ARCH_MASK \| KVM_REG_SIZE_MASK)
14	14
	15	+static bool isa_ext_cant_disable[KVM_RISCV_ISA_EXT_MAX];
	16	+
15	17	bool filter_reg(__u64 reg)
16	18	{
	19	+ switch (reg & ~REG_MASK) {
17	20	/*
18		− * Some ISA extensions are optional and not present on all host,
19		− * but they can't be disabled through ISA_EXT registers when present.
20		− * So, to make life easy, just filtering out these kind of registers.
	21	+ * Same set of ISA_EXT registers are not present on all host because
	22	+ * ISA_EXT registers are visible to the KVM user space based on the
	23	+ * ISA extensions available on the host. Also, disabling an ISA
	24	+ * extension using corresponding ISA_EXT register does not affect
	25	+ * the visibility of the ISA_EXT register itself.
	26	+ *
	27	+ * Based on above, we should filter-out all ISA_EXT registers.
21	28	*/
22		− switch (reg & ~REG_MASK) {
	29	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_A:
	30	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_C:
	31	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_D:
	32	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_F:
	33	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_H:
	34	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_I:
	35	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_M:
	36	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_SVPBMT:
23	37	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_SSTC:
24	38	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_SVINVAL:
25	39	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
	40	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZICBOM:
	41	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZICBOZ:
26	42	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZBB:
27	43	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_SSAIA:
	44	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_V:
	45	+ case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_SVNAPOT:
28	46	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZBA:
29	47	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZBS:
30	48	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZICNTR:
@@ -32,6 +50,15 @@ bool filter_reg(__u64 reg) · 1 unmodified line
32	50	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZIFENCEI:
33	51	case KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_ZIHPM:
34	52	return true;
	53	+ /* AIA registers are always available when Ssaia can't be disabled */
	54	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(siselect):
	55	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(iprio1):
	56	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(iprio2):
	57	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(sieh):
	58	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(siph):
	59	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(iprio1h):
	60	+ case KVM_REG_RISCV_CSR \| KVM_REG_RISCV_CSR_AIA \| KVM_REG_RISCV_CSR_AIA_REG(iprio2h):
	61	+ return isa_ext_cant_disable[KVM_RISCV_ISA_EXT_SSAIA];
35	62	default:
36	63	break;
37	64	}
@@ -50,24 +77,27 @@ static inline bool vcpu_has_ext(struct kvm_vcpu *vcpu, int ext) · 12 unmodified lines
50	77	unsigned long value;
51	78
52	79	ret = __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(ext), &value);
53		− if (ret) {
54		− printf("Failed to get ext %d", ext);
55		− return false;
56		− }
57		−
58		− return !!value;
	80	+ return (ret) ? false : !!value;
59	81	}
60	82
61	83	void finalize_vcpu(struct kvm_vcpu vcpu, struct vcpu_reg_list c)
62	84	{
	85	+ unsigned long isa_ext_state[KVM_RISCV_ISA_EXT_MAX] = { 0 };
63	86	struct vcpu_reg_sublist *s;
	87	+ int rc;
	88	+
	89	+ for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++)
	90	+ __vcpu_get_reg(vcpu, RISCV_ISA_EXT_REG(i), &isa_ext_state[i]);
64	91
65	92	/*
66	93	* Disable all extensions which were enabled by default
67	94	* if they were available in the risc-v host.
68	95	*/
69		− for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++)
70		− __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(i), 0);
	96	+ for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
	97	+ rc = __vcpu_set_reg(vcpu, RISCV_ISA_EXT_REG(i), 0);
	98	+ if (rc && isa_ext_state[i])
	99	+ isa_ext_cant_disable[i] = true;
	100	+ }
71	101
72	102	for_each_sublist(c, s) {
73	103	if (!s->feature)
@@ -506,10 +536,6 @@ static __u64 base_regs[] = { · 432 unmodified lines
506	536	KVM_REG_RISCV \| KVM_REG_SIZE_U64 \| KVM_REG_RISCV_TIMER \| KVM_REG_RISCV_TIMER_REG(time),
507	537	KVM_REG_RISCV \| KVM_REG_SIZE_U64 \| KVM_REG_RISCV_TIMER \| KVM_REG_RISCV_TIMER_REG(compare),
508	538	KVM_REG_RISCV \| KVM_REG_SIZE_U64 \| KVM_REG_RISCV_TIMER \| KVM_REG_RISCV_TIMER_REG(state),
509		− KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_A,
510		− KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_C,
511		− KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_I,
512		− KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_ISA_EXT \| KVM_RISCV_ISA_EXT_M,
513	539	KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_SBI_EXT \| KVM_REG_RISCV_SBI_SINGLE \| KVM_RISCV_SBI_EXT_V01,
514	540	KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_SBI_EXT \| KVM_REG_RISCV_SBI_SINGLE \| KVM_RISCV_SBI_EXT_TIME,
515	541	KVM_REG_RISCV \| KVM_REG_SIZE_ULONG \| KVM_REG_RISCV_SBI_EXT \| KVM_REG_RISCV_SBI_SINGLE \| KVM_RISCV_SBI_EXT_IPI,