History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull kvm fixes from Paolo Bonzini: "s390: - Fix S390_USER_OPEREXEC so it can now be enabled regardless of other unrelated capabilities - Fix handling of the _PAGE_UNUSED pte bit that could lead to guest memory corruption in some scenarios - A bunch of misc gmap fixes (locking, behaviour under memory pressure) - Fix CMMA dirty tracking x86: - Tidy up some WARN_ON() and BUG_ON(), replacing them with WARN_ON_ONCE() or KVM_BUG_ON(). All of these have obviously never triggered, or somebody would have been annoyed earlier, but still... - Fix missing interrupt due to stale CR8 intercept - Add a statistic that can come in handy to debug leaks as well as the vulnerability to a class of recently-discovered issues - Do not ask arch/x86/kernel to export default_cpu_present_to_apicid() just for KVM" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (22 commits) x86/apic: KVM: Use cpu_physical_id() to get APIC ID of running vCPU for AVIC KVM: x86/mmu: Expose number of shadow MMU shadow pages as a stat KVM: x86: Unconditionally recompute CR8 intercept on PPR update KVM: VMX: Grab vmcs12 on CR8 interception update iff vCPU is in guest mode KVM: x86: WARN (once) if RTC pending EOI tracking goes off the rails KVM: x86: WARN and fail kvm_set_irq() if a PIC or I/O APIC vector is invalid KVM: x86: Bug the VM, not the kernel, if the ISR count {under,over}flows KVM: x86/mmu: Bug the VM, not the host kernel, if KVM write-protects upper SPTEs KVM: x86: Replace BUG_ON() with WARN_ON_ONCE() on "bad" nested GPA translation KVM: Replace guest-triggerable BUG_ON() in ioeventfd datamatch with get_unaligned() KVM: s390: Return failure in case of failure in kvm_s390_set_cmma_bits() KVM: s390: selftests: Fix cmma selftest KVM: s390: Fix cmma dirty tracking KVM: s390: Fix locking in kvm_s390_set_mem_control() KVM: s390: Fix handle_{sske,pfmf} under memory pressure KVM: s390: Fix code typo in gmap_protect_asce_top_level() KVM: s390: Do not set special large pages dirty KVM: s390: Fix dat_peek_cmma() overflow s390/mm: Fix handling of _PAGE_UNUSED pte bit KVM: s390: Fix typo in UCONTROL documentation ...

Linus Torvalds committed 4 months ago

Commit c75597c2 parents a142da0 098e32c25 files · +265 −85Browse files

Unified · Split

25 files
  1. Documentation/virt/kvm/api.rst +1 −1
  2. arch/s390/include/asm/pgtable.h +2 −2
  3. arch/s390/kvm/dat.c +4 −1
  4. arch/s390/kvm/gmap.c +29 −6
  5. arch/s390/kvm/gmap.h +11 −1
  6. arch/s390/kvm/kvm-s390.c +48 −23
  7. arch/s390/kvm/priv.c +7 −3
  8. arch/s390/mm/gmap_helpers.c +2 −1
  9. arch/x86/include/asm/kvm_host.h +1 −11
  10. arch/x86/kernel/apic/apic_common.c +0 −1
  11. arch/x86/kvm/i8259.c +2 −1
  12. arch/x86/kvm/ioapic.c +4 −3
  13. arch/x86/kvm/lapic.c +4 −2
  14. arch/x86/kvm/mmu/mmu.c +7 −7
  15. arch/x86/kvm/mmu/mmutrace.h +1 −1
  16. arch/x86/kvm/mmu/tdp_mmu.c +5 −3
  17. arch/x86/kvm/svm/avic.c +3 −3
  18. arch/x86/kvm/svm/nested.c +2 −1
  19. arch/x86/kvm/vmx/nested.c +2 −1
  20. arch/x86/kvm/vmx/vmx.c +1 −2
  21. arch/x86/kvm/x86.c +2 −4
  22. tools/testing/selftests/kvm/include/s390/facility.h +6 −0
  23. tools/testing/selftests/kvm/s390/cmma_test.c +6 −0
  24. tools/testing/selftests/kvm/s390/user_operexec.c +110 −0
  25. virt/kvm/eventfd.c +5 −7
Documentation/virt/kvm/api.rst+1−1
@@ -6840,7 +6840,7 @@ s390 specific. · 6839 unmodified lines
68406840 } s390_ucontrol;
68416841
68426842 s390 specific. A page fault has occurred for a user controlled virtual
6843−machine (KVM_VM_S390_UNCONTROL) on its host page table that cannot be
6843+machine (KVM_VM_S390_UCONTROL) on its host page table that cannot be
68446844 resolved by the kernel.
68456845 The program code and the translation exception code that were placed
68466846 in the cpu's lowcore are presented here as defined by the z Architecture
arch/s390/include/asm/pgtable.h+2−2
@@ -980,6 +980,8 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) · 979 unmodified lines
980980
981981 static inline void set_pte(pte_t *ptep, pte_t pte)
982982 {
983+ if (pte_present(pte))
984+ pte = clear_pte_bit(pte, __pgprot(_PAGE_UNUSED));
983985 WRITE_ONCE(*ptep, pte);
984986 }
985987
@@ -1359,8 +1361,6 @@ pgprot_t pgprot_writecombine(pgprot_t prot); · 373 unmodified lines
13591361 static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
13601362 pte_t *ptep, pte_t entry, unsigned int nr)
13611363 {
1362− if (pte_present(entry))
1363− entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
13641364 page_table_check_ptes_set(mm, addr, ptep, entry, nr);
13651365 for (;;) {
13661366 set_pte(ptep, entry);
arch/s390/kvm/dat.c+4−1
@@ -1209,7 +1209,7 @@ int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values) · 1208 unmodified lines
12091209 int rc;
12101210
12111211 rc = _dat_walk_gfn_range(start, start + *count, asce, &ops, DAT_WALK_DEFAULT, &state);
1212− *count = state.end - start;
1212+ *count = state.end >= start ? state.end - start : 0;
12131213 /* Return success if at least one value was saved, otherwise an error. */
12141214 return (rc == -EFAULT && *count > 0) ? 0 : rc;
12151215 }
@@ -1253,6 +1253,9 @@ int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, · 37 unmodified lines
12531253 };
12541254
12551255 _dat_walk_gfn_range(*start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, &state);
1256+ /* If no dirty pages were found, wrap around and continue searching */
1257+ if (*start && state.start == -1)
1258+ _dat_walk_gfn_range(0, *start, asce, &ops, DAT_WALK_IGN_HOLES, &state);
12561259
12571260 if (state.start == -1) {
12581261 *count = 0;
arch/s390/kvm/gmap.c+29−6
@@ -332,7 +332,7 @@ static long gmap_clear_young_crste(union crste *crstep, gfn_t gfn, gfn_t end, st · 331 unmodified lines
332332 new.h.i = 1;
333333 new.s.fc1.y = 0;
334334 new.s.fc1.prefix_notif = 0;
335− if (new.s.fc1.d || !new.h.p)
335+ if ((new.s.fc1.d || !new.h.p) && !new.s.fc1.s)
336336 folio_set_dirty(phys_to_folio(crste_origin_large(crste)));
337337 new.s.fc1.d = 0;
338338 new.h.p = 1;
@@ -1098,23 +1098,46 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf · 759 unmodified lines
10981098 return 0;
10991099 }
11001100
1101+static long __set_cmma_clean_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
1102+{
1103+ union pgste pgste;
1104+
1105+ pgste = pgste_get_lock(ptep);
1106+ pgste.cmma_d = 0;
1107+ pgste_set_unlock(ptep, pgste);
1108+
1109+ if (need_resched())
1110+ return next;
1111+ return 0;
1112+}
1113+
11011114 static long __set_cmma_dirty_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
11021115 {
1103− __atomic64_or(PGSTE_CMMA_D_BIT, &pgste_of(ptep)->val);
1116+ union pgste pgste;
1117+
1118+ pgste = pgste_get_lock(ptep);
1119+ if (!pgste.cmma_d)
1120+ atomic64_inc(walk->priv);
1121+ pgste.cmma_d = 1;
1122+ pgste_set_unlock(ptep, pgste);
1123+
11041124 if (need_resched())
11051125 return next;
11061126 return 0;
11071127 }
11081128
1109−void gmap_set_cmma_all_dirty(struct gmap *gmap)
1129+void _gmap_set_cmma_all(struct gmap *gmap, bool dirty)
11101130 {
1111− const struct dat_walk_ops ops = { .pte_entry = __set_cmma_dirty_pte, };
1131+ const struct dat_walk_ops ops = {
1132+ .pte_entry = dirty ? __set_cmma_dirty_pte : __set_cmma_clean_pte,
1133+ };
11121134 gfn_t gfn = 0;
11131135
11141136 do {
11151137 scoped_guard(read_lock, &gmap->kvm->mmu_lock)
11161138 gfn = _dat_walk_gfn_range(gfn, asce_end(gmap->asce), gmap->asce, &ops,
1117− DAT_WALK_IGN_HOLES, NULL);
1139+ DAT_WALK_IGN_HOLES,
1140+ &gmap->kvm->arch.cmma_dirty_pages);
11181141 cond_resched();
11191142 } while (gfn);
11201143 }
@@ -1287,7 +1310,7 @@ static int gmap_protect_asce_top_level(struct kvm_s390_mmu_cache *mc, struct gma · 166 unmodified lines
12871310 /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
12881311 smp_rmb();
12891312
1290− rc = kvm_s390_get_guest_pages(sg->kvm, context.f, asce.rsto, asce.dt + 1, false);
1313+ rc = kvm_s390_get_guest_pages(sg->kvm, context.f, asce.rsto, asce.tl + 1, false);
12911314 if (rc > 0)
12921315 rc = -EFAULT;
12931316 if (!rc)
arch/s390/kvm/gmap.h+11−1
@@ -104,7 +104,7 @@ int gmap_insert_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn · 103 unmodified lines
104104 gfn_t r_gfn, int level);
105105 int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
106106 kvm_pfn_t pfn, int level, bool wr);
107−void gmap_set_cmma_all_dirty(struct gmap *gmap);
107+void _gmap_set_cmma_all(struct gmap *gmap, bool dirty);
108108 void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn);
109109 struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *gmap,
110110 union asce asce, int edat_level);
@@ -198,6 +198,16 @@ static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union · 87 unmodified lines
198198 return !newpte.h.p || !newpte.s.pr;
199199 }
200200
201+static inline void gmap_set_cmma_all_dirty(struct gmap *gmap)
202+{
203+ _gmap_set_cmma_all(gmap, true);
204+}
205+
206+static inline void gmap_set_cmma_all_clean(struct gmap *gmap)
207+{
208+ _gmap_set_cmma_all(gmap, false);
209+}
210+
201211 static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte,
202212 union pgste pgste, gfn_t gfn, bool needs_lock)
203213 {
arch/s390/kvm/kvm-s390.c+48−23
@@ -1022,9 +1022,11 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att · 1021 unmodified lines
10221022 if (!kvm->arch.use_cmma)
10231023 break;
10241024
1025+ guard(mutex)(&kvm->lock);
10251026 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
10261027 do {
1027− start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
1028+ scoped_guard(read_lock, &kvm->mmu_lock)
1029+ start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
10281030 cond_resched();
10291031 } while (start_gfn);
10301032 ret = 0;
@@ -1217,13 +1219,13 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) · 186 unmodified lines
12171219
12181220 /*
12191221 * Must be called with kvm->srcu held to avoid races on memslots, and with
1220− * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1222+ * kvm->slots_lock to avoid races with ourselves, kvm_s390_vm_stop_migration(),
1223+ * and kvm_s390_get_cmma_bits().
12211224 */
12221225 static int kvm_s390_vm_start_migration(struct kvm *kvm)
12231226 {
12241227 struct kvm_memory_slot *ms;
12251228 struct kvm_memslots *slots;
1226− unsigned long ram_pages = 0;
12271229 int bkt;
12281230
12291231 /* migration mode already enabled */
@@ -1240,28 +1242,54 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) · 10 unmodified lines
12401242 kvm_for_each_memslot(ms, bkt, slots) {
12411243 if (!ms->dirty_bitmap)
12421244 return -EINVAL;
1243− ram_pages += ms->npages;
12441245 }
1245− /* mark all the pages as dirty */
1246− gmap_set_cmma_all_dirty(kvm->arch.gmap);
1247− atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1248− kvm->arch.migration_mode = 1;
1246+ /*
1247+ * Set the flag and let KVM handle ESSA manually, potentially setting
1248+ * the cmma_d bit in some PGSTEs and increasing cmma_dirty_pages.
1249+ * At this point cmma_dirty_pages is still 0, and all existing PGSTEs
1250+ * have their cmma_d bit set to 0.
1251+ * Any newly allocated page table has its entries marked as cmma-clean,
1252+ * which is fine because the CMMA values are not dirty.
1253+ */
1254+ WRITE_ONCE(kvm->arch.migration_mode, 1);
12491255 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1256+ /*
1257+ * Mark all PGSTEs as cmma-dirty, increasing cmma_dirty_pages as needed,
1258+ * but without double-counting pages that have become dirty on their own
1259+ * in the meantime.
1260+ * At this point some pages might have become dirty on their own already
1261+ * and cmma_dirty_pages might therefore be non-zero.
1262+ */
1263+ gmap_set_cmma_all_dirty(kvm->arch.gmap);
12501264 return 0;
12511265 }
12521266
12531267 /*
1254− * Must be called with kvm->slots_lock to avoid races with ourselves and
1255− * kvm_s390_vm_start_migration.
1268+ * Must be called with kvm->slots_lock to avoid races with ourselves,
1269+ * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits().
12561270 */
12571271 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
12581272 {
12591273 /* migration mode already disabled */
12601274 if (!kvm->arch.migration_mode)
12611275 return 0;
1262− kvm->arch.migration_mode = 0;
1276+ /*
1277+ * Unset the flag and propagate to all vCPUs. From now on the cmma_d
1278+ * bit will not be touched on any PGSTE.
1279+ * At this point cmma_dirty_pages is possibly non-zero, and thus some
1280+ * PGSTEs might have cmma_d set.
1281+ */
1282+ WRITE_ONCE(kvm->arch.migration_mode, 0);
12631283 if (kvm->arch.use_cmma)
12641284 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1285+ /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */
1286+ gmap_set_cmma_all_clean(kvm->arch.gmap);
1287+ atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
1288+ /*
1289+ * At this point the system has the expected state: migration_mode is 0,
1290+ * cmma_dirty_pages is 0, and all existing PGSTEs have their cmma_d bit
1291+ * set to 0.
1292+ */
12651293 return 0;
12661294 }
12671295
@@ -2317,8 +2345,8 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, · 1049 unmodified lines
23172345 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
23182346 const struct kvm_s390_cmma_log *args)
23192347 {
2320− struct kvm_s390_mmu_cache *mc;
2321− u8 *bits = NULL;
2348+ struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
2349+ u8 *bits __free(kvfree) = NULL;
23222350 int r = 0;
23232351
23242352 if (!kvm->arch.use_cmma)
@@ -2338,18 +2366,16 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, · 13 unmodified lines
23382366 return -ENOMEM;
23392367 bits = vmalloc(array_size(sizeof(*bits), args->count));
23402368 if (!bits)
2341− goto out;
2369+ return -ENOMEM;
23422370
23432371 r = copy_from_user(bits, (void __user *)args->values, args->count);
2344− if (r) {
2345− r = -EFAULT;
2346− goto out;
2347− }
2372+ if (r)
2373+ return -EFAULT;
23482374
23492375 do {
23502376 r = kvm_s390_mmu_cache_topup(mc);
23512377 if (r)
2352− break;
2378+ return r;
23532379 scoped_guard(read_lock, &kvm->mmu_lock) {
23542380 r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
23552381 args->count, args->mask, bits);
@@ -2357,9 +2383,7 @@ static int kvm_s390_set_cmma_bits(struct kvm *kvm, · 1 unmodified line
23572383 } while (r == -ENOMEM);
23582384
23592385 set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
2360−out:
2361− kvm_s390_free_mmu_cache(mc);
2362− vfree(bits);
2386+
23632387 return r;
23642388 }
23652389
@@ -3584,7 +3608,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) · 1218 unmodified lines
35843608 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
35853609 sca_add_vcpu(vcpu);
35863610 }
3587− if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3611+ if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
3612+ vcpu->kvm->arch.user_operexec)
35883613 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
35893614 }
35903615
arch/s390/kvm/priv.c+7−3
@@ -366,7 +366,9 @@ static int handle_sske(struct kvm_vcpu *vcpu) · 365 unmodified lines
366366 if (rc > 1)
367367 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
368368 if (rc == -ENOMEM) {
369− kvm_s390_mmu_cache_topup(vcpu->arch.mc);
369+ rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc);
370+ if (rc)
371+ return rc;
370372 continue;
371373 }
372374 if (rc < 0)
@@ -1122,7 +1124,9 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 749 unmodified lines
11221124 if (rc > 1)
11231125 return kvm_s390_inject_program_int(vcpu, rc);
11241126 if (rc == -ENOMEM) {
1125− kvm_s390_mmu_cache_topup(vcpu->arch.mc);
1127+ rc = kvm_s390_mmu_cache_topup(vcpu->arch.mc);
1128+ if (rc)
1129+ return rc;
11261130 continue;
11271131 }
11281132 if (rc < 0)
@@ -1232,7 +1236,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) · 103 unmodified lines
12321236 : ESSA_SET_STABLE_IF_RESIDENT))
12331237 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
12341238
1235− if (!vcpu->kvm->arch.migration_mode) {
1239+ if (!READ_ONCE(vcpu->kvm->arch.migration_mode)) {
12361240 /*
12371241 * CMMA is enabled in the KVM settings, but is disabled in
12381242 * the SIE block and in the mm_context, and we are not doing
arch/s390/mm/gmap_helpers.c+2−1
@@ -181,7 +181,8 @@ void gmap_helper_try_set_pte_unused(struct mm_struct *mm, unsigned long vmaddr) · 180 unmodified lines
181181 if (IS_ERR_OR_NULL(ptep))
182182 return;
183183
184− __atomic64_or(_PAGE_UNUSED, (long *)ptep);
184+ if (pte_present(*ptep))
185+ __atomic64_or(_PAGE_UNUSED, (long *)ptep);
185186 pte_unmap_unlock(ptep, ptl);
186187 }
187188 EXPORT_SYMBOL_GPL(gmap_helper_try_set_pte_unused);
arch/x86/include/asm/kvm_host.h+1−11
@@ -1434,7 +1434,6 @@ enum kvm_mmu_type { · 1433 unmodified lines
14341434 };
14351435
14361436 struct kvm_arch {
1437− unsigned long n_used_mmu_pages;
14381437 unsigned long n_requested_mmu_pages;
14391438 unsigned long n_max_mmu_pages;
14401439 unsigned int indirect_shadow_pages;
@@ -1700,6 +1699,7 @@ struct kvm_vm_stat { · 259 unmodified lines
17001699 u64 mmu_recycled;
17011700 u64 mmu_cache_miss;
17021701 u64 mmu_unsync;
1702+ u64 mmu_shadow_pages;
17031703 union {
17041704 struct {
17051705 atomic64_t pages_4k;
@@ -2525,16 +2525,6 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) · 819 unmodified lines
25252525 kvm_x86_call(vcpu_unblocking)(vcpu);
25262526 }
25272527
2528−static inline int kvm_cpu_get_apicid(int mps_cpu)
2529−{
2530−#ifdef CONFIG_X86_LOCAL_APIC
2531− return default_cpu_present_to_apicid(mps_cpu);
2532−#else
2533− WARN_ON_ONCE(1);
2534− return BAD_APICID;
2535−#endif
2536−}
2537−
25382528 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
25392529
25402530 #define KVM_CLOCK_VALID_FLAGS \
arch/x86/kernel/apic/apic_common.c+0−1
@@ -26,7 +26,6 @@ u32 default_cpu_present_to_apicid(int mps_cpu) · 25 unmodified lines
2626 else
2727 return BAD_APICID;
2828 }
29−EXPORT_SYMBOL_FOR_KVM(default_cpu_present_to_apicid);
3029
3130 /*
3231 * Set up the logical destination ID when the APIC operates in logical
arch/x86/kvm/i8259.c+2−1
@@ -194,7 +194,8 @@ int kvm_pic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, · 193 unmodified lines
194194 int irq = e->irqchip.pin;
195195 int ret, irq_level;
196196
197− BUG_ON(irq < 0 || irq >= PIC_NUM_PINS);
197+ if (WARN_ON_ONCE(irq < 0 || irq >= PIC_NUM_PINS))
198+ return -1;
198199
199200 pic_lock(s);
200201 irq_level = __kvm_irq_line_state(&s->irq_states[irq],
arch/x86/kvm/ioapic.c+4−3
@@ -84,7 +84,7 @@ static void kvm_rtc_eoi_tracking_restore_all(struct kvm_ioapic *ioapic); · 83 unmodified lines
8484
8585 static void rtc_status_pending_eoi_check_valid(struct kvm_ioapic *ioapic)
8686 {
87− if (WARN_ON(ioapic->rtc_status.pending_eoi < 0))
87+ if (WARN_ON_ONCE(ioapic->rtc_status.pending_eoi < 0))
8888 kvm_rtc_eoi_tracking_restore_all(ioapic);
8989 }
9090
@@ -484,7 +484,7 @@ static int ioapic_service(struct kvm_ioapic *ioapic, int irq, bool line_status) · 393 unmodified lines
484484 * ensures that it is only called if it is >= zero, namely
485485 * if rtc_irq_check_coalesced returns false).
486486 */
487− BUG_ON(ioapic->rtc_status.pending_eoi != 0);
487+ WARN_ON_ONCE(ioapic->rtc_status.pending_eoi);
488488 ret = __kvm_irq_delivery_to_apic(ioapic->kvm, NULL, &irqe,
489489 &ioapic->rtc_status);
490490 ioapic->rtc_status.pending_eoi = (ret < 0 ? 0 : ret);
@@ -504,7 +504,8 @@ int kvm_ioapic_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, · 13 unmodified lines
504504 int irq = e->irqchip.pin;
505505 int ret, irq_level;
506506
507− BUG_ON(irq < 0 || irq >= IOAPIC_NUM_PINS);
507+ if (WARN_ON_ONCE(irq < 0 || irq >= IOAPIC_NUM_PINS))
508+ return -1;
508509
509510 spin_lock(&ioapic->lock);
510511 irq_level = __kvm_irq_line_state(&ioapic->irq_states[irq],
arch/x86/kvm/lapic.c+4−2
@@ -767,7 +767,7 @@ static inline void apic_set_isr(int vec, struct kvm_lapic *apic) · 766 unmodified lines
767767 kvm_x86_call(hwapic_isr_update)(apic->vcpu, vec);
768768 else {
769769 ++apic->isr_count;
770− BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
770+ KVM_BUG_ON(apic->isr_count > MAX_APIC_VECTOR, apic->vcpu->kvm);
771771 /*
772772 * ISR (in service register) bit is set when injecting an interrupt.
773773 * The highest vector is injected. Thus the latest bit set matches
@@ -808,7 +808,7 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) · 34 unmodified lines
808808 kvm_x86_call(hwapic_isr_update)(apic->vcpu, apic_find_highest_isr(apic));
809809 else {
810810 --apic->isr_count;
811− BUG_ON(apic->isr_count < 0);
811+ KVM_BUG_ON(apic->isr_count < 0, apic->vcpu->kvm);
812812 apic->highest_isr_cache = -1;
813813 }
814814 }
@@ -980,6 +980,8 @@ static void apic_update_ppr(struct kvm_lapic *apic) · 165 unmodified lines
980980 if (__apic_update_ppr(apic, &ppr) &&
981981 apic_has_interrupt_for_ppr(apic, ppr) != -1)
982982 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
983+ else
984+ kvm_lapic_update_cr8_intercept(apic->vcpu);
983985 }
984986
985987 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
arch/x86/kvm/mmu/mmu.c+7−7
@@ -1802,13 +1802,13 @@ static void kvm_mmu_check_sptes_at_free(struct kvm_mmu_page *sp) · 1801 unmodified lines
18021802
18031803 static void kvm_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
18041804 {
1805− kvm->arch.n_used_mmu_pages++;
1805+ kvm->stat.mmu_shadow_pages++;
18061806 kvm_account_pgtable_pages((void *)sp->spt, +1);
18071807 }
18081808
18091809 static void kvm_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
18101810 {
1811− kvm->arch.n_used_mmu_pages--;
1811+ kvm->stat.mmu_shadow_pages--;
18121812 kvm_account_pgtable_pages((void *)sp->spt, -1);
18131813 }
18141814
@@ -2836,9 +2836,9 @@ restart: · 1021 unmodified lines
28362836
28372837 static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
28382838 {
2839− if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
2839+ if (kvm->arch.n_max_mmu_pages > kvm->stat.mmu_shadow_pages)
28402840 return kvm->arch.n_max_mmu_pages -
2841− kvm->arch.n_used_mmu_pages;
2841+ kvm->stat.mmu_shadow_pages;
28422842
28432843 return 0;
28442844 }
@@ -2874,11 +2874,11 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) · 29 unmodified lines
28742874 {
28752875 write_lock(&kvm->mmu_lock);
28762876
2877− if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2878− kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
2877+ if (kvm->stat.mmu_shadow_pages > goal_nr_mmu_pages) {
2878+ kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->stat.mmu_shadow_pages -
28792879 goal_nr_mmu_pages);
28802880
2881− goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2881+ goal_nr_mmu_pages = kvm->stat.mmu_shadow_pages;
28822882 }
28832883
28842884 kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
arch/x86/kvm/mmu/mmutrace.h+1−1
@@ -303,7 +303,7 @@ TRACE_EVENT( · 302 unmodified lines
303303
304304 TP_fast_assign(
305305 __entry->mmu_valid_gen = kvm->arch.mmu_valid_gen;
306− __entry->mmu_used_pages = kvm->arch.n_used_mmu_pages;
306+ __entry->mmu_used_pages = kvm->stat.mmu_shadow_pages;
307307 ),
308308
309309 TP_printk("kvm-mmu-valid-gen %u used_pages %x",
arch/x86/kvm/mmu/tdp_mmu.c+5−3
@@ -1410,9 +1410,10 @@ static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, · 1409 unmodified lines
14101410 u64 new_spte;
14111411 bool spte_set = false;
14121412
1413− rcu_read_lock();
1413+ if (KVM_BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL, kvm))
1414+ return false;
14141415
1415− BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
1416+ rcu_read_lock();
14161417
14171418 for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end) {
14181419 retry:
@@ -1844,7 +1845,8 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, · 425 unmodified lines
18441845 u64 new_spte;
18451846 bool spte_set = false;
18461847
1847− BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
1848+ if (KVM_BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL, kvm))
1849+ return false;
18481850
18491851 rcu_read_lock();
18501852
arch/x86/kvm/svm/avic.c+3−3
@@ -460,8 +460,8 @@ void avic_ring_doorbell(struct kvm_vcpu *vcpu) · 459 unmodified lines
460460 int cpu = READ_ONCE(vcpu->cpu);
461461
462462 if (cpu != get_cpu()) {
463− wrmsrq(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
464− trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu));
463+ wrmsrq(MSR_AMD64_SVM_AVIC_DOORBELL, cpu_physical_id(cpu));
464+ trace_kvm_avic_doorbell(vcpu->vcpu_id, cpu_physical_id(cpu));
465465 }
466466 put_cpu();
467467 }
@@ -1013,7 +1013,7 @@ static void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu, · 545 unmodified lines
10131013 enum avic_vcpu_action action)
10141014 {
10151015 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1016− int h_physical_id = kvm_cpu_get_apicid(cpu);
1016+ int h_physical_id = cpu_physical_id(cpu);
10171017 struct vcpu_svm *svm = to_svm(vcpu);
10181018 unsigned long flags;
10191019 u64 entry;
arch/x86/kvm/svm/nested.c+2−1
@@ -2152,7 +2152,8 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, · 2151 unmodified lines
21522152 struct vcpu_svm *svm = to_svm(vcpu);
21532153 struct kvm_mmu *mmu = vcpu->arch.mmu;
21542154
2155− BUG_ON(!mmu_is_nested(vcpu));
2155+ if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
2156+ return gpa;
21562157
21572158 /* Non-GMET walks are always user-walks */
21582159 if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET))
arch/x86/kvm/vmx/nested.c+2−1
@@ -7470,7 +7470,8 @@ static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, · 7469 unmodified lines
74707470 {
74717471 struct kvm_mmu *mmu = vcpu->arch.mmu;
74727472
7473− BUG_ON(!mmu_is_nested(vcpu));
7473+ if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
7474+ return gpa;
74747475
74757476 /*
74767477 * MBEC differentiates based on the effective U/S bit of
arch/x86/kvm/vmx/vmx.c+1−2
@@ -6873,11 +6873,10 @@ int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) · 6872 unmodified lines
68736873
68746874 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
68756875 {
6876− struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
68776876 int tpr_threshold;
68786877
68796878 if (is_guest_mode(vcpu) &&
6880− nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
6879+ nested_cpu_has(get_vmcs12(vcpu), CPU_BASED_TPR_SHADOW))
68816880 return;
68826881
68836882 guard(vmx_vmcs01)(vcpu);
arch/x86/kvm/x86.c+2−4
@@ -244,6 +244,7 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = { · 243 unmodified lines
244244 STATS_DESC_COUNTER(VM, mmu_recycled),
245245 STATS_DESC_COUNTER(VM, mmu_cache_miss),
246246 STATS_DESC_ICOUNTER(VM, mmu_unsync),
247+ STATS_DESC_ICOUNTER(VM, mmu_shadow_pages),
247248 STATS_DESC_ICOUNTER(VM, pages_4k),
248249 STATS_DESC_ICOUNTER(VM, pages_2m),
249250 STATS_DESC_ICOUNTER(VM, pages_1g),
@@ -5317,7 +5318,6 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, · 5067 unmodified lines
53175318 r = kvm_apic_set_state(vcpu, s);
53185319 if (r)
53195320 return r;
5320− kvm_lapic_update_cr8_intercept(vcpu);
53215321
53225322 return 0;
53235323 }
@@ -12418,8 +12418,6 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, · 7094 unmodified lines
1241812418 kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
1241912419 kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3);
1242012420
12421− kvm_set_cr8(vcpu, sregs->cr8);
12422−
1242312421 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
1242412422 kvm_x86_call(set_efer)(vcpu, sregs->efer);
1242512423
@@ -12448,7 +12446,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, · 22 unmodified lines
1244812446 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1244912447 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1245012448
12451− kvm_lapic_update_cr8_intercept(vcpu);
12449+ kvm_set_cr8(vcpu, sregs->cr8);
1245212450
1245312451 /* Older userspace won't unhalt the vcpu on reset. */
1245412452 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
tools/testing/selftests/kvm/include/s390/facility.h+6−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 #ifndef SELFTEST_KVM_FACILITY_H
1212 #define SELFTEST_KVM_FACILITY_H
1313
14+#include <linux/atomic.h>
1415 #include <linux/bitops.h>
1516
1617 /* alt_stfle_fac_list[16] + stfle_fac_list[16] */
@@ -19,6 +20,11 @@ · 2 unmodified lines
1920 extern u64 stfl_doublewords[NB_STFL_DOUBLEWORDS];
2021 extern bool stfle_flag;
2122
23+static inline bool clear_bit_inv(unsigned long nr, unsigned long *ptr)
24+{
25+ return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
26+}
27+
2228 static inline bool test_bit_inv(unsigned long nr, const unsigned long *ptr)
2329 {
2430 return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
tools/testing/selftests/kvm/s390/cmma_test.c+6−0
@@ -34,16 +34,22 @@ static char cmma_value_buf[MAIN_PAGE_COUNT + TEST_DATA_PAGE_COUNT]; · 33 unmodified lines
3434 /**
3535 * Dirty CMMA attributes of exactly one page in the TEST_DATA memslot,
3636 * so use_cmma goes on and the CMMA related ioctls do something.
37+ * Touch the page at offset 1M inside TEST_DATA to make sure its page
38+ * tables are allocated in the host.
3739 */
3840 static void guest_do_one_essa(void)
3941 {
4042 asm volatile(
4143 /* load TEST_DATA_START_GFN into r1 */
44+ " xgr 1,1\n"
4245 " llilf 1,%[start_gfn]\n"
4346 /* calculate the address from the gfn */
4447 " sllg 1,1,12(0)\n"
4548 /* set the first page in TEST_DATA memslot to STABLE */
4649 " .insn rrf,0xb9ab0000,2,1,1,0\n"
50+ " agfi 1,0x100000\n"
51+ /* also touch the first page of the second MB of TEST_DATA */
52+ " .insn rrf,0xb9ab0000,2,1,1,0\n"
4753 /* hypercall */
4854 " diag 0,0,0x501\n"
4955 "0: j 0b"
tools/testing/selftests/kvm/s390/user_operexec.c+110−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 * Authors:
77 * Janosch Frank <frankja@linux.ibm.com>
88 */
9+#include "facility.h"
910 #include "kselftest.h"
1011 #include "kvm_util.h"
1112 #include "test_util.h"
@@ -109,6 +110,111 @@ static void test_user_operexec_combined(void) · 97 unmodified lines
109110 kvm_vm_free(vm);
110111 }
111112
113+static struct kvm_vm *create_vm_without_sthyi(void)
114+{
115+ struct kvm_s390_vm_cpu_processor info;
116+ struct kvm_vm *vm;
117+
118+ vm = vm_create(1);
119+
120+ kvm_device_attr_get(vm->fd, KVM_S390_VM_CPU_MODEL,
121+ KVM_S390_VM_CPU_PROCESSOR, &info);
122+
123+ clear_bit_inv(74, (unsigned long *)&info.fac_list);
124+ kvm_device_attr_set(vm->fd, KVM_S390_VM_CPU_MODEL,
125+ KVM_S390_VM_CPU_PROCESSOR, &info);
126+
127+ return vm;
128+}
129+
130+static void test_user_instr0_no_stfle_74(void)
131+{
132+ struct kvm_vcpu *vcpu;
133+ struct kvm_vm *vm;
134+ int rc;
135+
136+ vm = create_vm_without_sthyi();
137+
138+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
139+ TEST_ASSERT_EQ(0, rc);
140+
141+ vcpu = vm_vcpu_add(vm, 0, guest_code_instr0);
142+
143+ vcpu_run(vcpu);
144+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
145+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
146+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0000);
147+
148+ kvm_vm_free(vm);
149+}
150+
151+static void test_user_operexec_no_stfle_74(void)
152+{
153+ struct kvm_vcpu *vcpu;
154+ struct kvm_vm *vm;
155+ int rc;
156+
157+ vm = create_vm_without_sthyi();
158+
159+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
160+ TEST_ASSERT_EQ(0, rc);
161+
162+ vcpu = vm_vcpu_add(vm, 0, guest_code_user_operexec);
163+
164+ vcpu_run(vcpu);
165+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
166+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
167+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807);
168+
169+ kvm_vm_free(vm);
170+}
171+
172+static void test_instr0_combined_no_stfle_74(void)
173+{
174+ struct kvm_vcpu *vcpu;
175+ struct kvm_vm *vm;
176+ int rc;
177+
178+ vm = create_vm_without_sthyi();
179+
180+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
181+ TEST_ASSERT_EQ(0, rc);
182+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
183+ TEST_ASSERT_EQ(0, rc);
184+
185+ vcpu = vm_vcpu_add(vm, 0, guest_code_instr0);
186+
187+ vcpu_run(vcpu);
188+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
189+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
190+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0000);
191+
192+ kvm_vm_free(vm);
193+}
194+
195+static void test_operexec_combined_no_stfle_74(void)
196+{
197+ struct kvm_vcpu *vcpu;
198+ struct kvm_vm *vm;
199+ int rc;
200+
201+ vm = create_vm_without_sthyi();
202+
203+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_INSTR0, 0);
204+ TEST_ASSERT_EQ(0, rc);
205+ rc = __vm_enable_cap(vm, KVM_CAP_S390_USER_OPEREXEC, 0);
206+ TEST_ASSERT_EQ(0, rc);
207+
208+ vcpu = vm_vcpu_add(vm, 0, guest_code_user_operexec);
209+
210+ vcpu_run(vcpu);
211+ TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_S390_SIEIC);
212+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.icptcode, ICPT_OPEREXC);
213+ TEST_ASSERT_EQ(vcpu->run->s390_sieic.ipa, 0x0807);
214+
215+ kvm_vm_free(vm);
216+}
217+
112218 /*
113219 * Run all tests above.
114220 *
@@ -122,6 +228,10 @@ static struct testdef { · 7 unmodified lines
122228 { "instr0", test_user_instr0 },
123229 { "operexec", test_user_operexec },
124230 { "operexec_combined", test_user_operexec_combined},
231+ { "instr0_no_stfle_74", test_user_instr0_no_stfle_74 },
232+ { "instr0_combined_no_stfle_74", test_instr0_combined_no_stfle_74 },
233+ { "operexec_combined_no_stfle_74", test_operexec_combined_no_stfle_74 },
234+ { "operexec_no_stfle_74", test_user_operexec_no_stfle_74 },
125235 };
126236
127237 int main(int argc, char *argv[])
virt/kvm/eventfd.c+5−7
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/slab.h>
2525 #include <linux/seqlock.h>
2626 #include <linux/irqbypass.h>
27+#include <linux/unaligned.h>
2728 #include <trace/events/kvm.h>
2829
2930 #include <kvm/iodev.h>
@@ -779,21 +780,18 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) · 749 unmodified lines
779780 return true;
780781
781782 /* otherwise, we have to actually compare the data */
782−
783− BUG_ON(!IS_ALIGNED((unsigned long)val, len));
784−
785783 switch (len) {
786784 case 1:
787− _val = *(u8 *)val;
785+ _val = get_unaligned((u8 *)val);
788786 break;
789787 case 2:
790− _val = *(u16 *)val;
788+ _val = get_unaligned((u16 *)val);
791789 break;
792790 case 4:
793− _val = *(u32 *)val;
791+ _val = get_unaligned((u32 *)val);
794792 break;
795793 case 8:
796− _val = *(u64 *)val;
794+ _val = get_unaligned((u64 *)val);
797795 break;
798796 default:
799797 return false;