History

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull KVM fixes from Paolo Bonzini: "ARM: - Avoid use of uninitialized memcache pointer in user_mem_abort() - Always set HCR_EL2.xMO bits when running in VHE, allowing interrupts to be taken while TGE=0 and fixing an ugly bug on AmpereOne that occurs when taking an interrupt while clearing the xMO bits (AC03_CPU_36) - Prevent VMMs from hiding support for AArch64 at any EL virtualized by KVM - Save/restore the host value for HCRX_EL2 instead of restoring an incorrect fixed value - Make host_stage2_set_owner_locked() check that the entire requested range is memory rather than just the first page RISC-V: - Add missing reset of smstateen CSRs x86: - Forcibly leave SMM on SHUTDOWN interception on AMD CPUs to avoid causing problems due to KVM stuffing INIT on SHUTDOWN (KVM needs to sanitize the VMCB as its state is undefined after SHUTDOWN, emulating INIT is the least awful choice). - Track the valid sync/dirty fields in kvm_run as a u64 to ensure KVM KVM doesn't goof a sanity check in the future. - Free obsolete roots when (re)loading the MMU to fix a bug where pre-faulting memory can get stuck due to always encountering a stale root. - When dumping GHCB state, use KVM's snapshot instead of the raw GHCB page to print state, so that KVM doesn't print stale/wrong information. - When changing memory attributes (e.g. shared <=> private), add potential hugepage ranges to the mmu_invalidate_range_{start,end} set so that KVM doesn't create a shared/private hugepage when the the corresponding attributes will become mixed (the attributes are commited *after* KVM finishes the invalidation). - Rework the SRSO mitigation to enable BP_SPEC_REDUCE only when KVM has at least one active VM. Effectively BP_SPEC_REDUCE when KVM is loaded led to very measurable performance regressions for non-KVM workloads" * tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: KVM: SVM: Set/clear SRSO's BP_SPEC_REDUCE on 0 <=> 1 VM count transitions KVM: arm64: Fix memory check in host_stage2_set_owner_locked() KVM: arm64: Kill HCRX_HOST_FLAGS KVM: arm64: Properly save/restore HCRX_EL2 KVM: arm64: selftest: Don't try to disable AArch64 support KVM: arm64: Prevent userspace from disabling AArch64 support at any virtualisable EL KVM: arm64: Force HCR_EL2.xMO to 1 at all times in VHE mode KVM: arm64: Fix uninitialized memcache pointer in user_mem_abort() KVM: x86/mmu: Prevent installing hugepages when mem attributes are changing KVM: SVM: Update dump_ghcb() to use the GHCB snapshot fields KVM: RISC-V: reset smstateen CSRs KVM: x86/mmu: Check and free obsolete roots in kvm_mmu_reload() KVM: x86: Check that the high 32bits are clear in kvm_arch_vcpu_ioctl_run() KVM: SVM: Forcibly leave SMM mode on SHUTDOWN interception

Linus Torvalds committed 1 year ago

Commit cd802e72 parents ecb9194 add203216 files · +200 −72Browse files

Unified · Split

16 files
  1. arch/arm64/include/asm/el2_setup.h +1 −1
  2. arch/arm64/include/asm/kvm_arm.h +1 −2
  3. arch/arm64/kvm/hyp/include/hyp/switch.h +6 −7
  4. arch/arm64/kvm/hyp/nvhe/mem_protect.c +1 −1
  5. arch/arm64/kvm/hyp/vgic-v3-sr.c +21 −15
  6. arch/arm64/kvm/mmu.c +8 −5
  7. arch/arm64/kvm/sys_regs.c +6 −0
  8. arch/riscv/kvm/vcpu.c +2 −0
  9. arch/x86/kvm/mmu.h +3 −0
  10. arch/x86/kvm/mmu/mmu.c +54 −16
  11. arch/x86/kvm/smm.c +1 −0
  12. arch/x86/kvm/svm/sev.c +19 −13
  13. arch/x86/kvm/svm/svm.c +69 −6
  14. arch/x86/kvm/svm/svm.h +2 −0
  15. arch/x86/kvm/x86.c +2 −2
  16. tools/testing/selftests/kvm/arm64/set_id_regs.c +4 −4
arch/arm64/include/asm/el2_setup.h+1−1
@@ -52,7 +52,7 @@ · 51 unmodified lines
52 mrs x0, id_aa64mmfr1_el152 mrs x0, id_aa64mmfr1_el1
53 ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #453 ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
54 cbz x0, .Lskip_hcrx_\@54 cbz x0, .Lskip_hcrx_\@
55 mov_q x0, HCRX_HOST_FLAGS55 mov_q x0, (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En | HCRX_EL2_EnFPM)
56 56
57 /* Enable GCS if supported */57 /* Enable GCS if supported */
58 mrs_s x1, SYS_ID_AA64PFR1_EL158 mrs_s x1, SYS_ID_AA64PFR1_EL1
arch/arm64/include/asm/kvm_arm.h+1−2
@@ -100,9 +100,8 @@ · 99 unmodified lines
100 HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1)100 HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID1)
101#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA)101#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA)
102#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)102#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)
103#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)103#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H | HCR_AMO | HCR_IMO | HCR_FMO)
104 104
105#define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn | HCRX_EL2_TCR2En | HCRX_EL2_EnFPM)
106#define MPAMHCR_HOST_FLAGS 0105#define MPAMHCR_HOST_FLAGS 0
107 106
108/* TCR_EL2 Registers bits */107/* TCR_EL2 Registers bits */
arch/arm64/kvm/hyp/include/hyp/switch.h+6−7
@@ -235,6 +235,8 @@ static inline void __deactivate_traps_mpam(void) · 234 unmodified lines
235 235
236static inline void __activate_traps_common(struct kvm_vcpu *vcpu)236static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
237{237{
238 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
239
238 /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */240 /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */
239 write_sysreg(1 << 15, hstr_el2);241 write_sysreg(1 << 15, hstr_el2);
240 242
@@ -245,11 +247,8 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) · 4 unmodified lines
245 * EL1 instead of being trapped to EL2.247 * EL1 instead of being trapped to EL2.
246 */248 */
247 if (system_supports_pmuv3()) {249 if (system_supports_pmuv3()) {
248 struct kvm_cpu_context *hctxt;
249
250 write_sysreg(0, pmselr_el0);250 write_sysreg(0, pmselr_el0);
251 251
252 hctxt = host_data_ptr(host_ctxt);
253 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = read_sysreg(pmuserenr_el0);252 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = read_sysreg(pmuserenr_el0);
254 write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);253 write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
255 vcpu_set_flag(vcpu, PMUSERENR_ON_CPU);254 vcpu_set_flag(vcpu, PMUSERENR_ON_CPU);
@@ -269,6 +268,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) · 13 unmodified lines
269 hcrx &= ~clr;268 hcrx &= ~clr;
270 }269 }
271 270
271 ctxt_sys_reg(hctxt, HCRX_EL2) = read_sysreg_s(SYS_HCRX_EL2);
272 write_sysreg_s(hcrx, SYS_HCRX_EL2);272 write_sysreg_s(hcrx, SYS_HCRX_EL2);
273 }273 }
274 274
@@ -278,19 +278,18 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) · 3 unmodified lines
278 278
279static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu)279static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu)
280{280{
281 struct kvm_cpu_context *hctxt = host_data_ptr(host_ctxt);
282
281 write_sysreg(*host_data_ptr(host_debug_state.mdcr_el2), mdcr_el2);283 write_sysreg(*host_data_ptr(host_debug_state.mdcr_el2), mdcr_el2);
282 284
283 write_sysreg(0, hstr_el2);285 write_sysreg(0, hstr_el2);
284 if (system_supports_pmuv3()) {286 if (system_supports_pmuv3()) {
285 struct kvm_cpu_context *hctxt;
286
287 hctxt = host_data_ptr(host_ctxt);
288 write_sysreg(ctxt_sys_reg(hctxt, PMUSERENR_EL0), pmuserenr_el0);287 write_sysreg(ctxt_sys_reg(hctxt, PMUSERENR_EL0), pmuserenr_el0);
289 vcpu_clear_flag(vcpu, PMUSERENR_ON_CPU);288 vcpu_clear_flag(vcpu, PMUSERENR_ON_CPU);
290 }289 }
291 290
292 if (cpus_have_final_cap(ARM64_HAS_HCX))291 if (cpus_have_final_cap(ARM64_HAS_HCX))
293 write_sysreg_s(HCRX_HOST_FLAGS, SYS_HCRX_EL2);292 write_sysreg_s(ctxt_sys_reg(hctxt, HCRX_EL2), SYS_HCRX_EL2);
294 293
295 __deactivate_traps_hfgxtr(vcpu);294 __deactivate_traps_hfgxtr(vcpu);
296 __deactivate_traps_mpam();295 __deactivate_traps_mpam();
arch/arm64/kvm/hyp/nvhe/mem_protect.c+1−1
@@ -503,7 +503,7 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id) · 502 unmodified lines
503{503{
504 int ret;504 int ret;
505 505
506 if (!addr_is_memory(addr))506 if (!range_is_memory(addr, addr + size))
507 return -EPERM;507 return -EPERM;
508 508
509 ret = host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,509 ret = host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
arch/arm64/kvm/hyp/vgic-v3-sr.c+21−15
@@ -429,23 +429,27 @@ u64 __vgic_v3_get_gic_config(void) · 428 unmodified lines
429 /*429 /*
430 * To check whether we have a MMIO-based (GICv2 compatible)430 * To check whether we have a MMIO-based (GICv2 compatible)
431 * CPU interface, we need to disable the system register431 * CPU interface, we need to disable the system register
432 * view. To do that safely, we have to prevent any interrupt432 * view.
433 * from firing (which would be deadly).
434 *433 *
435 * Note that this only makes sense on VHE, as interrupts are
436 * already masked for nVHE as part of the exception entry to
437 * EL2.
438 */
439 if (has_vhe())
440 flags = local_daif_save();
441
442 /*
443 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates434 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
444 * that to be able to set ICC_SRE_EL1.SRE to 0, all the435 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
445 * interrupt overrides must be set. You've got to love this.436 * interrupt overrides must be set. You've got to love this.
437 *
438 * As we always run VHE with HCR_xMO set, no extra xMO
439 * manipulation is required in that case.
440 *
441 * To safely disable SRE, we have to prevent any interrupt
442 * from firing (which would be deadly). This only makes sense
443 * on VHE, as interrupts are already masked for nVHE as part
444 * of the exception entry to EL2.
446 */445 */
447 sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);446 if (has_vhe()) {
448 isb();447 flags = local_daif_save();
448 } else {
449 sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
450 isb();
451 }
452
449 write_gicreg(0, ICC_SRE_EL1);453 write_gicreg(0, ICC_SRE_EL1);
450 isb();454 isb();
451 455
@@ -453,11 +457,13 @@ u64 __vgic_v3_get_gic_config(void) · 1 unmodified line
453 457
454 write_gicreg(sre, ICC_SRE_EL1);458 write_gicreg(sre, ICC_SRE_EL1);
455 isb();459 isb();
456 sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
457 isb();
458 460
459 if (has_vhe())461 if (has_vhe()) {
460 local_daif_restore(flags);462 local_daif_restore(flags);
463 } else {
464 sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
465 isb();
466 }
461 467
462 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);468 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
463 val |= read_gicreg(ICH_VTR_EL2);469 val |= read_gicreg(ICH_VTR_EL2);
arch/arm64/kvm/mmu.c+8−5
@@ -1501,6 +1501,11 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 1500 unmodified lines
1501 return -EFAULT;1501 return -EFAULT;
1502 }1502 }
1503 1503
1504 if (!is_protected_kvm_enabled())
1505 memcache = &vcpu->arch.mmu_page_cache;
1506 else
1507 memcache = &vcpu->arch.pkvm_memcache;
1508
1504 /*1509 /*
1505 * Permission faults just need to update the existing leaf entry,1510 * Permission faults just need to update the existing leaf entry,
1506 * and so normally don't require allocations from the memcache. The1511 * and so normally don't require allocations from the memcache. The
@@ -1510,13 +1515,11 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, · 3 unmodified lines
1510 if (!fault_is_perm || (logging_active && write_fault)) {1515 if (!fault_is_perm || (logging_active && write_fault)) {
1511 int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu);1516 int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu);
1512 1517
1513 if (!is_protected_kvm_enabled()) {1518 if (!is_protected_kvm_enabled())
1514 memcache = &vcpu->arch.mmu_page_cache;
1515 ret = kvm_mmu_topup_memory_cache(memcache, min_pages);1519 ret = kvm_mmu_topup_memory_cache(memcache, min_pages);
1516 } else {1520 else
1517 memcache = &vcpu->arch.pkvm_memcache;
1518 ret = topup_hyp_memcache(memcache, min_pages);1521 ret = topup_hyp_memcache(memcache, min_pages);
1519 }1522
1520 if (ret)1523 if (ret)
1521 return ret;1524 return ret;
1522 }1525 }
arch/arm64/kvm/sys_regs.c+6−0
@@ -1945,6 +1945,12 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, · 1944 unmodified lines
1945 if ((hw_val & mpam_mask) == (user_val & mpam_mask))1945 if ((hw_val & mpam_mask) == (user_val & mpam_mask))
1946 user_val &= ~ID_AA64PFR0_EL1_MPAM_MASK;1946 user_val &= ~ID_AA64PFR0_EL1_MPAM_MASK;
1947 1947
1948 /* Fail the guest's request to disable the AA64 ISA at EL{0,1,2} */
1949 if (!FIELD_GET(ID_AA64PFR0_EL1_EL0, user_val) ||
1950 !FIELD_GET(ID_AA64PFR0_EL1_EL1, user_val) ||
1951 (vcpu_has_nv(vcpu) && !FIELD_GET(ID_AA64PFR0_EL1_EL2, user_val)))
1952 return -EINVAL;
1953
1948 return set_id_reg(vcpu, rd, user_val);1954 return set_id_reg(vcpu, rd, user_val);
1949}1955}
1950 1956
arch/riscv/kvm/vcpu.c+2−0
@@ -77,6 +77,8 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) · 76 unmodified lines
77 memcpy(cntx, reset_cntx, sizeof(*cntx));77 memcpy(cntx, reset_cntx, sizeof(*cntx));
78 spin_unlock(&vcpu->arch.reset_cntx_lock);78 spin_unlock(&vcpu->arch.reset_cntx_lock);
79 79
80 memset(&vcpu->arch.smstateen_csr, 0, sizeof(vcpu->arch.smstateen_csr));
81
80 kvm_riscv_vcpu_fp_reset(vcpu);82 kvm_riscv_vcpu_fp_reset(vcpu);
81 83
82 kvm_riscv_vcpu_vector_reset(vcpu);84 kvm_riscv_vcpu_vector_reset(vcpu);
arch/x86/kvm/mmu.h+3−0
@@ -104,6 +104,9 @@ void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, · 103 unmodified lines
104 104
105static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)105static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
106{106{
107 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
108 kvm_mmu_free_obsolete_roots(vcpu);
109
107 /*110 /*
108 * Checking root.hpa is sufficient even when KVM has mirror root.111 * Checking root.hpa is sufficient even when KVM has mirror root.
109 * We can have either:112 * We can have either:
arch/x86/kvm/mmu/mmu.c+54−16
@@ -5974,6 +5974,7 @@ void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu) · 5973 unmodified lines
5974 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);5974 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
5975 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);5975 __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
5976}5976}
5977EXPORT_SYMBOL_GPL(kvm_mmu_free_obsolete_roots);
5977 5978
5978static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,5979static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5979 int *bytes)5980 int *bytes)
@@ -7669,9 +7670,30 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) · 1689 unmodified lines
7669}7670}
7670 7671
7671#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES7672#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
7673static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7674 int level)
7675{
7676 return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;
7677}
7678
7679static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7680 int level)
7681{
7682 lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;
7683}
7684
7685static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7686 int level)
7687{
7688 lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
7689}
7690
7672bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,7691bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
7673 struct kvm_gfn_range *range)7692 struct kvm_gfn_range *range)
7674{7693{
7694 struct kvm_memory_slot *slot = range->slot;
7695 int level;
7696
7675 /*7697 /*
7676 * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only7698 * Zap SPTEs even if the slot can't be mapped PRIVATE. KVM x86 only
7677 * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM7699 * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM
@@ -7686,6 +7708,38 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · 8 unmodified lines
7686 if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))7708 if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
7687 return false;7709 return false;
7688 7710
7711 if (WARN_ON_ONCE(range->end <= range->start))
7712 return false;
7713
7714 /*
7715 * If the head and tail pages of the range currently allow a hugepage,
7716 * i.e. reside fully in the slot and don't have mixed attributes, then
7717 * add each corresponding hugepage range to the ongoing invalidation,
7718 * e.g. to prevent KVM from creating a hugepage in response to a fault
7719 * for a gfn whose attributes aren't changing. Note, only the range
7720 * of gfns whose attributes are being modified needs to be explicitly
7721 * unmapped, as that will unmap any existing hugepages.
7722 */
7723 for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
7724 gfn_t start = gfn_round_for_level(range->start, level);
7725 gfn_t end = gfn_round_for_level(range->end - 1, level);
7726 gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level);
7727
7728 if ((start != range->start || start + nr_pages > range->end) &&
7729 start >= slot->base_gfn &&
7730 start + nr_pages <= slot->base_gfn + slot->npages &&
7731 !hugepage_test_mixed(slot, start, level))
7732 kvm_mmu_invalidate_range_add(kvm, start, start + nr_pages);
7733
7734 if (end == start)
7735 continue;
7736
7737 if ((end + nr_pages) > range->end &&
7738 (end + nr_pages) <= (slot->base_gfn + slot->npages) &&
7739 !hugepage_test_mixed(slot, end, level))
7740 kvm_mmu_invalidate_range_add(kvm, end, end + nr_pages);
7741 }
7742
7689 /* Unmap the old attribute page. */7743 /* Unmap the old attribute page. */
7690 if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE)7744 if (range->arg.attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE)
7691 range->attr_filter = KVM_FILTER_SHARED;7745 range->attr_filter = KVM_FILTER_SHARED;
@@ -7695,23 +7749,7 @@ bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, · 3 unmodified lines
7695 return kvm_unmap_gfn_range(kvm, range);7749 return kvm_unmap_gfn_range(kvm, range);
7696}7750}
7697 7751
7698static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7699 int level)
7700{
7701 return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;
7702}
7703
7704static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7705 int level)
7706{
7707 lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;
7708}
7709 7752
7710static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
7711 int level)
7712{
7713 lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
7714}
7715 7753
7716static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,7754static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
7717 gfn_t gfn, int level, unsigned long attrs)7755 gfn_t gfn, int level, unsigned long attrs)
arch/x86/kvm/smm.c+1−0
@@ -131,6 +131,7 @@ void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm) · 130 unmodified lines
131 131
132 kvm_mmu_reset_context(vcpu);132 kvm_mmu_reset_context(vcpu);
133}133}
134EXPORT_SYMBOL_GPL(kvm_smm_changed);
134 135
135void process_smi(struct kvm_vcpu *vcpu)136void process_smi(struct kvm_vcpu *vcpu)
136{137{
arch/x86/kvm/svm/sev.c+19−13
@@ -3173,9 +3173,14 @@ skip_vmsa_free: · 3172 unmodified lines
3173 kvfree(svm->sev_es.ghcb_sa);3173 kvfree(svm->sev_es.ghcb_sa);
3174}3174}
3175 3175
3176static u64 kvm_ghcb_get_sw_exit_code(struct vmcb_control_area *control)
3177{
3178 return (((u64)control->exit_code_hi) << 32) | control->exit_code;
3179}
3180
3176static void dump_ghcb(struct vcpu_svm *svm)3181static void dump_ghcb(struct vcpu_svm *svm)
3177{3182{
3178 struct ghcb *ghcb = svm->sev_es.ghcb;3183 struct vmcb_control_area *control = &svm->vmcb->control;
3179 unsigned int nbits;3184 unsigned int nbits;
3180 3185
3181 /* Re-use the dump_invalid_vmcb module parameter */3186 /* Re-use the dump_invalid_vmcb module parameter */
@@ -3184,18 +3189,24 @@ static void dump_ghcb(struct vcpu_svm *svm) · 2 unmodified lines
3184 return;3189 return;
3185 }3190 }
3186 3191
3187 nbits = sizeof(ghcb->save.valid_bitmap) * 8;3192 nbits = sizeof(svm->sev_es.valid_bitmap) * 8;
3188 3193
3189 pr_err("GHCB (GPA=%016llx):\n", svm->vmcb->control.ghcb_gpa);3194 /*
3195 * Print KVM's snapshot of the GHCB values that were (unsuccessfully)
3196 * used to handle the exit. If the guest has since modified the GHCB
3197 * itself, dumping the raw GHCB won't help debug why KVM was unable to
3198 * handle the VMGEXIT that KVM observed.
3199 */
3200 pr_err("GHCB (GPA=%016llx) snapshot:\n", svm->vmcb->control.ghcb_gpa);
3190 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code",3201 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code",
3191 ghcb->save.sw_exit_code, ghcb_sw_exit_code_is_valid(ghcb));3202 kvm_ghcb_get_sw_exit_code(control), kvm_ghcb_sw_exit_code_is_valid(svm));
3192 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1",3203 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1",
3193 ghcb->save.sw_exit_info_1, ghcb_sw_exit_info_1_is_valid(ghcb));3204 control->exit_info_1, kvm_ghcb_sw_exit_info_1_is_valid(svm));
3194 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2",3205 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2",
3195 ghcb->save.sw_exit_info_2, ghcb_sw_exit_info_2_is_valid(ghcb));3206 control->exit_info_2, kvm_ghcb_sw_exit_info_2_is_valid(svm));
3196 pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch",3207 pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch",
3197 ghcb->save.sw_scratch, ghcb_sw_scratch_is_valid(ghcb));3208 svm->sev_es.sw_scratch, kvm_ghcb_sw_scratch_is_valid(svm));
3198 pr_err("%-20s%*pb\n", "valid_bitmap", nbits, ghcb->save.valid_bitmap);3209 pr_err("%-20s%*pb\n", "valid_bitmap", nbits, svm->sev_es.valid_bitmap);
3199}3210}
3200 3211
3201static void sev_es_sync_to_ghcb(struct vcpu_svm *svm)3212static void sev_es_sync_to_ghcb(struct vcpu_svm *svm)
@@ -3266,11 +3277,6 @@ static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) · 64 unmodified lines
3266 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));3277 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
3267}3278}
3268 3279
3269static u64 kvm_ghcb_get_sw_exit_code(struct vmcb_control_area *control)
3270{
3271 return (((u64)control->exit_code_hi) << 32) | control->exit_code;
3272}
3273
3274static int sev_es_validate_vmgexit(struct vcpu_svm *svm)3280static int sev_es_validate_vmgexit(struct vcpu_svm *svm)
3275{3281{
3276 struct vmcb_control_area *control = &svm->vmcb->control;3282 struct vmcb_control_area *control = &svm->vmcb->control;
arch/x86/kvm/svm/svm.c+69−6
@@ -607,9 +607,6 @@ static void svm_disable_virtualization_cpu(void) · 606 unmodified lines
607 kvm_cpu_svm_disable();607 kvm_cpu_svm_disable();
608 608
609 amd_pmu_disable_virt();609 amd_pmu_disable_virt();
610
611 if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
612 msr_clear_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
613}610}
614 611
615static int svm_enable_virtualization_cpu(void)612static int svm_enable_virtualization_cpu(void)
@@ -687,9 +684,6 @@ static int svm_enable_virtualization_cpu(void) · 71 unmodified lines
687 rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi);684 rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi);
688 }685 }
689 686
690 if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
691 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
692
693 return 0;687 return 0;
694}688}
695 689
@@ -1518,6 +1512,63 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) · 822 unmodified lines
1518 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));1512 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
1519}1513}
1520 1514
1515#ifdef CONFIG_CPU_MITIGATIONS
1516static DEFINE_SPINLOCK(srso_lock);
1517static atomic_t srso_nr_vms;
1518
1519static void svm_srso_clear_bp_spec_reduce(void *ign)
1520{
1521 struct svm_cpu_data *sd = this_cpu_ptr(&svm_data);
1522
1523 if (!sd->bp_spec_reduce_set)
1524 return;
1525
1526 msr_clear_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1527 sd->bp_spec_reduce_set = false;
1528}
1529
1530static void svm_srso_vm_destroy(void)
1531{
1532 if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1533 return;
1534
1535 if (atomic_dec_return(&srso_nr_vms))
1536 return;
1537
1538 guard(spinlock)(&srso_lock);
1539
1540 /*
1541 * Verify a new VM didn't come along, acquire the lock, and increment
1542 * the count before this task acquired the lock.
1543 */
1544 if (atomic_read(&srso_nr_vms))
1545 return;
1546
1547 on_each_cpu(svm_srso_clear_bp_spec_reduce, NULL, 1);
1548}
1549
1550static void svm_srso_vm_init(void)
1551{
1552 if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1553 return;
1554
1555 /*
1556 * Acquire the lock on 0 => 1 transitions to ensure a potential 1 => 0
1557 * transition, i.e. destroying the last VM, is fully complete, e.g. so
1558 * that a delayed IPI doesn't clear BP_SPEC_REDUCE after a vCPU runs.
1559 */
1560 if (atomic_inc_not_zero(&srso_nr_vms))
1561 return;
1562
1563 guard(spinlock)(&srso_lock);
1564
1565 atomic_inc(&srso_nr_vms);
1566}
1567#else
1568static void svm_srso_vm_init(void) { }
1569static void svm_srso_vm_destroy(void) { }
1570#endif
1571
1521static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)1572static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1522{1573{
1523 struct vcpu_svm *svm = to_svm(vcpu);1574 struct vcpu_svm *svm = to_svm(vcpu);
@@ -1550,6 +1601,11 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) · 26 unmodified lines
1550 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))1601 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1551 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);1602 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1552 1603
1604 if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE) &&
1605 !sd->bp_spec_reduce_set) {
1606 sd->bp_spec_reduce_set = true;
1607 msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1608 }
1553 svm->guest_state_loaded = true;1609 svm->guest_state_loaded = true;
1554}1610}
1555 1611
@@ -2231,6 +2287,10 @@ static int shutdown_interception(struct kvm_vcpu *vcpu) · 675 unmodified lines
2231 */2287 */
2232 if (!sev_es_guest(vcpu->kvm)) {2288 if (!sev_es_guest(vcpu->kvm)) {
2233 clear_page(svm->vmcb);2289 clear_page(svm->vmcb);
2290#ifdef CONFIG_KVM_SMM
2291 if (is_smm(vcpu))
2292 kvm_smm_changed(vcpu, false);
2293#endif
2234 kvm_vcpu_reset(vcpu, true);2294 kvm_vcpu_reset(vcpu, true);
2235 }2295 }
2236 2296
@@ -5036,6 +5096,8 @@ static void svm_vm_destroy(struct kvm *kvm) · 2799 unmodified lines
5036{5096{
5037 avic_vm_destroy(kvm);5097 avic_vm_destroy(kvm);
5038 sev_vm_destroy(kvm);5098 sev_vm_destroy(kvm);
5099
5100 svm_srso_vm_destroy();
5039}5101}
5040 5102
5041static int svm_vm_init(struct kvm *kvm)5103static int svm_vm_init(struct kvm *kvm)
@@ -5061,6 +5123,7 @@ static int svm_vm_init(struct kvm *kvm) · 19 unmodified lines
5061 return ret;5123 return ret;
5062 }5124 }
5063 5125
5126 svm_srso_vm_init();
5064 return 0;5127 return 0;
5065}5128}
5066 5129
arch/x86/kvm/svm/svm.h+2−0
@@ -335,6 +335,8 @@ struct svm_cpu_data { · 334 unmodified lines
335 u32 next_asid;335 u32 next_asid;
336 u32 min_asid;336 u32 min_asid;
337 337
338 bool bp_spec_reduce_set;
339
338 struct vmcb *save_area;340 struct vmcb *save_area;
339 unsigned long save_area_pa;341 unsigned long save_area_pa;
340 342
arch/x86/kvm/x86.c+2−2
@@ -4597,7 +4597,7 @@ static bool kvm_is_vm_type_supported(unsigned long type) · 4596 unmodified lines
4597 return type < 32 && (kvm_caps.supported_vm_types & BIT(type));4597 return type < 32 && (kvm_caps.supported_vm_types & BIT(type));
4598}4598}
4599 4599
4600static inline u32 kvm_sync_valid_fields(struct kvm *kvm)4600static inline u64 kvm_sync_valid_fields(struct kvm *kvm)
4601{4601{
4602 return kvm && kvm->arch.has_protected_state ? 0 : KVM_SYNC_X86_VALID_FIELDS;4602 return kvm && kvm->arch.has_protected_state ? 0 : KVM_SYNC_X86_VALID_FIELDS;
4603}4603}
@@ -11493,7 +11493,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 6889 unmodified lines
11493{11493{
11494 struct kvm_queued_exception *ex = &vcpu->arch.exception;11494 struct kvm_queued_exception *ex = &vcpu->arch.exception;
11495 struct kvm_run *kvm_run = vcpu->run;11495 struct kvm_run *kvm_run = vcpu->run;
11496 u32 sync_valid_fields;11496 u64 sync_valid_fields;
11497 int r;11497 int r;
11498 11498
11499 r = kvm_mmu_post_init_vm(vcpu->kvm);11499 r = kvm_mmu_post_init_vm(vcpu->kvm);
tools/testing/selftests/kvm/arm64/set_id_regs.c+4−4
@@ -129,10 +129,10 @@ static const struct reg_ftr_bits ftr_id_aa64pfr0_el1[] = { · 128 unmodified lines
129 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, DIT, 0),129 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, DIT, 0),
130 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, SEL2, 0),130 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, SEL2, 0),
131 REG_FTR_BITS(FTR_EXACT, ID_AA64PFR0_EL1, GIC, 0),131 REG_FTR_BITS(FTR_EXACT, ID_AA64PFR0_EL1, GIC, 0),
132 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL3, 0),132 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL3, 1),
133 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL2, 0),133 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL2, 1),
134 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL1, 0),134 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL1, 1),
135 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL0, 0),135 REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR0_EL1, EL0, 1),
136 REG_FTR_END,136 REG_FTR_END,
137};137};
138 138

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t