History

Merge tag 'x86-urgent-2026-05-24' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar: - On SEV guests, handle set_memory_{encrypted,decrypted}() failures more conservatively by assuming that all affected pages are unencrypted (Carlos López) - Disable broadcast TLB flush when PCID is disabled (Tom Lendacky) - Fix VMX vs. hrtimer_rearm_deferred() regression (Peter Zijlstra) - Move IRQ/NMI dispatch code from KVM into x86 core, to prepare for a KVM x2apic fix (Peter Zijlstra) - Fix incorrect munmap() size on map_vdso() failure (Guilherme Giacomo Simoes) * tag 'x86-urgent-2026-05-24' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: virt: sev-guest: Explicitly leak pages in unknown state x86/mm: Disable broadcast TLB flush when PCID is disabled x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred() x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core x86/vdso: Fix incorrect size in munmap() on map_vdso() failure

Linus Torvalds committed 5 months ago

Commit 3526d742 parents a674bf7 fd948c315 files · +141 −72Browse files

Unified · Split

15 files
  1. arch/x86/entry/Makefile +1 −1
  2. arch/x86/entry/common.c +61 −0
  3. arch/x86/entry/entry.S +46 −0
  4. arch/x86/entry/entry_64_fred.S +0 −1
  5. arch/x86/entry/vdso/vma.c +1 −1
  6. arch/x86/include/asm/desc.h +4 −0
  7. arch/x86/include/asm/desc_defs.h +1 −1
  8. arch/x86/include/asm/entry-common.h +2 −0
  9. arch/x86/include/asm/fred.h +0 −1
  10. arch/x86/kernel/cpu/cpuid-deps.c +1 −0
  11. arch/x86/kernel/idt.c +15 −0
  12. arch/x86/kernel/nmi.c +0 −1
  13. arch/x86/kvm/vmx/vmenter.S +0 −46
  14. arch/x86/kvm/vmx/vmx.c +2 −17
  15. drivers/virt/coco/sev-guest/sev-guest.c +7 −3
arch/x86/entry/Makefile+1−1
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE) · 12 unmodified lines
13CFLAGS_syscall_32.o += -fno-stack-protector13CFLAGS_syscall_32.o += -fno-stack-protector
14CFLAGS_syscall_64.o += -fno-stack-protector14CFLAGS_syscall_64.o += -fno-stack-protector
15 15
16obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o16obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o
17 17
18obj-y += vdso/18obj-y += vdso/
19obj-y += vsyscall/19obj-y += vsyscall/
arch/x86/entry/common.c+61−0
@@ -0,0 +1,61 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#include <linux/entry-common.h>
4#include <linux/kvm_types.h>
5#include <linux/hrtimer_rearm.h>
6#include <asm/fred.h>
7#include <asm/desc.h>
8
9#if IS_ENABLED(CONFIG_KVM_INTEL)
10/*
11 * On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as
12 * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
13 * x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs
14 * to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
15 * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
16 * the VM-Exit is held pending until it's unblocked in the host.
17 */
18noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
19{
20 if (event_type == EVENT_TYPE_EXTINT) {
21#ifdef CONFIG_X86_64
22 /*
23 * Use FRED dispatch, even when running IDT. The dispatch
24 * tables are kept in sync between FRED and IDT, and the FRED
25 * dispatch works well with CFI.
26 */
27 fred_entry_from_kvm(event_type, vector);
28#else
29 idt_entry_from_kvm(vector);
30#endif
31 /*
32 * Strictly speaking, only the NMI path requires noinstr.
33 */
34 instrumentation_begin();
35 /*
36 * KVM/VMX will dispatch from IRQ-disabled but for a context
37 * that will have IRQs-enabled. This confuses the entry code
38 * and it will not have reprogrammed the timer. Do so now.
39 */
40 hrtimer_rearm_deferred();
41 instrumentation_end();
42
43 return;
44 }
45
46 WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);
47
48#ifdef CONFIG_X86_64
49 if (cpu_feature_enabled(X86_FEATURE_FRED))
50 return fred_entry_from_kvm(event_type, vector);
51#endif
52
53 /*
54 * Notably, we must use IDT dispatch for NMI when running in IDT mode.
55 * The FRED NMI context is significantly different and will not work
56 * right (specifically FRED fixed the NMI recursion issue).
57 */
58 idt_entry_from_kvm(vector);
59}
60EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);
61#endif
arch/x86/entry/entry.S+46−0
@@ -75,3 +75,49 @@ THUNK warn_thunk_thunk, __warn_thunk · 74 unmodified lines
75#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)75#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
76EXPORT_SYMBOL(__ref_stack_chk_guard);76EXPORT_SYMBOL(__ref_stack_chk_guard);
77#endif77#endif
78
79#if IS_ENABLED(CONFIG_KVM_INTEL)
80.macro IDT_DO_EVENT_IRQOFF call_insn call_target
81 /*
82 * Unconditionally create a stack frame, getting the correct RSP on the
83 * stack (for x86-64) would take two instructions anyways, and RBP can
84 * be used to restore RSP to make objtool happy (see below).
85 */
86 push %_ASM_BP
87 mov %_ASM_SP, %_ASM_BP
88
89#ifdef CONFIG_X86_64
90 /*
91 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
92 * creating the synthetic interrupt stack frame for the IRQ/NMI.
93 */
94 and $-16, %rsp
95 push $__KERNEL_DS
96 push %rbp
97#endif
98 pushf
99 push $__KERNEL_CS
100 \call_insn \call_target
101
102 /*
103 * "Restore" RSP from RBP, even though IRET has already unwound RSP to
104 * the correct value. objtool doesn't know the callee will IRET and,
105 * without the explicit restore, thinks the stack is getting walloped.
106 * Using an unwind hint is problematic due to x86-64's dynamic alignment.
107 */
108 leave
109 RET
110.endm
111
112.pushsection .text, "ax"
113SYM_FUNC_START(idt_do_interrupt_irqoff)
114 IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
115SYM_FUNC_END(idt_do_interrupt_irqoff)
116.popsection
117
118.pushsection .noinstr.text, "ax"
119SYM_FUNC_START(idt_do_nmi_irqoff)
120 IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
121SYM_FUNC_END(idt_do_nmi_irqoff)
122.popsection
123#endif
arch/x86/entry/entry_64_fred.S+0−1
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) · 146 unmodified lines
147 RET147 RET
148 148
149SYM_FUNC_END(asm_fred_entry_from_kvm)149SYM_FUNC_END(asm_fred_entry_from_kvm)
150EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
151#endif150#endif
arch/x86/entry/vdso/vma.c+1−1
@@ -178,7 +178,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr) · 177 unmodified lines
178 if (IS_ERR(vma)) {178 if (IS_ERR(vma)) {
179 ret = PTR_ERR(vma);179 ret = PTR_ERR(vma);
180 do_munmap(mm, text_start, image->size, NULL);180 do_munmap(mm, text_start, image->size, NULL);
181 do_munmap(mm, addr, image->size, NULL);181 do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
182 goto up_fail;182 goto up_fail;
183 }183 }
184 184
arch/x86/include/asm/desc.h+4−0
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void); · 437 unmodified lines
438extern void idt_setup_apic_and_irq_gates(void);438extern void idt_setup_apic_and_irq_gates(void);
439extern bool idt_is_f00f_address(unsigned long address);439extern bool idt_is_f00f_address(unsigned long address);
440 440
441extern void idt_do_interrupt_irqoff(unsigned long address);
442extern void idt_do_nmi_irqoff(void);
443extern void idt_entry_from_kvm(unsigned int vector);
444
441#ifdef CONFIG_X86_64445#ifdef CONFIG_X86_64
442extern void idt_setup_early_pf(void);446extern void idt_setup_early_pf(void);
443#else447#else
arch/x86/include/asm/desc_defs.h+1−1
@@ -145,7 +145,7 @@ struct gate_struct { · 144 unmodified lines
145typedef struct gate_struct gate_desc;145typedef struct gate_struct gate_desc;
146 146
147#ifndef _SETUP147#ifndef _SETUP
148static inline unsigned long gate_offset(const gate_desc *g)148static __always_inline unsigned long gate_offset(const gate_desc *g)
149{149{
150#ifdef CONFIG_X86_64150#ifdef CONFIG_X86_64
151 return g->offset_low | ((unsigned long)g->offset_middle << 16) |151 return g->offset_low | ((unsigned long)g->offset_middle << 16) |
arch/x86/include/asm/entry-common.h+2−0
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void) · 96 unmodified lines
97}97}
98#define arch_exit_to_user_mode arch_exit_to_user_mode98#define arch_exit_to_user_mode arch_exit_to_user_mode
99 99
100extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
101
100#endif102#endif
arch/x86/include/asm/fred.h+0−1
@@ -110,7 +110,6 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { ret · 109 unmodified lines
110static inline void cpu_init_fred_exceptions(void) { }110static inline void cpu_init_fred_exceptions(void) { }
111static inline void cpu_init_fred_rsps(void) { }111static inline void cpu_init_fred_rsps(void) { }
112static inline void fred_complete_exception_setup(void) { }112static inline void fred_complete_exception_setup(void) { }
113static inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }
114static inline void fred_sync_rsp0(unsigned long rsp0) { }113static inline void fred_sync_rsp0(unsigned long rsp0) { }
115static inline void fred_update_rsp0(void) { }114static inline void fred_update_rsp0(void) { }
116#endif /* CONFIG_X86_FRED */115#endif /* CONFIG_X86_FRED */
arch/x86/kernel/cpu/cpuid-deps.c+1−0
@@ -92,6 +92,7 @@ static const struct cpuid_dep cpuid_deps[] = { · 91 unmodified lines
92 { X86_FEATURE_FRED, X86_FEATURE_LKGS },92 { X86_FEATURE_FRED, X86_FEATURE_LKGS },
93 { X86_FEATURE_SPEC_CTRL_SSBD, X86_FEATURE_SPEC_CTRL },93 { X86_FEATURE_SPEC_CTRL_SSBD, X86_FEATURE_SPEC_CTRL },
94 { X86_FEATURE_LASS, X86_FEATURE_SMAP },94 { X86_FEATURE_LASS, X86_FEATURE_SMAP },
95 { X86_FEATURE_INVLPGB, X86_FEATURE_PCID },
95 {}96 {}
96};97};
97 98
arch/x86/kernel/idt.c+15−0
@@ -268,6 +268,21 @@ void __init idt_setup_early_pf(void) · 267 unmodified lines
268}268}
269#endif269#endif
270 270
271#if IS_ENABLED(CONFIG_KVM_INTEL)
272noinstr void idt_entry_from_kvm(unsigned int vector)
273{
274 if (vector == NMI_VECTOR)
275 return idt_do_nmi_irqoff();
276
277 /*
278 * Only the NMI path requires noinstr.
279 */
280 instrumentation_begin();
281 idt_do_interrupt_irqoff(gate_offset(idt_table + vector));
282 instrumentation_end();
283}
284#endif
285
271static void __init idt_map_in_cea(void)286static void __init idt_map_in_cea(void)
272{287{
273 /*288 /*
arch/x86/kernel/nmi.c+0−1
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) · 613 unmodified lines
614{614{
615 exc_nmi(regs);615 exc_nmi(regs);
616}616}
617EXPORT_SYMBOL_FOR_KVM(asm_exc_nmi_kvm_vmx);
618#endif617#endif
619 618
620#ifdef CONFIG_NMI_CHECK_CPU619#ifdef CONFIG_NMI_CHECK_CPU
arch/x86/kvm/vmx/vmenter.S+0−46
@@ -31,38 +31,6 @@ · 30 unmodified lines
31#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE31#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
32#endif32#endif
33 33
34.macro VMX_DO_EVENT_IRQOFF call_insn call_target
35 /*
36 * Unconditionally create a stack frame, getting the correct RSP on the
37 * stack (for x86-64) would take two instructions anyways, and RBP can
38 * be used to restore RSP to make objtool happy (see below).
39 */
40 push %_ASM_BP
41 mov %_ASM_SP, %_ASM_BP
42
43#ifdef CONFIG_X86_64
44 /*
45 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
46 * creating the synthetic interrupt stack frame for the IRQ/NMI.
47 */
48 and $-16, %rsp
49 push $__KERNEL_DS
50 push %rbp
51#endif
52 pushf
53 push $__KERNEL_CS
54 \call_insn \call_target
55
56 /*
57 * "Restore" RSP from RBP, even though IRET has already unwound RSP to
58 * the correct value. objtool doesn't know the callee will IRET and,
59 * without the explicit restore, thinks the stack is getting walloped.
60 * Using an unwind hint is problematic due to x86-64's dynamic alignment.
61 */
62 leave
63 RET
64.endm
65
66.section .noinstr.text, "ax"34.section .noinstr.text, "ax"
67 35
68/**36/**
@@ -320,10 +288,6 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) · 251 unmodified lines
320 288
321SYM_FUNC_END(__vmx_vcpu_run)289SYM_FUNC_END(__vmx_vcpu_run)
322 290
323SYM_FUNC_START(vmx_do_nmi_irqoff)
324 VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
325SYM_FUNC_END(vmx_do_nmi_irqoff)
326
327#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT291#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
328 292
329/**293/**
@@ -375,13 +339,3 @@ SYM_FUNC_START(vmread_error_trampoline) · 45 unmodified lines
375 RET339 RET
376SYM_FUNC_END(vmread_error_trampoline)340SYM_FUNC_END(vmread_error_trampoline)
377#endif341#endif
378
379.section .text, "ax"
380
381#ifndef CONFIG_X86_FRED
382
383SYM_FUNC_START(vmx_do_interrupt_irqoff)
384 VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
385SYM_FUNC_END(vmx_do_interrupt_irqoff)
386
387#endif
arch/x86/kvm/vmx/vmx.c+2−17
@@ -7117,9 +7117,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) · 7116 unmodified lines
7117 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);7117 vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
7118}7118}
7119 7119
7120void vmx_do_interrupt_irqoff(unsigned long entry);
7121void vmx_do_nmi_irqoff(void);
7122
7123static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)7120static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
7124{7121{
7125 /*7122 /*
@@ -7161,17 +7158,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, · 35 unmodified lines
7161 "unexpected VM-Exit interrupt info: 0x%x", intr_info))7158 "unexpected VM-Exit interrupt info: 0x%x", intr_info))
7162 return;7159 return;
7163 7160
7164 /*
7165 * Invoke the kernel's IRQ handler for the vector. Use the FRED path
7166 * when it's available even if FRED isn't fully enabled, e.g. even if
7167 * FRED isn't supported in hardware, in order to avoid the indirect
7168 * CALL in the non-FRED path.
7169 */
7170 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);7161 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
7171 if (IS_ENABLED(CONFIG_X86_FRED))7162 x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
7172 fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
7173 else
7174 vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));
7175 kvm_after_interrupt(vcpu);7163 kvm_after_interrupt(vcpu);
7176 7164
7177 vcpu->arch.at_instruction_boundary = true;7165 vcpu->arch.at_instruction_boundary = true;
@@ -7481,10 +7469,7 @@ noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) · 303 unmodified lines
7481 return;7469 return;
7482 7470
7483 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);7471 kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
7484 if (cpu_feature_enabled(X86_FEATURE_FRED))7472 x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
7485 fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
7486 else
7487 vmx_do_nmi_irqoff();
7488 kvm_after_interrupt(vcpu);7473 kvm_after_interrupt(vcpu);
7489}7474}
7490 7475
drivers/virt/coco/sev-guest/sev-guest.c+7−3
@@ -176,6 +176,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques · 175 unmodified lines
176 struct snp_guest_req req = {};176 struct snp_guest_req req = {};
177 int ret, npages = 0, resp_len;177 int ret, npages = 0, resp_len;
178 sockptr_t certs_address;178 sockptr_t certs_address;
179 u64 pfn;
179 180
180 if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data))181 if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data))
181 return -EINVAL;182 return -EINVAL;
@@ -215,10 +216,11 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques · 33 unmodified lines
215 if (!req.certs_data)216 if (!req.certs_data)
216 return -ENOMEM;217 return -ENOMEM;
217 218
219 pfn = PHYS_PFN(virt_to_phys(req.certs_data));
218 ret = set_memory_decrypted((unsigned long)req.certs_data, npages);220 ret = set_memory_decrypted((unsigned long)req.certs_data, npages);
219 if (ret) {221 if (ret) {
220 pr_err("failed to mark page shared, ret=%d\n", ret);222 pr_err("failed to mark page shared, ret=%d\n", ret);
221 free_pages_exact(req.certs_data, npages << PAGE_SHIFT);223 snp_leak_pages(pfn, npages);
222 return -EFAULT;224 return -EFAULT;
223 }225 }
224 226
@@ -272,10 +274,12 @@ e_free: · 47 unmodified lines
272 kfree(report_resp);274 kfree(report_resp);
273e_free_data:275e_free_data:
274 if (npages) {276 if (npages) {
275 if (set_memory_encrypted((unsigned long)req.certs_data, npages))277 if (set_memory_encrypted((unsigned long)req.certs_data, npages)) {
276 WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");278 WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
277 else279 snp_leak_pages(pfn, npages);
280 } else {
278 free_pages_exact(req.certs_data, npages << PAGE_SHIFT);281 free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
282 }
279 }283 }
280 return ret;284 return ret;
281}285}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t