Merge tag 'x86-urgent-2026-05-24' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar: - On SEV guests, handle set_memory_{encrypted,decrypted}() failures more conservatively by assuming that all affected pages are unencrypted (Carlos López) - Disable broadcast TLB flush when PCID is disabled (Tom Lendacky) - Fix VMX vs. hrtimer_rearm_deferred() regression (Peter Zijlstra) - Move IRQ/NMI dispatch code from KVM into x86 core, to prepare for a KVM x2apic fix (Peter Zijlstra) - Fix incorrect munmap() size on map_vdso() failure (Guilherme Giacomo Simoes) * tag 'x86-urgent-2026-05-24' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: virt: sev-guest: Explicitly leak pages in unknown state x86/mm: Disable broadcast TLB flush when PCID is disabled x86/kvm/vmx: Fix VMX vs hrtimer_rearm_deferred() x86/kvm/vmx: Move IRQ/NMI dispatch from KVM into x86 core x86/vdso: Fix incorrect size in munmap() on map_vdso() failure

Linus Torvalds committed 5 months ago

Commit 3526d742 parents a674bf7 fd948c315 files · +141 −72Browse files

Unified · Split

15 files

arch/x86/entry/Makefile +1 −1
arch/x86/entry/common.c +61 −0
arch/x86/entry/entry.S +46 −0
arch/x86/entry/entry_64_fred.S +0 −1
arch/x86/entry/vdso/vma.c +1 −1
arch/x86/include/asm/desc.h +4 −0
arch/x86/include/asm/desc_defs.h +1 −1
arch/x86/include/asm/entry-common.h +2 −0
arch/x86/include/asm/fred.h +0 −1
arch/x86/kernel/cpu/cpuid-deps.c +1 −0
arch/x86/kernel/idt.c +15 −0
arch/x86/kernel/nmi.c +0 −1
arch/x86/kvm/vmx/vmenter.S +0 −46
arch/x86/kvm/vmx/vmx.c +2 −17
drivers/virt/coco/sev-guest/sev-guest.c +7 −3

arch/x86/entry/Makefile+1−1

@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE) · 12 unmodified lines
13	13	CFLAGS_syscall_32.o += -fno-stack-protector
14	14	CFLAGS_syscall_64.o += -fno-stack-protector
15	15
16		−obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o
	16	+obj-y := entry.o entry_$(BITS).o syscall_$(BITS).o common.o
17	17
18	18	obj-y += vdso/
19	19	obj-y += vsyscall/

arch/x86/entry/common.c+61−0

@@ -0,0 +1,61 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#include <linux/entry-common.h>
	4	+#include <linux/kvm_types.h>
	5	+#include <linux/hrtimer_rearm.h>
	6	+#include <asm/fred.h>
	7	+#include <asm/desc.h>
	8	+
	9	+#if IS_ENABLED(CONFIG_KVM_INTEL)
	10	+/*
	11	+ * On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as
	12	+ * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
	13	+ * x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs
	14	+ * to the kernel for servicing. On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
	15	+ * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
	16	+ * the VM-Exit is held pending until it's unblocked in the host.
	17	+ */
	18	+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
	19	+{
	20	+ if (event_type == EVENT_TYPE_EXTINT) {
	21	+#ifdef CONFIG_X86_64
	22	+ /*
	23	+ * Use FRED dispatch, even when running IDT. The dispatch
	24	+ * tables are kept in sync between FRED and IDT, and the FRED
	25	+ * dispatch works well with CFI.
	26	+ */
	27	+ fred_entry_from_kvm(event_type, vector);
	28	+#else
	29	+ idt_entry_from_kvm(vector);
	30	+#endif
	31	+ /*
	32	+ * Strictly speaking, only the NMI path requires noinstr.
	33	+ */
	34	+ instrumentation_begin();
	35	+ /*
	36	+ * KVM/VMX will dispatch from IRQ-disabled but for a context
	37	+ * that will have IRQs-enabled. This confuses the entry code
	38	+ * and it will not have reprogrammed the timer. Do so now.
	39	+ */
	40	+ hrtimer_rearm_deferred();
	41	+ instrumentation_end();
	42	+
	43	+ return;
	44	+ }
	45	+
	46	+ WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);
	47	+
	48	+#ifdef CONFIG_X86_64
	49	+ if (cpu_feature_enabled(X86_FEATURE_FRED))
	50	+ return fred_entry_from_kvm(event_type, vector);
	51	+#endif
	52	+
	53	+ /*
	54	+ * Notably, we must use IDT dispatch for NMI when running in IDT mode.
	55	+ * The FRED NMI context is significantly different and will not work
	56	+ * right (specifically FRED fixed the NMI recursion issue).
	57	+ */
	58	+ idt_entry_from_kvm(vector);
	59	+}
	60	+EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);
	61	+#endif

arch/x86/entry/entry.S+46−0

@@ -75,3 +75,49 @@ THUNK warn_thunk_thunk, __warn_thunk · 74 unmodified lines
75	75	#if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
76	76	EXPORT_SYMBOL(__ref_stack_chk_guard);
77	77	#endif
	78	+
	79	+#if IS_ENABLED(CONFIG_KVM_INTEL)
	80	+.macro IDT_DO_EVENT_IRQOFF call_insn call_target
	81	+ /*
	82	+ * Unconditionally create a stack frame, getting the correct RSP on the
	83	+ * stack (for x86-64) would take two instructions anyways, and RBP can
	84	+ * be used to restore RSP to make objtool happy (see below).
	85	+ */
	86	+ push %_ASM_BP
	87	+ mov %_ASM_SP, %_ASM_BP
	88	+
	89	+#ifdef CONFIG_X86_64
	90	+ /*
	91	+ * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
	92	+ * creating the synthetic interrupt stack frame for the IRQ/NMI.
	93	+ */
	94	+ and $-16, %rsp
	95	+ push $__KERNEL_DS
	96	+ push %rbp
	97	+#endif
	98	+ pushf
	99	+ push $__KERNEL_CS
	100	+ \call_insn \call_target
	101	+
	102	+ /*
	103	+ * "Restore" RSP from RBP, even though IRET has already unwound RSP to
	104	+ * the correct value. objtool doesn't know the callee will IRET and,
	105	+ * without the explicit restore, thinks the stack is getting walloped.
	106	+ * Using an unwind hint is problematic due to x86-64's dynamic alignment.
	107	+ */
	108	+ leave
	109	+ RET
	110	+.endm
	111	+
	112	+.pushsection .text, "ax"
	113	+SYM_FUNC_START(idt_do_interrupt_irqoff)
	114	+ IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
	115	+SYM_FUNC_END(idt_do_interrupt_irqoff)
	116	+.popsection
	117	+
	118	+.pushsection .noinstr.text, "ax"
	119	+SYM_FUNC_START(idt_do_nmi_irqoff)
	120	+ IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
	121	+SYM_FUNC_END(idt_do_nmi_irqoff)
	122	+.popsection
	123	+#endif

arch/x86/entry/entry_64_fred.S+0−1

@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) · 146 unmodified lines
147	147	RET
148	148
149	149	SYM_FUNC_END(asm_fred_entry_from_kvm)
150		−EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
151	150	#endif

arch/x86/entry/vdso/vma.c+1−1

@@ -178,7 +178,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr) · 177 unmodified lines
178	178	if (IS_ERR(vma)) {
179	179	ret = PTR_ERR(vma);
180	180	do_munmap(mm, text_start, image->size, NULL);
181		− do_munmap(mm, addr, image->size, NULL);
	181	+ do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
182	182	goto up_fail;
183	183	}
184	184

arch/x86/include/asm/desc.h+4−0

@@ -438,6 +438,10 @@ extern void idt_setup_traps(void); · 437 unmodified lines
438	438	extern void idt_setup_apic_and_irq_gates(void);
439	439	extern bool idt_is_f00f_address(unsigned long address);
440	440
	441	+extern void idt_do_interrupt_irqoff(unsigned long address);
	442	+extern void idt_do_nmi_irqoff(void);
	443	+extern void idt_entry_from_kvm(unsigned int vector);
	444	+
441	445	#ifdef CONFIG_X86_64
442	446	extern void idt_setup_early_pf(void);
443	447	#else

arch/x86/include/asm/desc_defs.h+1−1

@@ -145,7 +145,7 @@ struct gate_struct { · 144 unmodified lines
145	145	typedef struct gate_struct gate_desc;
146	146
147	147	#ifndef _SETUP
148		−static inline unsigned long gate_offset(const gate_desc *g)
	148	+static __always_inline unsigned long gate_offset(const gate_desc *g)
149	149	{
150	150	#ifdef CONFIG_X86_64
151	151	return g->offset_low \| ((unsigned long)g->offset_middle << 16) \|

arch/x86/include/asm/entry-common.h+2−0

@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void) · 96 unmodified lines
97	97	}
98	98	#define arch_exit_to_user_mode arch_exit_to_user_mode
99	99
	100	+extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
	101	+
100	102	#endif

arch/x86/include/asm/fred.h+0−1

@@ -110,7 +110,6 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { ret · 109 unmodified lines
110	110	static inline void cpu_init_fred_exceptions(void) { }
111	111	static inline void cpu_init_fred_rsps(void) { }
112	112	static inline void fred_complete_exception_setup(void) { }
113		−static inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }
114	113	static inline void fred_sync_rsp0(unsigned long rsp0) { }
115	114	static inline void fred_update_rsp0(void) { }
116	115	#endif /* CONFIG_X86_FRED */

arch/x86/kernel/cpu/cpuid-deps.c+1−0

@@ -92,6 +92,7 @@ static const struct cpuid_dep cpuid_deps[] = { · 91 unmodified lines
92	92	{ X86_FEATURE_FRED, X86_FEATURE_LKGS },
93	93	{ X86_FEATURE_SPEC_CTRL_SSBD, X86_FEATURE_SPEC_CTRL },
94	94	{ X86_FEATURE_LASS, X86_FEATURE_SMAP },
	95	+ { X86_FEATURE_INVLPGB, X86_FEATURE_PCID },
95	96	{}
96	97	};
97	98

arch/x86/kernel/idt.c+15−0

@@ -268,6 +268,21 @@ void __init idt_setup_early_pf(void) · 267 unmodified lines
268	268	}
269	269	#endif
270	270
	271	+#if IS_ENABLED(CONFIG_KVM_INTEL)
	272	+noinstr void idt_entry_from_kvm(unsigned int vector)
	273	+{
	274	+ if (vector == NMI_VECTOR)
	275	+ return idt_do_nmi_irqoff();
	276	+
	277	+ /*
	278	+ * Only the NMI path requires noinstr.
	279	+ */
	280	+ instrumentation_begin();
	281	+ idt_do_interrupt_irqoff(gate_offset(idt_table + vector));
	282	+ instrumentation_end();
	283	+}
	284	+#endif
	285	+
271	286	static void __init idt_map_in_cea(void)
272	287	{
273	288	/*

arch/x86/kernel/nmi.c+0−1

@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) · 613 unmodified lines
614	614	{
615	615	exc_nmi(regs);
616	616	}
617		−EXPORT_SYMBOL_FOR_KVM(asm_exc_nmi_kvm_vmx);
618	617	#endif
619	618
620	619	#ifdef CONFIG_NMI_CHECK_CPU

arch/x86/kvm/vmx/vmenter.S+0−46

@@ -31,38 +31,6 @@ · 30 unmodified lines
31	31	#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
32	32	#endif
33	33
34		−.macro VMX_DO_EVENT_IRQOFF call_insn call_target
35		− /*
36		− * Unconditionally create a stack frame, getting the correct RSP on the
37		− * stack (for x86-64) would take two instructions anyways, and RBP can
38		− * be used to restore RSP to make objtool happy (see below).
39		− */
40		− push %_ASM_BP
41		− mov %_ASM_SP, %_ASM_BP
42		−
43		−#ifdef CONFIG_X86_64
44		− /*
45		− * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
46		− * creating the synthetic interrupt stack frame for the IRQ/NMI.
47		− */
48		− and $-16, %rsp
49		− push $__KERNEL_DS
50		− push %rbp
51		−#endif
52		− pushf
53		− push $__KERNEL_CS
54		− \call_insn \call_target
55		−
56		− /*
57		− * "Restore" RSP from RBP, even though IRET has already unwound RSP to
58		− * the correct value. objtool doesn't know the callee will IRET and,
59		− * without the explicit restore, thinks the stack is getting walloped.
60		− * Using an unwind hint is problematic due to x86-64's dynamic alignment.
61		− */
62		− leave
63		− RET
64		−.endm
65		−
66	34	.section .noinstr.text, "ax"
67	35
68	36	/**
@@ -320,10 +288,6 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) · 251 unmodified lines
320	288
321	289	SYM_FUNC_END(__vmx_vcpu_run)
322	290
323		−SYM_FUNC_START(vmx_do_nmi_irqoff)
324		− VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
325		−SYM_FUNC_END(vmx_do_nmi_irqoff)
326		−
327	291	#ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
328	292
329	293	/**
@@ -375,13 +339,3 @@ SYM_FUNC_START(vmread_error_trampoline) · 45 unmodified lines
375	339	RET
376	340	SYM_FUNC_END(vmread_error_trampoline)
377	341	#endif
378		−
379		−.section .text, "ax"
380		−
381		−#ifndef CONFIG_X86_FRED
382		−
383		−SYM_FUNC_START(vmx_do_interrupt_irqoff)
384		− VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
385		−SYM_FUNC_END(vmx_do_interrupt_irqoff)
386		−
387		−#endif

arch/x86/kvm/vmx/vmx.c+2−17

@@ -7117,9 +7117,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu vcpu, u64 eoi_exit_bitmap) · 7116 unmodified lines
7117	7117	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
7118	7118	}
7119	7119
7120		−void vmx_do_interrupt_irqoff(unsigned long entry);
7121		−void vmx_do_nmi_irqoff(void);
7122		−
7123	7120	static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
7124	7121	{
7125	7122	/*
@@ -7161,17 +7158,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, · 35 unmodified lines
7161	7158	"unexpected VM-Exit interrupt info: 0x%x", intr_info))
7162	7159	return;
7163	7160
7164		− /*
7165		− * Invoke the kernel's IRQ handler for the vector. Use the FRED path
7166		− * when it's available even if FRED isn't fully enabled, e.g. even if
7167		− * FRED isn't supported in hardware, in order to avoid the indirect
7168		− * CALL in the non-FRED path.
7169		− */
7170	7161	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
7171		− if (IS_ENABLED(CONFIG_X86_FRED))
7172		− fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
7173		− else
7174		− vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));
	7162	+ x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
7175	7163	kvm_after_interrupt(vcpu);
7176	7164
7177	7165	vcpu->arch.at_instruction_boundary = true;
@@ -7481,10 +7469,7 @@ noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) · 303 unmodified lines
7481	7469	return;
7482	7470
7483	7471	kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
7484		− if (cpu_feature_enabled(X86_FEATURE_FRED))
7485		− fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
7486		− else
7487		− vmx_do_nmi_irqoff();
	7472	+ x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
7488	7473	kvm_after_interrupt(vcpu);
7489	7474	}
7490	7475

drivers/virt/coco/sev-guest/sev-guest.c+7−3

@@ -176,6 +176,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques · 175 unmodified lines
176	176	struct snp_guest_req req = {};
177	177	int ret, npages = 0, resp_len;
178	178	sockptr_t certs_address;
	179	+ u64 pfn;
179	180
180	181	if (sockptr_is_null(io->req_data) \|\| sockptr_is_null(io->resp_data))
181	182	return -EINVAL;
@@ -215,10 +216,11 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques · 33 unmodified lines
215	216	if (!req.certs_data)
216	217	return -ENOMEM;
217	218
	219	+ pfn = PHYS_PFN(virt_to_phys(req.certs_data));
218	220	ret = set_memory_decrypted((unsigned long)req.certs_data, npages);
219	221	if (ret) {
220	222	pr_err("failed to mark page shared, ret=%d\n", ret);
221		− free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
	223	+ snp_leak_pages(pfn, npages);
222	224	return -EFAULT;
223	225	}
224	226
@@ -272,10 +274,12 @@ e_free: · 47 unmodified lines
272	274	kfree(report_resp);
273	275	e_free_data:
274	276	if (npages) {
275		− if (set_memory_encrypted((unsigned long)req.certs_data, npages))
	277	+ if (set_memory_encrypted((unsigned long)req.certs_data, npages)) {
276	278	WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
277		− else
	279	+ snp_leak_pages(pfn, npages);
	280	+ } else {
278	281	free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
	282	+ }
279	283	}
280	284	return ret;
281	285	}

Merge tag 'x86-urgent-2026-05-24' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code