Merge tag 'x86_core_for_v6.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull more x86 updates from Borislav Petkov: - Remove a bunch of asm implementing condition flags testing in KVM's emulator in favor of int3_emulate_jcc() which is written in C - Replace KVM fastops with C-based stubs which avoids problems with the fastop infra related to latter not adhering to the C ABI due to their special calling convention and, more importantly, bypassing compiler control-flow integrity checking because they're written in asm - Remove wrongly used static branches and other ugliness accumulated over time in hyperv's hypercall implementation with a proper static function call to the correct hypervisor call variant - Add some fixes and modifications to allow running FRED-enabled kernels in KVM even on non-FRED hardware - Add kCFI improvements like validating indirect calls and prepare for enabling kCFI with GCC. Add cmdline params documentation and other code cleanups - Use the single-byte 0xd6 insn as the official #UD single-byte undefined opcode instruction as agreed upon by both x86 vendors - Other smaller cleanups and touchups all over the place * tag 'x86_core_for_v6.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (24 commits) x86,retpoline: Optimize patch_retpoline() x86,ibt: Use UDB instead of 0xEA x86/cfi: Remove __noinitretpoline and __noretpoline x86/cfi: Add "debug" option to "cfi=" bootparam x86/cfi: Standardize on common "CFI:" prefix for CFI reports x86/cfi: Document the "cfi=" bootparam options x86/traps: Clarify KCFI instruction layout compiler_types.h: Move __nocfi out of compiler-specific header objtool: Validate kCFI calls x86/fred: KVM: VMX: Always use FRED for IRQs when CONFIG_X86_FRED=y x86/fred: Play nice with invoking asm_fred_entry_from_kvm() on non-FRED hardware x86/fred: Install system vector handlers even if FRED isn't fully enabled x86/hyperv: Use direct call to hypercall-page x86/hyperv: Clean up hv_do_hypercall() KVM: x86: Remove fastops KVM: x86: Convert em_salc() to C KVM: x86: Introduce EM_ASM_3WCL KVM: x86: Introduce EM_ASM_1SRC2 KVM: x86: Introduce EM_ASM_2CL KVM: x86: Introduce EM_ASM_2W ...

Linus Torvalds committed 1 year ago

Commit 9591fdb2 parents 2f0a750 4a1e02b36 files · +728 −665Browse files

Unified · Split

36 files

Documentation/admin-guide/kernel-parameters.txt +18 −0
arch/x86/entry/calling.h +5 −6
arch/x86/entry/entry_64_fred.S +26 −7
arch/x86/hyperv/hv_init.c +44 −25
arch/x86/hyperv/ivm.c +15 −0
arch/x86/include/asm/bug.h +7 −2
arch/x86/include/asm/cfi.h +6 −8
arch/x86/include/asm/ibt.h +3 −7
arch/x86/include/asm/idtentry.h +2 −7
arch/x86/include/asm/mshyperv.h +41 −96
arch/x86/include/asm/text-patching.h +13 −7
arch/x86/kernel/alternative.c +176 −116
arch/x86/kernel/asm-offsets.c +1 −0
arch/x86/kernel/cfi.c +1 −1
arch/x86/kernel/cpu/mshyperv.c +13 −6
arch/x86/kernel/irqinit.c +4 −2
arch/x86/kernel/machine_kexec_64.c +4 −0
arch/x86/kernel/traps.c +4 −4
arch/x86/kvm/Kconfig +1 −0
arch/x86/kvm/emulate.c +232 −318
arch/x86/kvm/vmx/vmenter.S +4 −0
arch/x86/kvm/vmx/vmx.c +7 −1
arch/x86/lib/bhi.S +29 −29
arch/x86/lib/retpoline.S +2 −2
arch/x86/net/bpf_jit_comp.c +3 −3
arch/x86/platform/efi/efi_stub_64.S +4 −0
drivers/misc/lkdtm/perms.c +5 −0
include/linux/compiler-clang.h +0 −5
include/linux/compiler-gcc.h +0 −4
include/linux/compiler_types.h +3 −1
include/linux/init.h +0 −8
include/linux/objtool.h +10 −0
include/linux/objtool_types.h +1 −0
tools/include/linux/objtool_types.h +1 −0
tools/objtool/check.c +42 −0
tools/objtool/include/objtool/elf.h +1 −0

Documentation/admin-guide/kernel-parameters.txt+18−0

@@ -608,6 +608,24 @@ · 607 unmodified lines
608	608	ccw_timeout_log [S390]
609	609	See Documentation/arch/s390/common_io.rst for details.
610	610
	611	+ cfi= [X86-64] Set Control Flow Integrity checking features
	612	+ when CONFIG_FINEIBT is enabled.
	613	+ Format: feature[,feature...]
	614	+ Default: auto
	615	+
	616	+ auto: Use FineIBT if IBT available, otherwise kCFI.
	617	+ Under FineIBT, enable "paranoid" mode when
	618	+ FRED is not available.
	619	+ off: Turn off CFI checking.
	620	+ kcfi: Use kCFI (disable FineIBT).
	621	+ fineibt: Use FineIBT (even if IBT not available).
	622	+ norand: Do not re-randomize CFI hashes.
	623	+ paranoid: Add caller hash checking under FineIBT.
	624	+ bhi: Enable register poisoning to stop speculation
	625	+ across FineIBT. (Disabled by default.)
	626	+ warn: Do not enforce CFI checking: warn only.
	627	+ debug: Report CFI initialization details.
	628	+
611	629	cgroup_disable= [KNL] Disable a particular controller or optional feature
612	630	Format: {name of the controller(s) or feature(s) to disable}
613	631	The effects of cgroup_disable=foo are:

arch/x86/entry/calling.h+5−6

@@ -99,7 +99,7 @@ For 32-bit we have the following conventions - kernel is built with · 98 unmodified lines
99	99	.endif
100	100	.endm
101	101
102		−.macro CLEAR_REGS clear_bp=1
	102	+.macro CLEAR_REGS clear_callee=1
103	103	/*
104	104	* Sanitize registers of values that a speculation attack might
105	105	* otherwise want to exploit. The lower registers are likely clobbered
@@ -113,20 +113,19 @@ For 32-bit we have the following conventions - kernel is built with · 7 unmodified lines
113	113	xorl %r9d, %r9d /* nospec r9 */
114	114	xorl %r10d, %r10d /* nospec r10 */
115	115	xorl %r11d, %r11d /* nospec r11 */
	116	+ .if \clear_callee
116	117	xorl %ebx, %ebx /* nospec rbx */
117		− .if \clear_bp
118	118	xorl %ebp, %ebp /* nospec rbp */
119		− .endif
120	119	xorl %r12d, %r12d /* nospec r12 */
121	120	xorl %r13d, %r13d /* nospec r13 */
122	121	xorl %r14d, %r14d /* nospec r14 */
123	122	xorl %r15d, %r15d /* nospec r15 */
124		−
	123	+ .endif
125	124	.endm
126	125
127		−.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1
	126	+.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1
128	127	PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint
129		− CLEAR_REGS clear_bp=\clear_bp
	128	+ CLEAR_REGS clear_callee=\clear_callee
130	129	.endm
131	130
132	131	.macro POP_REGS pop_rdi=1

arch/x86/entry/entry_64_fred.S+26−7

@@ -111,18 +111,37 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) · 110 unmodified lines
111	111	push %rax /* Return RIP */
112	112	push $0 /* Error code, 0 for IRQ/NMI */
113	113
114		− PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0
	114	+ PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
	115	+
115	116	movq %rsp, %rdi /* %rdi -> pt_regs */
	117	+ /*
	118	+ * At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
	119	+ * are clobbered, which corresponds to: arguments, extra caller-saved
	120	+ * and return. All registers a C function is allowed to clobber.
	121	+ *
	122	+ * Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
	123	+ * are untouched, with the exception of rbp, which carries the stack
	124	+ * frame and will be restored before exit.
	125	+ *
	126	+ * Further calling another C function will not alter this state.
	127	+ */
116	128	call __fred_entry_from_kvm /* Call the C entry point */
117		− POP_REGS
118		− ERETS
119		−1:
	129	+
120	130	/*
121		− * Objtool doesn't understand what ERETS does, this hint tells it that
122		− * yes, we'll reach here and with what stack state. A save/restore pair
123		− * isn't strictly needed, but it's the simplest form.
	131	+ * When FRED, use ERETS to potentially clear NMIs, otherwise simply
	132	+ * restore the stack pointer.
	133	+ */
	134	+ ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
	135	+ __stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
	136	+ X86_FEATURE_FRED
	137	+
	138	+1: /*
	139	+ * Objtool doesn't understand ERETS, and the cfi register state is
	140	+ * different from initial_func_cfi due to PUSH_REGS. Tell it the state
	141	+ * is similar to where UNWIND_HINT_SAVE is.
124	142	*/
125	143	UNWIND_HINT_RESTORE
	144	+
126	145	pop %rbp
127	146	RET
128	147

arch/x86/hyperv/hv_init.c+44−25

@@ -17,7 +17,6 @@ · 16 unmodified lines
17	17	#include <asm/desc.h>
18	18	#include <asm/e820/api.h>
19	19	#include <asm/sev.h>
20		−#include <asm/ibt.h>
21	20	#include <asm/hypervisor.h>
22	21	#include <hyperv/hvhdk.h>
23	22	#include <asm/mshyperv.h>
@@ -37,7 +36,45 @@ · 13 unmodified lines
37	36	#include <linux/export.h>
38	37
39	38	void *hv_hypercall_pg;
	39	+
	40	+#ifdef CONFIG_X86_64
	41	+static u64 __hv_hyperfail(u64 control, u64 param1, u64 param2)
	42	+{
	43	+ return U64_MAX;
	44	+}
	45	+
	46	+DEFINE_STATIC_CALL(__hv_hypercall, __hv_hyperfail);
	47	+
	48	+u64 hv_std_hypercall(u64 control, u64 param1, u64 param2)
	49	+{
	50	+ u64 hv_status;
	51	+
	52	+ register u64 __r8 asm("r8") = param2;
	53	+ asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall)
	54	+ : "=a" (hv_status), ASM_CALL_CONSTRAINT,
	55	+ "+c" (control), "+d" (param1), "+r" (__r8)
	56	+ : : "cc", "memory", "r9", "r10", "r11");
	57	+
	58	+ return hv_status;
	59	+}
	60	+
	61	+typedef u64 (*hv_hypercall_f)(u64 control, u64 param1, u64 param2);
	62	+
	63	+static inline void hv_set_hypercall_pg(void *ptr)
	64	+{
	65	+ hv_hypercall_pg = ptr;
	66	+
	67	+ if (!ptr)
	68	+ ptr = &__hv_hyperfail;
	69	+ static_call_update(__hv_hypercall, (hv_hypercall_f)ptr);
	70	+}
	71	+#else
	72	+static inline void hv_set_hypercall_pg(void *ptr)
	73	+{
	74	+ hv_hypercall_pg = ptr;
	75	+}
40	76	EXPORT_SYMBOL_GPL(hv_hypercall_pg);
	77	+#endif
41	78
42	79	union hv_ghcb * __percpu *hv_ghcb_pg;
43	80
@@ -330,7 +367,7 @@ static int hv_suspend(void) · 286 unmodified lines
330	367	* pointer is restored on resume.
331	368	*/
332	369	hv_hypercall_pg_saved = hv_hypercall_pg;
333		− hv_hypercall_pg = NULL;
	370	+ hv_set_hypercall_pg(NULL);
334	371
335	372	/* Disable the hypercall page in the hypervisor */
336	373	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
@@ -356,7 +393,7 @@ static void hv_resume(void) · 19 unmodified lines
356	393	vmalloc_to_pfn(hv_hypercall_pg_saved);
357	394	wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
358	395
359		− hv_hypercall_pg = hv_hypercall_pg_saved;
	396	+ hv_set_hypercall_pg(hv_hypercall_pg_saved);
360	397	hv_hypercall_pg_saved = NULL;
361	398
362	399	/*
@@ -476,8 +513,8 @@ void __init hyperv_init(void) · 113 unmodified lines
476	513	if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present)
477	514	goto skip_hypercall_pg_init;
478	515
479		− hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START,
480		− VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX,
	516	+ hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR,
	517	+ MODULES_END, GFP_KERNEL, PAGE_KERNEL_ROX,
481	518	VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
482	519	__builtin_return_address(0));
483	520	if (hv_hypercall_pg == NULL)
@@ -515,27 +552,9 @@ void __init hyperv_init(void) · 31 unmodified lines
515	552	wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
516	553	}
517	554
518		−skip_hypercall_pg_init:
519		− /*
520		− * Some versions of Hyper-V that provide IBT in guest VMs have a bug
521		− * in that there's no ENDBR64 instruction at the entry to the
522		− * hypercall page. Because hypercalls are invoked via an indirect call
523		− * to the hypercall page, all hypercall attempts fail when IBT is
524		− * enabled, and Linux panics. For such buggy versions, disable IBT.
525		− *
526		− * Fixed versions of Hyper-V always provide ENDBR64 on the hypercall
527		− * page, so if future Linux kernel versions enable IBT for 32-bit
528		− * builds, additional hypercall page hackery will be required here
529		− * to provide an ENDBR32.
530		− */
531		−#ifdef CONFIG_X86_KERNEL_IBT
532		− if (cpu_feature_enabled(X86_FEATURE_IBT) &&
533		− (u32 )hv_hypercall_pg != gen_endbr()) {
534		− setup_clear_cpu_cap(X86_FEATURE_IBT);
535		− pr_warn("Disabling IBT because of Hyper-V bug\n");
536		− }
537		−#endif
	555	+ hv_set_hypercall_pg(hv_hypercall_pg);
538	556
	557	+skip_hypercall_pg_init:
539	558	/*
540	559	* hyperv_init() is called before LAPIC is initialized: see
541	560	* apic_intr_mode_init() -> x86_platform.apic_post_init() and

arch/x86/hyperv/ivm.c+15−0

@@ -385,9 +385,23 @@ int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu) · 384 unmodified lines
385	385	return ret;
386	386	}
387	387
	388	+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2)
	389	+{
	390	+ u64 hv_status;
	391	+
	392	+ register u64 __r8 asm("r8") = param2;
	393	+ asm volatile("vmmcall"
	394	+ : "=a" (hv_status), ASM_CALL_CONSTRAINT,
	395	+ "+c" (control), "+d" (param1), "+r" (__r8)
	396	+ : : "cc", "memory", "r9", "r10", "r11");
	397	+
	398	+ return hv_status;
	399	+}
	400	+
388	401	#else
389	402	static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
390	403	static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
	404	+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
391	405	#endif /* CONFIG_AMD_MEM_ENCRYPT */
392	406
393	407	#ifdef CONFIG_INTEL_TDX_GUEST
@@ -437,6 +451,7 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) · 43 unmodified lines
437	451	#else
438	452	static inline void hv_tdx_msr_write(u64 msr, u64 value) {}
439	453	static inline void hv_tdx_msr_read(u64 msr, u64 *value) {}
	454	+u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
440	455	#endif /* CONFIG_INTEL_TDX_GUEST */
441	456
442	457	#if defined(CONFIG_AMD_MEM_ENCRYPT) \|\| defined(CONFIG_INTEL_TDX_GUEST)

arch/x86/include/asm/bug.h+7−2

@@ -5,14 +5,19 @@ · 4 unmodified lines
5	5	#include <linux/stringify.h>
6	6	#include <linux/instrumentation.h>
7	7	#include <linux/objtool.h>
	8	+#include <asm/asm.h>
8	9
9	10	/*
10	11	* Despite that some emulators terminate on UD2, we use it for WARN().
11	12	*/
12		−#define ASM_UD2 ".byte 0x0f, 0x0b"
	13	+#define ASM_UD2 _ASM_BYTES(0x0f, 0x0b)
13	14	#define INSN_UD2 0x0b0f
14	15	#define LEN_UD2 2
15	16
	17	+#define ASM_UDB _ASM_BYTES(0xd6)
	18	+#define INSN_UDB 0xd6
	19	+#define LEN_UDB 1
	20	+
16	21	/*
17	22	* In clang we have UD1s reporting UBSAN failures on X86, 64 and 32bit.
18	23	*/
@@ -26,7 +31,7 @@ · 7 unmodified lines
26	31	#define BUG_UD2 0xfffe
27	32	#define BUG_UD1 0xfffd
28	33	#define BUG_UD1_UBSAN 0xfffc
29		−#define BUG_EA 0xffea
	34	+#define BUG_UDB 0xffd6
30	35	#define BUG_LOCK 0xfff0
31	36
32	37	#ifdef CONFIG_GENERIC_BUG

arch/x86/include/asm/cfi.h+6−8

@@ -71,12 +71,10 @@ · 70 unmodified lines
71	71	*
72	72	* __cfi_foo:
73	73	* endbr64
74		− * subl 0x12345678, %r10d
75		− * jz foo
76		− * ud2
77		− * nop
	74	+ * subl 0x12345678, %eax
	75	+ * jne.32,pn foo+3
78	76	* foo:
79		− * osp nop3 # was endbr64
	77	+ * nopl -42(%rax) # was endbr64
80	78	* ... code here ...
81	79	* ret
82	80	*
@@ -86,9 +84,9 @@ · 3 unmodified lines
86	84	* indirect caller:
87	85	* lea foo(%rip), %r11
88	86	* ...
89		− * movl $0x12345678, %r10d
90		− * subl $16, %r11
91		− * nop4
	87	+ * movl $0x12345678, %eax
	88	+ * lea -0x10(%r11), %r11
	89	+ * nop5
92	90	* call *%r11
93	91	*
94	92	*/

arch/x86/include/asm/ibt.h+3−7

@@ -59,10 +59,10 @@ static __always_inline __attribute_const__ u32 gen_endbr(void) · 58 unmodified lines
59	59	static __always_inline __attribute_const__ u32 gen_endbr_poison(void)
60	60	{
61	61	/*
62		− * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
63		− * will be unique to (former) ENDBR sites.
	62	+ * 4 byte NOP that isn't NOP4, such that it will be unique to (former)
	63	+ * ENDBR sites. Additionally it carries UDB as immediate.
64	64	*/
65		− return 0x001f0f66; /* osp nopl (%rax) */
	65	+ return 0xd6401f0f; /* nopl -42(%rax) */
66	66	}
67	67
68	68	static inline bool __is_endbr(u32 val)
@@ -70,10 +70,6 @@ static inline bool __is_endbr(u32 val) · 1 unmodified line
70	70	if (val == gen_endbr_poison())
71	71	return true;
72	72
73		− /* See cfi_fineibt_bhi_preamble() */
74		− if (IS_ENABLED(CONFIG_FINEIBT_BHI) && val == 0x001f0ff5)
75		− return true;
76		−
77	73	val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
78	74	return val == gen_endbr();
79	75	}

arch/x86/include/asm/idtentry.h+2−7

@@ -460,17 +460,12 @@ __visible noinstr void func(struct pt_regs *regs, \ · 459 unmodified lines
460	460	#endif
461	461
462	462	void idt_install_sysvec(unsigned int n, const void *function);
463		−
464		−#ifdef CONFIG_X86_FRED
465	463	void fred_install_sysvec(unsigned int vector, const idtentry_t function);
466		−#else
467		−static inline void fred_install_sysvec(unsigned int vector, const idtentry_t function) { }
468		−#endif
469	464
470	465	#define sysvec_install(vector, function) { \
471		− if (cpu_feature_enabled(X86_FEATURE_FRED)) \
	466	+ if (IS_ENABLED(CONFIG_X86_FRED)) \
472	467	fred_install_sysvec(vector, function); \
473		− else \
	468	+ if (!cpu_feature_enabled(X86_FEATURE_FRED)) \
474	469	idt_install_sysvec(vector, asm_##function); \
475	470	}
476	471

arch/x86/include/asm/mshyperv.h+41−96

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/nmi.h>
7	7	#include <linux/msi.h>
8	8	#include <linux/io.h>
	9	+#include <linux/static_call.h>
9	10	#include <asm/nospec-branch.h>
10	11	#include <asm/paravirt.h>
11	12	#include <asm/msr.h>
@@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_reason(void) · 27 unmodified lines
39	40	return 0;
40	41	}
41	42
42		−#if IS_ENABLED(CONFIG_HYPERV)
43		−extern bool hyperv_paravisor_present;
	43	+extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
	44	+extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2);
	45	+extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2);
44	46
	47	+#if IS_ENABLED(CONFIG_HYPERV)
45	48	extern void *hv_hypercall_pg;
46	49
47	50	extern union hv_ghcb * __percpu *hv_ghcb_pg;
48	51
49	52	bool hv_isolation_type_snp(void);
50	53	bool hv_isolation_type_tdx(void);
51		−u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
	54	+
	55	+#ifdef CONFIG_X86_64
	56	+DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
	57	+#endif
52	58
53	59	/*
54	60	* DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA
@@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 control, void input, void output) · 10 unmodified lines
65	71	{
66	72	u64 input_address = input ? virt_to_phys(input) : 0;
67	73	u64 output_address = output ? virt_to_phys(output) : 0;
68		− u64 hv_status;
69	74
70	75	#ifdef CONFIG_X86_64
71		− if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
72		− return hv_tdx_hypercall(control, input_address, output_address);
73		−
74		− if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
75		− __asm__ __volatile__("mov %[output_address], %%r8\n"
76		− "vmmcall"
77		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
78		− "+c" (control), "+d" (input_address)
79		− : [output_address] "r" (output_address)
80		− : "cc", "memory", "r8", "r9", "r10", "r11");
81		− return hv_status;
82		− }
83		−
84		− if (!hv_hypercall_pg)
85		− return U64_MAX;
86		−
87		− __asm__ __volatile__("mov %[output_address], %%r8\n"
88		− CALL_NOSPEC
89		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
90		− "+c" (control), "+d" (input_address)
91		− : [output_address] "r" (output_address),
92		− THUNK_TARGET(hv_hypercall_pg)
93		− : "cc", "memory", "r8", "r9", "r10", "r11");
	76	+ return static_call_mod(hv_hypercall)(control, input_address, output_address);
94	77	#else
95	78	u32 input_address_hi = upper_32_bits(input_address);
96	79	u32 input_address_lo = lower_32_bits(input_address);
97	80	u32 output_address_hi = upper_32_bits(output_address);
98	81	u32 output_address_lo = lower_32_bits(output_address);
	82	+ u64 hv_status;
99	83
100	84	if (!hv_hypercall_pg)
101	85	return U64_MAX;
@@ -108,48 +92,30 @@ static inline u64 hv_do_hypercall(u64 control, void input, void output) · 6 unmodified lines
108	92	"D"(output_address_hi), "S"(output_address_lo),
109	93	THUNK_TARGET(hv_hypercall_pg)
110	94	: "cc", "memory");
111		−#endif /* !x86_64 */
112	95	return hv_status;
	96	+#endif /* !x86_64 */
113	97	}
114	98
115	99	/* Fast hypercall with 8 bytes of input and no output */
116	100	static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)
117	101	{
118		− u64 hv_status;
119		−
120	102	#ifdef CONFIG_X86_64
121		− if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
122		− return hv_tdx_hypercall(control, input1, 0);
123		−
124		− if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
125		− __asm__ __volatile__(
126		− "vmmcall"
127		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
128		− "+c" (control), "+d" (input1)
129		− :: "cc", "r8", "r9", "r10", "r11");
130		− } else {
131		− __asm__ __volatile__(CALL_NOSPEC
132		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
133		− "+c" (control), "+d" (input1)
134		− : THUNK_TARGET(hv_hypercall_pg)
135		− : "cc", "r8", "r9", "r10", "r11");
136		− }
	103	+ return static_call_mod(hv_hypercall)(control, input1, 0);
137	104	#else
138		− {
139		− u32 input1_hi = upper_32_bits(input1);
140		− u32 input1_lo = lower_32_bits(input1);
141		−
142		− __asm__ __volatile__ (CALL_NOSPEC
143		− : "=A"(hv_status),
144		− "+c"(input1_lo),
145		− ASM_CALL_CONSTRAINT
146		− : "A" (control),
147		− "b" (input1_hi),
148		− THUNK_TARGET(hv_hypercall_pg)
149		− : "cc", "edi", "esi");
150		− }
151		−#endif
	105	+ u32 input1_hi = upper_32_bits(input1);
	106	+ u32 input1_lo = lower_32_bits(input1);
	107	+ u64 hv_status;
	108	+
	109	+ __asm__ __volatile__ (CALL_NOSPEC
	110	+ : "=A"(hv_status),
	111	+ "+c"(input1_lo),
	112	+ ASM_CALL_CONSTRAINT
	113	+ : "A" (control),
	114	+ "b" (input1_hi),
	115	+ THUNK_TARGET(hv_hypercall_pg)
	116	+ : "cc", "edi", "esi");
152	117	return hv_status;
	118	+#endif
153	119	}
154	120
155	121	static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -162,45 +128,24 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) · 6 unmodified lines
162	128	/* Fast hypercall with 16 bytes of input */
163	129	static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)
164	130	{
165		− u64 hv_status;
166		−
167	131	#ifdef CONFIG_X86_64
168		− if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
169		− return hv_tdx_hypercall(control, input1, input2);
170		−
171		− if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
172		− __asm__ __volatile__("mov %[input2], %%r8\n"
173		− "vmmcall"
174		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
175		− "+c" (control), "+d" (input1)
176		− : [input2] "r" (input2)
177		− : "cc", "r8", "r9", "r10", "r11");
178		− } else {
179		− __asm__ __volatile__("mov %[input2], %%r8\n"
180		− CALL_NOSPEC
181		− : "=a" (hv_status), ASM_CALL_CONSTRAINT,
182		− "+c" (control), "+d" (input1)
183		− : [input2] "r" (input2),
184		− THUNK_TARGET(hv_hypercall_pg)
185		− : "cc", "r8", "r9", "r10", "r11");
186		− }
	132	+ return static_call_mod(hv_hypercall)(control, input1, input2);
187	133	#else
188		− {
189		− u32 input1_hi = upper_32_bits(input1);
190		− u32 input1_lo = lower_32_bits(input1);
191		− u32 input2_hi = upper_32_bits(input2);
192		− u32 input2_lo = lower_32_bits(input2);
193		−
194		− __asm__ __volatile__ (CALL_NOSPEC
195		− : "=A"(hv_status),
196		− "+c"(input1_lo), ASM_CALL_CONSTRAINT
197		− : "A" (control), "b" (input1_hi),
198		− "D"(input2_hi), "S"(input2_lo),
199		− THUNK_TARGET(hv_hypercall_pg)
200		− : "cc");
201		− }
202		−#endif
	134	+ u32 input1_hi = upper_32_bits(input1);
	135	+ u32 input1_lo = lower_32_bits(input1);
	136	+ u32 input2_hi = upper_32_bits(input2);
	137	+ u32 input2_lo = lower_32_bits(input2);
	138	+ u64 hv_status;
	139	+
	140	+ __asm__ __volatile__ (CALL_NOSPEC
	141	+ : "=A"(hv_status),
	142	+ "+c"(input1_lo), ASM_CALL_CONSTRAINT
	143	+ : "A" (control), "b" (input1_hi),
	144	+ "D"(input2_hi), "S"(input2_lo),
	145	+ THUNK_TARGET(hv_hypercall_pg)
	146	+ : "cc");
203	147	return hv_status;
	148	+#endif
204	149	}
205	150
206	151	static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)

arch/x86/include/asm/text-patching.h+13−7

@@ -178,9 +178,9 @@ void int3_emulate_ret(struct pt_regs *regs) · 177 unmodified lines
178	178	}
179	179
180	180	static __always_inline
181		−void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
	181	+bool __emulate_cc(unsigned long flags, u8 cc)
182	182	{
183		− static const unsigned long jcc_mask[6] = {
	183	+ static const unsigned long cc_mask[6] = {
184	184	[0] = X86_EFLAGS_OF,
185	185	[1] = X86_EFLAGS_CF,
186	186	[2] = X86_EFLAGS_ZF,
@@ -193,15 +193,21 @@ void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned lo · 6 unmodified lines
193	193	bool match;
194	194
195	195	if (cc < 0xc) {
196		− match = regs->flags & jcc_mask[cc >> 1];
	196	+ match = flags & cc_mask[cc >> 1];
197	197	} else {
198		− match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
199		− ((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
	198	+ match = ((flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
	199	+ ((flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
200	200	if (cc >= 0xe)
201		− match = match \|\| (regs->flags & X86_EFLAGS_ZF);
	201	+ match = match \|\| (flags & X86_EFLAGS_ZF);
202	202	}
203	203
204		− if ((match && !invert) \|\| (!match && invert))
	204	+ return (match && !invert) \|\| (!match && invert);
	205	+}
	206	+
	207	+static __always_inline
	208	+void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
	209	+{
	210	+ if (__emulate_cc(regs->flags, cc))
205	211	ip += disp;
206	212
207	213	int3_emulate_jmp(regs, ip);

arch/x86/kernel/alternative.c+176−116

@@ -147,10 +147,10 @@ static void its_init_thunk(void thunk, int reg) · 146 unmodified lines
147	147	/*
148	148	* When ITS uses indirect branch thunk the fineibt_paranoid
149	149	* caller sequence doesn't fit in the caller site. So put the
150		− * remaining part of the sequence (<ea> + JNE) into the ITS
	150	+ * remaining part of the sequence (UDB + JNE) into the ITS
151	151	* thunk.
152	152	*/
153		− bytes[i++] = 0xea; /* invalid instruction */
	153	+ bytes[i++] = 0xd6; /* UDB */
154	154	bytes[i++] = 0x75; /* JNE */
155	155	bytes[i++] = 0xfd;
156	156
@@ -163,7 +163,7 @@ static void its_init_thunk(void thunk, int reg) · 6 unmodified lines
163	163	reg -= 8;
164	164	}
165	165	bytes[i++] = 0xff;
166		− bytes[i++] = 0xe0 + reg; /* jmp reg /
	166	+ bytes[i++] = 0xe0 + reg; /* JMP reg /
167	167	bytes[i++] = 0xcc;
168	168
169	169	return thunk + offset;
@@ -713,20 +713,33 @@ static inline bool is_jcc32(struct insn *insn) · 543 unmodified lines
713	713	#if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL)
714	714
715	715	/*
716		− * CALL/JMP *%\reg
	716	+ * [CS]{,3} CALL/JMP %\reg [INT3]
717	717	*/
718		−static int emit_indirect(int op, int reg, u8 *bytes)
	718	+static int emit_indirect(int op, int reg, u8 *bytes, int len)
719	719	{
	720	+ int cs = 0, bp = 0;
720	721	int i = 0;
721	722	u8 modrm;
722	723
	724	+ /*
	725	+ * Set @len to the excess bytes after writing the instruction.
	726	+ */
	727	+ len -= 2 + (reg >= 8);
	728	+ WARN_ON_ONCE(len < 0);
	729	+
723	730	switch (op) {
724	731	case CALL_INSN_OPCODE:
725	732	modrm = 0x10; /* Reg = 2; CALL r/m */
	733	+ /*
	734	+ * Additional NOP is better than prefix decode penalty.
	735	+ */
	736	+ if (len <= 3)
	737	+ cs = len;
726	738	break;
727	739
728	740	case JMP32_INSN_OPCODE:
729	741	modrm = 0x20; /* Reg = 4; JMP r/m */
	742	+ bp = len;
730	743	break;
731	744
732	745	default:
@@ -734,6 +747,9 @@ static int emit_indirect(int op, int reg, u8 *bytes) · 1 unmodified line
734	747	return -1;
735	748	}
736	749
	750	+ while (cs--)
	751	+ bytes[i++] = 0x2e; /* CS-prefix */
	752	+
737	753	if (reg >= 8) {
738	754	bytes[i++] = 0x41; /* REX.B prefix */
739	755	reg -= 8;
@@ -745,6 +761,9 @@ static int emit_indirect(int op, int reg, u8 *bytes) · 5 unmodified lines
745	761	bytes[i++] = 0xff; /* opcode */
746	762	bytes[i++] = modrm;
747	763
	764	+ while (bp--)
	765	+ bytes[i++] = 0xcc; /* INT3 */
	766	+
748	767	return i;
749	768	}
750	769
@@ -918,20 +937,11 @@ static int patch_retpoline(void addr, struct insn insn, u8 *bytes) · 167 unmodified lines
918	937	return emit_its_trampoline(addr, insn, reg, bytes);
919	938	#endif
920	939
921		− ret = emit_indirect(op, reg, bytes + i);
	940	+ ret = emit_indirect(op, reg, bytes + i, insn->length - i);
922	941	if (ret < 0)
923	942	return ret;
924	943	i += ret;
925	944
926		− /*
927		− * The compiler is supposed to EMIT an INT3 after every unconditional
928		− * JMP instruction due to AMD BTC. However, if the compiler is too old
929		− * or MITIGATION_SLS isn't enabled, we still need an INT3 after
930		− * indirect JMPs even on Intel.
931		− */
932		− if (op == JMP32_INSN_OPCODE && i < insn->length)
933		− bytes[i++] = INT3_INSN_OPCODE;
934		−
935	945	for (; i < insn->length;)
936	946	bytes[i++] = BYTES_NOP1;
937	947
@@ -970,7 +980,7 @@ void __init_or_module noinline apply_retpolines(s32 start, s32 end) · 32 unmodified lines
970	980	case JMP32_INSN_OPCODE:
971	981	/* Check for cfi_paranoid + ITS */
972	982	dest = addr + insn.length + insn.immediate.value;
973		− if (dest[-1] == 0xea && (dest[0] & 0xf0) == 0x70) {
	983	+ if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {
974	984	WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
975	985	continue;
976	986	}
@@ -1177,6 +1187,7 @@ void __init_or_module apply_seal_endbr(s32 start, s32 end) { } · 200 unmodified lines
1177	1187	#endif
1178	1188
1179	1189	enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
	1190	+static bool cfi_debug __ro_after_init;
1180	1191
1181	1192	#ifdef CONFIG_FINEIBT_BHI
1182	1193	bool cfi_bhi __ro_after_init = false;
@@ -1259,6 +1270,8 @@ static __init int cfi_parse_cmdline(char *str) · 76 unmodified lines
1259	1270	} else if (!strcmp(str, "off")) {
1260	1271	cfi_mode = CFI_OFF;
1261	1272	cfi_rand = false;
	1273	+ } else if (!strcmp(str, "debug")) {
	1274	+ cfi_debug = true;
1262	1275	} else if (!strcmp(str, "kcfi")) {
1263	1276	cfi_mode = CFI_KCFI;
1264	1277	} else if (!strcmp(str, "fineibt")) {
@@ -1266,26 +1279,26 @@ static __init int cfi_parse_cmdline(char *str) · 1 unmodified line
1266	1279	} else if (!strcmp(str, "norand")) {
1267	1280	cfi_rand = false;
1268	1281	} else if (!strcmp(str, "warn")) {
1269		− pr_alert("CFI mismatch non-fatal!\n");
	1282	+ pr_alert("CFI: mismatch non-fatal!\n");
1270	1283	cfi_warn = true;
1271	1284	} else if (!strcmp(str, "paranoid")) {
1272	1285	if (cfi_mode == CFI_FINEIBT) {
1273	1286	cfi_paranoid = true;
1274	1287	} else {
1275		− pr_err("Ignoring paranoid; depends on fineibt.\n");
	1288	+ pr_err("CFI: ignoring paranoid; depends on fineibt.\n");
1276	1289	}
1277	1290	} else if (!strcmp(str, "bhi")) {
1278	1291	#ifdef CONFIG_FINEIBT_BHI
1279	1292	if (cfi_mode == CFI_FINEIBT) {
1280	1293	cfi_bhi = true;
1281	1294	} else {
1282		− pr_err("Ignoring bhi; depends on fineibt.\n");
	1295	+ pr_err("CFI: ignoring bhi; depends on fineibt.\n");
1283	1296	}
1284	1297	#else
1285		− pr_err("Ignoring bhi; depends on FINEIBT_BHI=y.\n");
	1298	+ pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n");
1286	1299	#endif
1287	1300	} else {
1288		− pr_err("Ignoring unknown cfi option (%s).", str);
	1301	+ pr_err("CFI: Ignoring unknown option (%s).", str);
1289	1302	}
1290	1303
1291	1304	str = next;
@@ -1300,9 +1313,9 @@ early_param("cfi", cfi_parse_cmdline); · 8 unmodified lines
1300	1313	*
1301	1314	* __cfi_\func: __cfi_\func:
1302	1315	* movl $0x12345678,%eax // 5 endbr64 // 4
1303		− * nop subl $0x12345678,%r10d // 7
1304		− * nop jne __cfi_\func+6 // 2
1305		− * nop nop3 // 3
	1316	+ * nop subl $0x12345678,%eax // 5
	1317	+ * nop jne.d32,pn \func+3 // 7
	1318	+ * nop
1306	1319	* nop
1307	1320	* nop
1308	1321	* nop
@@ -1311,34 +1324,44 @@ early_param("cfi", cfi_parse_cmdline); · 2 unmodified lines
1311	1324	* nop
1312	1325	* nop
1313	1326	* nop
	1327	+ * \func: \func:
	1328	+ * endbr64 nopl -42(%rax)
1314	1329	*
1315	1330	*
1316	1331	* caller: caller:
1317		− * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
	1332	+ * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%eax // 5
1318	1333	* addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4
1319		− * je 1f // 2 nop4 // 4
	1334	+ * je 1f // 2 nop5 // 5
1320	1335	* ud2 // 2
1321	1336	* 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6
1322	1337	*
	1338	+ *
	1339	+ * Notably, the FineIBT sequences are crafted such that branches are presumed
	1340	+ * non-taken. This is based on Agner Fog's optimization manual, which states:
	1341	+ *
	1342	+ * "Make conditional jumps most often not taken: The efficiency and throughput
	1343	+ * for not-taken branches is better than for taken branches on most
	1344	+ * processors. Therefore, it is good to place the most frequent branch first"
1323	1345	*/
1324	1346
1325	1347	/*
1326	1348	* <fineibt_preamble_start>:
1327	1349	* 0: f3 0f 1e fa endbr64
1328		− * 4: 41 81 <ea> 78 56 34 12 sub $0x12345678, %r10d
1329		− * b: 75 f9 jne 6 <fineibt_preamble_start+0x6>
1330		− * d: 0f 1f 00 nopl (%rax)
	1350	+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
	1351	+ * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
	1352	+ * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
1331	1353	*
1332		− * Note that the JNE target is the 0xEA byte inside the SUB, this decodes as
1333		− * (bad) on x86_64 and raises #UD.
	1354	+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
	1355	+ * UDB on x86_64 and raises #UD.
1334	1356	*/
1335	1357	asm( ".pushsection .rodata \n"
1336	1358	"fineibt_preamble_start: \n"
1337	1359	" endbr64 \n"
1338		− " subl $0x12345678, %r10d \n"
	1360	+ " subl $0x12345678, %eax \n"
1339	1361	"fineibt_preamble_bhi: \n"
1340		− " jne fineibt_preamble_start+6 \n"
1341		− ASM_NOP3
	1362	+ " cs jne.d32 fineibt_preamble_start+0x13 \n"
	1363	+ "#fineibt_func: \n"
	1364	+ " nopl -42(%rax) \n"
1342	1365	"fineibt_preamble_end: \n"
1343	1366	".popsection\n"
1344	1367	);
@@ -1349,20 +1372,20 @@ extern u8 fineibt_preamble_end[]; · 4 unmodified lines
1349	1372
1350	1373	#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1351	1374	#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
1352		−#define fineibt_preamble_ud 6
1353		−#define fineibt_preamble_hash 7
	1375	+#define fineibt_preamble_ud 0x13
	1376	+#define fineibt_preamble_hash 5
1354	1377
1355	1378	/*
1356	1379	* <fineibt_caller_start>:
1357		− * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d
1358		− * 6: 4d 8d 5b f0 lea -0x10(%r11), %r11
1359		− * a: 0f 1f 40 00 nopl 0x0(%rax)
	1380	+ * 0: b8 78 56 34 12 mov $0x12345678, %eax
	1381	+ * 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
	1382	+ * 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
1360	1383	*/
1361	1384	asm( ".pushsection .rodata \n"
1362	1385	"fineibt_caller_start: \n"
1363		− " movl $0x12345678, %r10d \n"
	1386	+ " movl $0x12345678, %eax \n"
1364	1387	" lea -0x10(%r11), %r11 \n"
1365		− ASM_NOP4
	1388	+ ASM_NOP5
1366	1389	"fineibt_caller_end: \n"
1367	1390	".popsection \n"
1368	1391	);
@@ -1371,7 +1394,7 @@ extern u8 fineibt_caller_start[]; · 2 unmodified lines
1371	1394	extern u8 fineibt_caller_end[];
1372	1395
1373	1396	#define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1374		−#define fineibt_caller_hash 2
	1397	+#define fineibt_caller_hash 1
1375	1398
1376	1399	#define fineibt_caller_jmp (fineibt_caller_size - 2)
1377	1400
@@ -1388,9 +1411,9 @@ extern u8 fineibt_caller_end[]; · 10 unmodified lines
1388	1411	* of adding a load.
1389	1412	*
1390	1413	* <fineibt_paranoid_start>:
1391		− * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d
1392		− * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d
1393		− * a: 4d 8d 5b <f0> lea -0x10(%r11), %r11
	1414	+ * 0: b8 78 56 34 12 mov $0x12345678, %eax
	1415	+ * 5: 41 3b 43 f5 cmp -0x11(%r11), %eax
	1416	+ * 9: 2e 4d 8d 5b <f0> cs lea -0x10(%r11), %r11
1394	1417	* e: 75 fd jne d <fineibt_paranoid_start+0xd>
1395	1418	* 10: 41 ff d3 call *%r11
1396	1419	* 13: 90 nop
@@ -1402,13 +1425,13 @@ extern u8 fineibt_caller_end[]; · 5 unmodified lines
1402	1425	*/
1403	1426	asm( ".pushsection .rodata \n"
1404	1427	"fineibt_paranoid_start: \n"
1405		− " movl $0x12345678, %r10d \n"
1406		− " cmpl -9(%r11), %r10d \n"
1407		− " lea -0x10(%r11), %r11 \n"
	1428	+ " mov $0x12345678, %eax \n"
	1429	+ " cmpl -11(%r11), %eax \n"
	1430	+ " cs lea -0x10(%r11), %r11 \n"
	1431	+ "#fineibt_caller_size: \n"
1408	1432	" jne fineibt_paranoid_start+0xd \n"
1409	1433	"fineibt_paranoid_ind: \n"
1410		− " call *%r11 \n"
1411		− " nop \n"
	1434	+ " cs call *%r11 \n"
1412	1435	"fineibt_paranoid_end: \n"
1413	1436	".popsection \n"
1414	1437	);
@@ -1520,51 +1543,67 @@ static int cfi_rand_preamble(s32 start, s32 end) · 105 unmodified lines
1520	1543	return 0;
1521	1544	}
1522	1545
	1546	+/*
	1547	+ * Inline the bhi-arity 1 case:
	1548	+ *
	1549	+ * __cfi_foo:
	1550	+ * 0: f3 0f 1e fa endbr64
	1551	+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
	1552	+ * 9: 49 0f 45 fa cmovne %rax, %rdi
	1553	+ * d: 2e 75 03 jne,pn foo+0x3
	1554	+ *
	1555	+ * foo:
	1556	+ * 10: 0f 1f 40 <d6> nopl -42(%rax)
	1557	+ *
	1558	+ * Notably, this scheme is incompatible with permissive CFI
	1559	+ * because the CMOVcc is unconditional and RDI will have been
	1560	+ * clobbered.
	1561	+ */
	1562	+asm( ".pushsection .rodata \n"
	1563	+ "fineibt_bhi1_start: \n"
	1564	+ " cmovne %rax, %rdi \n"
	1565	+ " cs jne fineibt_bhi1_func + 0x3 \n"
	1566	+ "fineibt_bhi1_func: \n"
	1567	+ " nopl -42(%rax) \n"
	1568	+ "fineibt_bhi1_end: \n"
	1569	+ ".popsection \n"
	1570	+);
	1571	+
	1572	+extern u8 fineibt_bhi1_start[];
	1573	+extern u8 fineibt_bhi1_end[];
	1574	+
	1575	+#define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
	1576	+
1523	1577	static void cfi_fineibt_bhi_preamble(void *addr, int arity)
1524	1578	{
	1579	+ u8 bytes[MAX_INSN_SIZE];
	1580	+
1525	1581	if (!arity)
1526	1582	return;
1527	1583
1528	1584	if (!cfi_warn && arity == 1) {
1529		− /*
1530		− * Crazy scheme to allow arity-1 inline:
1531		− *
1532		− * __cfi_foo:
1533		− * 0: f3 0f 1e fa endbr64
1534		− * 4: 41 81 <ea> 78 56 34 12 sub 0x12345678, %r10d
1535		− * b: 49 0f 45 fa cmovne %r10, %rdi
1536		− * f: 75 f5 jne __cfi_foo+6
1537		− * 11: 0f 1f 00 nopl (%rax)
1538		− *
1539		− * Code that direct calls to foo()+0, decodes the tail end as:
1540		− *
1541		− * foo:
1542		− * 0: f5 cmc
1543		− * 1: 0f 1f 00 nopl (%rax)
1544		− *
1545		− * which clobbers CF, but does not affect anything ABI
1546		− * wise.
1547		− *
1548		− * Notably, this scheme is incompatible with permissive CFI
1549		− * because the CMOVcc is unconditional and RDI will have been
1550		− * clobbered.
1551		− */
1552		− const u8 magic[9] = {
1553		− 0x49, 0x0f, 0x45, 0xfa,
1554		− 0x75, 0xf5,
1555		− BYTES_NOP3,
1556		− };
1557		−
1558		− text_poke_early(addr + fineibt_preamble_bhi, magic, 9);
1559		−
	1585	+ text_poke_early(addr + fineibt_preamble_bhi,
	1586	+ fineibt_bhi1_start, fineibt_bhi1_size);
1560	1587	return;
1561	1588	}
1562	1589
1563		− text_poke_early(addr + fineibt_preamble_bhi,
1564		− text_gen_insn(CALL_INSN_OPCODE,
1565		− addr + fineibt_preamble_bhi,
1566		− __bhi_args[arity]),
1567		− CALL_INSN_SIZE);
	1590	+ /*
	1591	+ * Replace the bytes at fineibt_preamble_bhi with a CALL instruction
	1592	+ * that lines up exactly with the end of the preamble, such that the
	1593	+ * return address will be foo+0.
	1594	+ *
	1595	+ * __cfi_foo:
	1596	+ * 0: f3 0f 1e fa endbr64
	1597	+ * 4: 2d 78 56 34 12 sub $0x12345678, %eax
	1598	+ * 9: 2e 2e e8 DD DD DD DD cs cs call __bhi_args[arity]
	1599	+ */
	1600	+ bytes[0] = 0x2e;
	1601	+ bytes[1] = 0x2e;
	1602	+ __text_gen_insn(bytes + 2, CALL_INSN_OPCODE,
	1603	+ addr + fineibt_preamble_bhi + 2,
	1604	+ __bhi_args[arity], CALL_INSN_SIZE);
	1605	+
	1606	+ text_poke_early(addr + fineibt_preamble_bhi, bytes, 7);
1568	1607	}
1569	1608
1570	1609	static int cfi_rewrite_preamble(s32 start, s32 end)
@@ -1655,8 +1694,6 @@ static int cfi_rewrite_callers(s32 start, s32 end) · 84 unmodified lines
1655	1694	{
1656	1695	s32 *s;
1657	1696
1658		− BUG_ON(fineibt_paranoid_size != 20);
1659		−
1660	1697	for (s = start; s < end; s++) {
1661	1698	void addr = (void )s + *s;
1662	1699	struct insn insn;
@@ -1696,8 +1733,9 @@ static int cfi_rewrite_callers(s32 start, s32 end) · 33 unmodified lines
1696	1733	emit_paranoid_trampoline(addr + fineibt_caller_size,
1697	1734	&insn, 11, bytes + fineibt_caller_size);
1698	1735	} else {
1699		− ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind);
1700		− if (WARN_ON_ONCE(ret != 3))
	1736	+ int len = fineibt_paranoid_size - fineibt_paranoid_ind;
	1737	+ ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len);
	1738	+ if (WARN_ON_ONCE(ret != len))
1701	1739	continue;
1702	1740	}
1703	1741
@@ -1707,13 +1745,20 @@ static int cfi_rewrite_callers(s32 start, s32 end) · 3 unmodified lines
1707	1745	return 0;
1708	1746	}
1709	1747
	1748	+#define pr_cfi_debug(X...) if (cfi_debug) pr_info(X)
	1749	+
	1750	+#define FINEIBT_WARN(_f, _v) \
	1751	+ WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
	1752	+
1710	1753	static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
1711	1754	s32 start_cfi, s32 end_cfi, bool builtin)
1712	1755	{
1713	1756	int ret;
1714	1757
1715		− if (WARN_ONCE(fineibt_preamble_size != 16,
1716		− "FineIBT preamble wrong size: %ld", fineibt_preamble_size))
	1758	+ if (FINEIBT_WARN(fineibt_preamble_size, 20) \|\|
	1759	+ FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) \|\|
	1760	+ FINEIBT_WARN(fineibt_caller_size, 14) \|\|
	1761	+ FINEIBT_WARN(fineibt_paranoid_size, 20))
1717	1762	return;
1718	1763
1719	1764	if (cfi_mode == CFI_AUTO) {
@@ -1734,6 +1779,7 @@ static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline, · 14 unmodified lines
1734	1779	* rewrite them. This disables all CFI. If this succeeds but any of the
1735	1780	* later stages fails, we're without CFI.
1736	1781	*/
	1782	+ pr_cfi_debug("CFI: disabling all indirect call checking\n");
1737	1783	ret = cfi_disable_callers(start_retpoline, end_retpoline);
1738	1784	if (ret)
1739	1785	goto err;
@@ -1744,43 +1790,53 @@ static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline, · 4 unmodified lines
1744	1790	cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
1745	1791	cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
1746	1792	}
	1793	+ pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed);
1747	1794
	1795	+ pr_cfi_debug("CFI: rehashing all preambles\n");
1748	1796	ret = cfi_rand_preamble(start_cfi, end_cfi);
1749	1797	if (ret)
1750	1798	goto err;
1751	1799
	1800	+ pr_cfi_debug("CFI: rehashing all indirect calls\n");
1752	1801	ret = cfi_rand_callers(start_retpoline, end_retpoline);
1753	1802	if (ret)
1754	1803	goto err;
	1804	+ } else {
	1805	+ pr_cfi_debug("CFI: rehashing disabled\n");
1755	1806	}
1756	1807
1757	1808	switch (cfi_mode) {
1758	1809	case CFI_OFF:
1759	1810	if (builtin)
1760		− pr_info("Disabling CFI\n");
	1811	+ pr_info("CFI: disabled\n");
1761	1812	return;
1762	1813
1763	1814	case CFI_KCFI:
	1815	+ pr_cfi_debug("CFI: re-enabling all indirect call checking\n");
1764	1816	ret = cfi_enable_callers(start_retpoline, end_retpoline);
1765	1817	if (ret)
1766	1818	goto err;
1767	1819
1768	1820	if (builtin)
1769		− pr_info("Using kCFI\n");
	1821	+ pr_info("CFI: Using %sretpoline kCFI\n",
	1822	+ cfi_rand ? "rehashed " : "");
1770	1823	return;
1771	1824
1772	1825	case CFI_FINEIBT:
	1826	+ pr_cfi_debug("CFI: adding FineIBT to all preambles\n");
1773	1827	/* place the FineIBT preamble at func()-16 */
1774	1828	ret = cfi_rewrite_preamble(start_cfi, end_cfi);
1775	1829	if (ret)
1776	1830	goto err;
1777	1831
1778	1832	/* rewrite the callers to target func()-16 */
	1833	+ pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n");
1779	1834	ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
1780	1835	if (ret)
1781	1836	goto err;
1782	1837
1783	1838	/* now that nobody targets func()+0, remove ENDBR there */
	1839	+ pr_cfi_debug("CFI: removing old endbr insns\n");
1784	1840	cfi_rewrite_endbr(start_cfi, end_cfi);
1785	1841
1786	1842	if (builtin) {
@@ -1823,11 +1879,11 @@ static void poison_cfi(void *addr) · 36 unmodified lines
1823	1879
1824	1880	/*
1825	1881	* __cfi_\func:
1826		− * osp nopl (%rax)
1827		− * subl $0, %r10d
1828		− * jz 1f
1829		− * ud2
1830		− * 1: nop
	1882	+ * nopl -42(%rax)
	1883	+ * sub $0, %eax
	1884	+ * jne \func+3
	1885	+ * \func:
	1886	+ * nopl -42(%rax)
1831	1887	*/
1832	1888	poison_endbr(addr);
1833	1889	poison_hash(addr + fineibt_preamble_hash);
@@ -1853,12 +1909,14 @@ static void poison_cfi(void *addr) · 19 unmodified lines
1853	1909	}
1854	1910	}
1855	1911
	1912	+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
	1913	+
1856	1914	/*
1857		− * When regs->ip points to a 0xEA byte in the FineIBT preamble,
	1915	+ * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
1858	1916	* return true and fill out target and type.
1859	1917	*
1860	1918	* We check the preamble by checking for the ENDBR instruction relative to the
1861		− * 0xEA instruction.
	1919	+ * UDB instruction.
1862	1920	*/
1863	1921	static bool decode_fineibt_preamble(struct pt_regs regs, unsigned long target, u32 *type)
1864	1922	{
@@ -1868,10 +1926,10 @@ static bool decode_fineibt_preamble(struct pt_regs regs, unsigned long target, · 3 unmodified lines
1868	1926	if (!exact_endbr((void *)addr))
1869	1927	return false;
1870	1928
1871		− *target = addr + fineibt_preamble_size;
	1929	+ *target = addr + fineibt_prefix_size;
1872	1930
1873	1931	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
1874		− *type = (u32)regs->r10 + hash;
	1932	+ *type = (u32)regs->ax + hash;
1875	1933
1876	1934	/*
1877	1935	* Since regs->ip points to the middle of an instruction; it cannot
@@ -1909,12 +1967,12 @@ static bool decode_fineibt_bhi(struct pt_regs regs, unsigned long target, u32 · 31 unmodified lines
1909	1967	__get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);
1910	1968	*target = addr;
1911	1969
1912		− addr -= fineibt_preamble_size;
	1970	+ addr -= fineibt_prefix_size;
1913	1971	if (!exact_endbr((void *)addr))
1914	1972	return false;
1915	1973
1916	1974	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
1917		− *type = (u32)regs->r10 + hash;
	1975	+ *type = (u32)regs->ax + hash;
1918	1976
1919	1977	/*
1920	1978	* The UD2 sites are constructed with a RET immediately following,
@@ -1931,7 +1989,7 @@ static bool is_paranoid_thunk(unsigned long addr) · 10 unmodified lines
1931	1989	u32 thunk;
1932	1990
1933	1991	__get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault);
1934		− return (thunk & 0x00FFFFFF) == 0xfd75ea;
	1992	+ return (thunk & 0x00FFFFFF) == 0xfd75d6;
1935	1993
1936	1994	Efault:
1937	1995	return false;
@@ -1939,8 +1997,7 @@ Efault: · 1 unmodified line
1939	1997
1940	1998	/*
1941	1999	* regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
1942		− * sequence, or to an invalid instruction (0xea) + Jcc.d8 for cfi_paranoid + ITS
1943		− * thunk.
	2000	+ * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk.
1944	2001	*/
1945	2002	static bool decode_fineibt_paranoid(struct pt_regs regs, unsigned long target, u32 *type)
1946	2003	{
@@ -1950,8 +2007,8 @@ static bool decode_fineibt_paranoid(struct pt_regs regs, unsigned long target, · 3 unmodified lines
1950	2007	return false;
1951	2008
1952	2009	if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {
1953		− *target = regs->r11 + fineibt_preamble_size;
1954		− *type = regs->r10;
	2010	+ *target = regs->r11 + fineibt_prefix_size;
	2011	+ *type = regs->ax;
1955	2012
1956	2013	/*
1957	2014	* Since the trapping instruction is the exact, but LOCK prefixed,
@@ -1963,14 +2020,14 @@ static bool decode_fineibt_paranoid(struct pt_regs regs, unsigned long target, · 5 unmodified lines
1963	2020	/*
1964	2021	* The cfi_paranoid + ITS thunk combination results in:
1965	2022	*
1966		− * 0: 41 ba 78 56 34 12 mov $0x12345678, %r10d
1967		− * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d
1968		− * a: 4d 8d 5b f0 lea -0x10(%r11), %r11
	2023	+ * 0: b8 78 56 34 12 mov $0x12345678, %eax
	2024	+ * 5: 41 3b 43 f7 cmp -11(%r11), %eax
	2025	+ * a: 2e 3d 8d 5b f0 cs lea -0x10(%r11), %r11
1969	2026	* e: 2e e8 XX XX XX XX cs call __x86_indirect_paranoid_thunk_r11
1970	2027	*
1971	2028	* Where the paranoid_thunk looks like:
1972	2029	*
1973		− * 1d: <ea> (bad)
	2030	+ * 1d: <d6> udb
1974	2031	* __x86_indirect_paranoid_thunk_r11:
1975	2032	* 1e: 75 fd jne 1d
1976	2033	* __x86_indirect_its_thunk_r11:
@@ -1979,8 +2036,8 @@ static bool decode_fineibt_paranoid(struct pt_regs regs, unsigned long target, · 2 unmodified lines
1979	2036	*
1980	2037	*/
1981	2038	if (is_paranoid_thunk(regs->ip)) {
1982		− *target = regs->r11 + fineibt_preamble_size;
1983		− *type = regs->r10;
	2039	+ *target = regs->r11 + fineibt_prefix_size;
	2040	+ *type = regs->ax;
1984	2041
1985	2042	regs->ip = *target;
1986	2043	return true;
@@ -2005,6 +2062,8 @@ bool decode_fineibt_insn(struct pt_regs regs, unsigned long target, u32 *type) · 18 unmodified lines
2005	2062	static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
2006	2063	s32 start_cfi, s32 end_cfi, bool builtin)
2007	2064	{
	2065	+ if (IS_ENABLED(CONFIG_CFI) && builtin)
	2066	+ pr_info("CFI: Using standard kCFI\n");
2008	2067	}
2009	2068
2010	2069	#ifdef CONFIG_X86_KERNEL_IBT
@@ -2321,6 +2380,7 @@ void __init alternative_instructions(void) · 310 unmodified lines
2321	2380
2322	2381	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,
2323	2382	__cfi_sites, __cfi_sites_end, true);
	2383	+ cfi_debug = false;
2324	2384
2325	2385	/*
2326	2386	* Rewrite the retpolines, must be done before alternatives since

arch/x86/kernel/asm-offsets.c+1−0

@@ -102,6 +102,7 @@ static void __used common(void) · 101 unmodified lines
102	102
103	103	BLANK();
104	104	DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
	105	+ OFFSET(C_PTREGS_SIZE, pt_regs, orig_ax);
105	106
106	107	/* TLB state for the entry code */
107	108	OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);

arch/x86/kernel/cfi.c+1−1

@@ -27,7 +27,7 @@ static bool decode_cfi_insn(struct pt_regs regs, unsigned long target, · 26 unmodified lines
27	27	* for indirect call checks:
28	28	*
29	29	* movl -<id>, %r10d ; 6 bytes
30		− * addl -4(%reg), %r10d ; 4 bytes
	30	+ * addl -<pos>(%reg), %r10d; 4 bytes
31	31	* je .Ltmp1 ; 2 bytes
32	32	* ud2 ; <- regs->ip
33	33	* .Ltmp1:

arch/x86/kernel/cpu/mshyperv.c+13−6

@@ -38,10 +38,6 @@ · 37 unmodified lines
38	38	bool hv_nested;
39	39	struct ms_hyperv_info ms_hyperv;
40	40
41		−/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */
42		−bool hyperv_paravisor_present __ro_after_init;
43		−EXPORT_SYMBOL_GPL(hyperv_paravisor_present);
44		−
45	41	#if IS_ENABLED(CONFIG_HYPERV)
46	42	static inline unsigned int hv_get_nested_msr(unsigned int reg)
47	43	{
@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc_pg_clock(void) · 240 unmodified lines
288	284	old_restore_sched_clock_state = x86_platform.restore_sched_clock_state;
289	285	x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state;
290	286	}
	287	+
	288	+#ifdef CONFIG_X86_64
	289	+DEFINE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
	290	+EXPORT_STATIC_CALL_TRAMP_GPL(hv_hypercall);
	291	+#define hypercall_update(hc) static_call_update(hv_hypercall, hc)
	292	+#endif
291	293	#endif /* CONFIG_HYPERV */
292	294
	295	+#ifndef hypercall_update
	296	+#define hypercall_update(hc) (void)hc
	297	+#endif
	298	+
293	299	static uint32_t __init ms_hyperv_platform(void)
294	300	{
295	301	u32 eax;
@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platform(void) · 188 unmodified lines
484	490	ms_hyperv.shared_gpa_boundary =
485	491	BIT_ULL(ms_hyperv.shared_gpa_boundary_bits);
486	492
487		− hyperv_paravisor_present = !!ms_hyperv.paravisor_present;
488		−
489	493	pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n",
490	494	ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);
491	495
492	496
493	497	if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) {
494	498	static_branch_enable(&isolation_type_snp);
	499	+ if (!ms_hyperv.paravisor_present)
	500	+ hypercall_update(hv_snp_hypercall);
495	501	} else if (hv_get_isolation_type() == HV_ISOLATION_TYPE_TDX) {
496	502	static_branch_enable(&isolation_type_tdx);
497	503
@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platform(void) · 1 unmodified line
499	505	ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED;
500	506
501	507	if (!ms_hyperv.paravisor_present) {
	508	+ hypercall_update(hv_tdx_hypercall);
502	509	/*
503	510	* Mark the Hyper-V TSC page feature as disabled
504	511	* in a TDX VM without paravisor so that the

arch/x86/kernel/irqinit.c+4−2

@@ -97,9 +97,11 @@ void __init native_init_IRQ(void) · 96 unmodified lines
97	97	/* Execute any quirks before the call gates are initialised: */
98	98	x86_init.irqs.pre_vector_init();
99	99
100		− if (cpu_feature_enabled(X86_FEATURE_FRED))
	100	+ /* FRED's IRQ path may be used even if FRED isn't fully enabled. */
	101	+ if (IS_ENABLED(CONFIG_X86_FRED))
101	102	fred_complete_exception_setup();
102		− else
	103	+
	104	+ if (!cpu_feature_enabled(X86_FEATURE_FRED))
103	105	idt_setup_apic_and_irq_gates();
104	106
105	107	lapic_assign_system_vectors();

arch/x86/kernel/machine_kexec_64.c+4−0

@@ -479,6 +479,10 @@ void __nocfi machine_kexec(struct kimage *image) · 478 unmodified lines
479	479
480	480	__ftrace_enabled_restore(save_ftrace_enabled);
481	481	}
	482	+/*
	483	+ * Handover to the next kernel, no CFI concern.
	484	+ */
	485	+ANNOTATE_NOCFI_SYM(machine_kexec);
482	486
483	487	/* arch-dependent functionality related to kexec file-based syscall */
484	488

arch/x86/kernel/traps.c+4−4

@@ -97,7 +97,7 @@ __always_inline int is_valid_bugaddr(unsigned long addr) · 96 unmodified lines
97	97	* Check for UD1 or UD2, accounting for Address Size Override Prefixes.
98	98	* If it's a UD1, further decode to determine its use:
99	99	*
100		− * FineIBT: ea (bad)
	100	+ * FineIBT: d6 udb
101	101	* FineIBT: f0 75 f9 lock jne . - 6
102	102	* UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax
103	103	* UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax
@@ -130,9 +130,9 @@ __always_inline int decode_bug(unsigned long addr, s32 imm, int len) · 26 unmodified lines
130	130	WARN_ON_ONCE(!lock);
131	131	return BUG_LOCK;
132	132
133		− case 0xea:
	133	+ case 0xd6:
134	134	*len = addr - start;
135		− return BUG_EA;
	135	+ return BUG_UDB;
136	136
137	137	case OPCODE_ESCAPE:
138	138	break;
@@ -341,7 +341,7 @@ static noinstr bool handle_bug(struct pt_regs *regs) · 202 unmodified lines
341	341	}
342	342	fallthrough;
343	343
344		− case BUG_EA:
	344	+ case BUG_UDB:
345	345	case BUG_LOCK:
346	346	if (handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
347	347	handled = true;

arch/x86/kvm/Kconfig+1−0

@@ -96,6 +96,7 @@ config KVM_SW_PROTECTED_VM · 95 unmodified lines
96	96	config KVM_INTEL
97	97	tristate "KVM for Intel (and compatible) processors support"
98	98	depends on KVM && IA32_FEAT_CTL
	99	+ select X86_FRED if X86_64
99	100	help
100	101	Provides support for KVM on processors equipped with Intel's VT
101	102	extensions, a.k.a. Virtual Machine Extensions (VMX).

arch/x86/kvm/emulate.c+232−318

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	26	#include <asm/debugreg.h>
27	27	#include <asm/nospec-branch.h>
28	28	#include <asm/ibt.h>
	29	+#include <asm/text-patching.h>
29	30
30	31	#include "x86.h"
31	32	#include "tss.h"
@@ -166,7 +167,6 @@ · 134 unmodified lines
166	167	#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
167	168	#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
168	169	#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
169		−#define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
170	170	#define NoWrite ((u64)1 << 45) /* No writeback */
171	171	#define SrcWrite ((u64)1 << 46) /* Write back src operand */
172	172	#define NoMod ((u64)1 << 47) /* Mod field is ignored */
@@ -203,7 +203,6 @@ struct opcode { · 30 unmodified lines
203	203	const struct escape *esc;
204	204	const struct instr_dual *idual;
205	205	const struct mode_dual *mdual;
206		− void (fastop)(struct fastop fake);
207	206	} u;
208	207	int (check_perm)(struct x86_emulate_ctxt ctxt);
209	208	};
@@ -267,186 +266,130 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt) · 57 unmodified lines
267	266	X86_EFLAGS_PF\|X86_EFLAGS_CF)
268	267
269	268	#ifdef CONFIG_X86_64
270		−#define ON64(x) x
	269	+#define ON64(x...) x
271	270	#else
272		−#define ON64(x)
	271	+#define ON64(x...)
273	272	#endif
274	273
275		−/*
276		− * fastop functions have a special calling convention:
277		− *
278		− * dst: rax (in/out)
279		− * src: rdx (in/out)
280		− * src2: rcx (in)
281		− * flags: rflags (in/out)
282		− * ex: rsi (in:fastop pointer, out:zero if exception)
283		− *
284		− * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
285		− * different operand sizes can be reached by calculation, rather than a jump
286		− * table (which would be bigger than the code).
287		− *
288		− * The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR
289		− * and 1 for the straight line speculation INT3, leaves 7 bytes for the
290		− * body of the function. Currently none is larger than 4.
291		− */
292		−static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
293		−
294		−#define FASTOP_SIZE 16
295		−
296		−#define __FOP_FUNC(name) \
297		− ".align " __stringify(FASTOP_SIZE) " \n\t" \
298		− ".type " name ", @function \n\t" \
299		− name ":\n\t" \
300		− ASM_ENDBR \
301		− IBT_NOSEAL(name)
302		−
303		−#define FOP_FUNC(name) \
304		− __FOP_FUNC(#name)
305		−
306		−#define __FOP_RET(name) \
307		− "11: " ASM_RET \
308		− ".size " name ", .-" name "\n\t"
309		−
310		−#define FOP_RET(name) \
311		− __FOP_RET(#name)
312		−
313		−#define __FOP_START(op, align) \
314		− extern void em_##op(struct fastop *fake); \
315		− asm(".pushsection .text, \"ax\" \n\t" \
316		− ".global em_" #op " \n\t" \
317		− ".align " __stringify(align) " \n\t" \
318		− "em_" #op ":\n\t"
319		−
320		−#define FOP_START(op) __FOP_START(op, FASTOP_SIZE)
321		−
322		−#define FOP_END \
323		− ".popsection")
324		−
325		−#define __FOPNOP(name) \
326		− __FOP_FUNC(name) \
327		− __FOP_RET(name)
328		−
329		−#define FOPNOP() \
330		− __FOPNOP(__stringify(__UNIQUE_ID(nop)))
331		−
332		−#define FOP1E(op, dst) \
333		− __FOP_FUNC(#op "_" #dst) \
334		− "10: " #op " %" #dst " \n\t" \
335		− __FOP_RET(#op "_" #dst)
336		−
337		−#define FOP1EEX(op, dst) \
338		− FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)
339		−
340		−#define FASTOP1(op) \
341		− FOP_START(op) \
342		− FOP1E(op##b, al) \
343		− FOP1E(op##w, ax) \
344		− FOP1E(op##l, eax) \
345		− ON64(FOP1E(op##q, rax)) \
346		− FOP_END
347		−
348		−/* 1-operand, using src2 (for MUL/DIV r/m) */
349		−#define FASTOP1SRC2(op, name) \
350		− FOP_START(name) \
351		− FOP1E(op, cl) \
352		− FOP1E(op, cx) \
353		− FOP1E(op, ecx) \
354		− ON64(FOP1E(op, rcx)) \
355		− FOP_END
356		−
357		−/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
358		−#define FASTOP1SRC2EX(op, name) \
359		− FOP_START(name) \
360		− FOP1EEX(op, cl) \
361		− FOP1EEX(op, cx) \
362		− FOP1EEX(op, ecx) \
363		− ON64(FOP1EEX(op, rcx)) \
364		− FOP_END
365		−
366		−#define FOP2E(op, dst, src) \
367		− __FOP_FUNC(#op "_" #dst "_" #src) \
368		− #op " %" #src ", %" #dst " \n\t" \
369		− __FOP_RET(#op "_" #dst "_" #src)
370		−
371		−#define FASTOP2(op) \
372		− FOP_START(op) \
373		− FOP2E(op##b, al, dl) \
374		− FOP2E(op##w, ax, dx) \
375		− FOP2E(op##l, eax, edx) \
376		− ON64(FOP2E(op##q, rax, rdx)) \
377		− FOP_END
378		−
379		−/* 2 operand, word only */
380		−#define FASTOP2W(op) \
381		− FOP_START(op) \
382		− FOPNOP() \
383		− FOP2E(op##w, ax, dx) \
384		− FOP2E(op##l, eax, edx) \
385		− ON64(FOP2E(op##q, rax, rdx)) \
386		− FOP_END
387		−
388		−/* 2 operand, src is CL */
389		−#define FASTOP2CL(op) \
390		− FOP_START(op) \
391		− FOP2E(op##b, al, cl) \
392		− FOP2E(op##w, ax, cl) \
393		− FOP2E(op##l, eax, cl) \
394		− ON64(FOP2E(op##q, rax, cl)) \
395		− FOP_END
396		−
397		−/* 2 operand, src and dest are reversed */
398		−#define FASTOP2R(op, name) \
399		− FOP_START(name) \
400		− FOP2E(op##b, dl, al) \
401		− FOP2E(op##w, dx, ax) \
402		− FOP2E(op##l, edx, eax) \
403		− ON64(FOP2E(op##q, rdx, rax)) \
404		− FOP_END
405		−
406		−#define FOP3E(op, dst, src, src2) \
407		− __FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408		− #op " %" #src2 ", %" #src ", %" #dst " \n\t"\
409		− __FOP_RET(#op "_" #dst "_" #src "_" #src2)
410		−
411		−/* 3-operand, word-only, src2=cl */
412		−#define FASTOP3WCL(op) \
413		− FOP_START(op) \
414		− FOPNOP() \
415		− FOP3E(op##w, ax, dx, cl) \
416		− FOP3E(op##l, eax, edx, cl) \
417		− ON64(FOP3E(op##q, rax, rdx, cl)) \
418		− FOP_END
419		−
420		−/* Special case for SETcc - 1 instruction per cc */
421		−#define FOP_SETCC(op) \
422		− FOP_FUNC(op) \
423		− #op " %al \n\t" \
424		− FOP_RET(op)
425		−
426		−FOP_START(setcc)
427		−FOP_SETCC(seto)
428		−FOP_SETCC(setno)
429		−FOP_SETCC(setc)
430		−FOP_SETCC(setnc)
431		−FOP_SETCC(setz)
432		−FOP_SETCC(setnz)
433		−FOP_SETCC(setbe)
434		−FOP_SETCC(setnbe)
435		−FOP_SETCC(sets)
436		−FOP_SETCC(setns)
437		−FOP_SETCC(setp)
438		−FOP_SETCC(setnp)
439		−FOP_SETCC(setl)
440		−FOP_SETCC(setnl)
441		−FOP_SETCC(setle)
442		−FOP_SETCC(setnle)
443		−FOP_END;
444		−
445		−FOP_START(salc)
446		−FOP_FUNC(salc)
447		−"pushf; sbb %al, %al; popf \n\t"
448		−FOP_RET(salc)
449		−FOP_END;
	274	+#define EM_ASM_START(op) \
	275	+static int em_##op(struct x86_emulate_ctxt *ctxt) \
	276	+{ \
	277	+ unsigned long flags = (ctxt->eflags & EFLAGS_MASK) \| X86_EFLAGS_IF; \
	278	+ int bytes = 1, ok = 1; \
	279	+ if (!(ctxt->d & ByteOp)) \
	280	+ bytes = ctxt->dst.bytes; \
	281	+ switch (bytes) {
	282	+
	283	+#define __EM_ASM(str) \
	284	+ asm("push %[flags]; popf \n\t" \
	285	+ "10: " str \
	286	+ "pushf; pop %[flags] \n\t" \
	287	+ "11: \n\t" \
	288	+ : "+a" (ctxt->dst.val), \
	289	+ "+d" (ctxt->src.val), \
	290	+ [flags] "+D" (flags), \
	291	+ "+S" (ok) \
	292	+ : "c" (ctxt->src2.val))
	293	+
	294	+#define __EM_ASM_1(op, dst) \
	295	+ __EM_ASM(#op " %%" #dst " \n\t")
	296	+
	297	+#define __EM_ASM_1_EX(op, dst) \
	298	+ __EM_ASM(#op " %%" #dst " \n\t" \
	299	+ _ASM_EXTABLE_TYPE_REG(10b, 11f, EX_TYPE_ZERO_REG, %%esi))
	300	+
	301	+#define __EM_ASM_2(op, dst, src) \
	302	+ __EM_ASM(#op " %%" #src ", %%" #dst " \n\t")
	303	+
	304	+#define __EM_ASM_3(op, dst, src, src2) \
	305	+ __EM_ASM(#op " %%" #src2 ", %%" #src ", %%" #dst " \n\t")
	306	+
	307	+#define EM_ASM_END \
	308	+ } \
	309	+ ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) \| (flags & EFLAGS_MASK); \
	310	+ return !ok ? emulate_de(ctxt) : X86EMUL_CONTINUE; \
	311	+}
	312	+
	313	+/* 1-operand, using "a" (dst) */
	314	+#define EM_ASM_1(op) \
	315	+ EM_ASM_START(op) \
	316	+ case 1: __EM_ASM_1(op##b, al); break; \
	317	+ case 2: __EM_ASM_1(op##w, ax); break; \
	318	+ case 4: __EM_ASM_1(op##l, eax); break; \
	319	+ ON64(case 8: __EM_ASM_1(op##q, rax); break;) \
	320	+ EM_ASM_END
	321	+
	322	+/* 1-operand, using "c" (src2) */
	323	+#define EM_ASM_1SRC2(op, name) \
	324	+ EM_ASM_START(name) \
	325	+ case 1: __EM_ASM_1(op##b, cl); break; \
	326	+ case 2: __EM_ASM_1(op##w, cx); break; \
	327	+ case 4: __EM_ASM_1(op##l, ecx); break; \
	328	+ ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \
	329	+ EM_ASM_END
	330	+
	331	+/* 1-operand, using "c" (src2) with exception */
	332	+#define EM_ASM_1SRC2EX(op, name) \
	333	+ EM_ASM_START(name) \
	334	+ case 1: __EM_ASM_1_EX(op##b, cl); break; \
	335	+ case 2: __EM_ASM_1_EX(op##w, cx); break; \
	336	+ case 4: __EM_ASM_1_EX(op##l, ecx); break; \
	337	+ ON64(case 8: __EM_ASM_1_EX(op##q, rcx); break;) \
	338	+ EM_ASM_END
	339	+
	340	+/* 2-operand, using "a" (dst), "d" (src) */
	341	+#define EM_ASM_2(op) \
	342	+ EM_ASM_START(op) \
	343	+ case 1: __EM_ASM_2(op##b, al, dl); break; \
	344	+ case 2: __EM_ASM_2(op##w, ax, dx); break; \
	345	+ case 4: __EM_ASM_2(op##l, eax, edx); break; \
	346	+ ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \
	347	+ EM_ASM_END
	348	+
	349	+/* 2-operand, reversed */
	350	+#define EM_ASM_2R(op, name) \
	351	+ EM_ASM_START(name) \
	352	+ case 1: __EM_ASM_2(op##b, dl, al); break; \
	353	+ case 2: __EM_ASM_2(op##w, dx, ax); break; \
	354	+ case 4: __EM_ASM_2(op##l, edx, eax); break; \
	355	+ ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \
	356	+ EM_ASM_END
	357	+
	358	+/* 2-operand, word only (no byte op) */
	359	+#define EM_ASM_2W(op) \
	360	+ EM_ASM_START(op) \
	361	+ case 1: break; \
	362	+ case 2: __EM_ASM_2(op##w, ax, dx); break; \
	363	+ case 4: __EM_ASM_2(op##l, eax, edx); break; \
	364	+ ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \
	365	+ EM_ASM_END
	366	+
	367	+/* 2-operand, using "a" (dst) and CL (src2) */
	368	+#define EM_ASM_2CL(op) \
	369	+ EM_ASM_START(op) \
	370	+ case 1: __EM_ASM_2(op##b, al, cl); break; \
	371	+ case 2: __EM_ASM_2(op##w, ax, cl); break; \
	372	+ case 4: __EM_ASM_2(op##l, eax, cl); break; \
	373	+ ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \
	374	+ EM_ASM_END
	375	+
	376	+/* 3-operand, using "a" (dst), "d" (src) and CL (src2) */
	377	+#define EM_ASM_3WCL(op) \
	378	+ EM_ASM_START(op) \
	379	+ case 1: break; \
	380	+ case 2: __EM_ASM_3(op##w, ax, dx, cl); break; \
	381	+ case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \
	382	+ ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \
	383	+ EM_ASM_END
	384	+
	385	+static int em_salc(struct x86_emulate_ctxt *ctxt)
	386	+{
	387	+ /*
	388	+ * Set AL 0xFF if CF is set, or 0x00 when clear.
	389	+ */
	390	+ ctxt->dst.val = 0xFF * !!(ctxt->eflags & X86_EFLAGS_CF);
	391	+ return X86EMUL_CONTINUE;
	392	+}
450	393
451	394	/*
452	395	* XXX: inoutclob user must know where the argument is being expanded.
@@ -1007,56 +950,55 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt, · 554 unmodified lines
1007	950	return rc;
1008	951	}
1009	952
1010		−FASTOP2(add);
1011		−FASTOP2(or);
1012		−FASTOP2(adc);
1013		−FASTOP2(sbb);
1014		−FASTOP2(and);
1015		−FASTOP2(sub);
1016		−FASTOP2(xor);
1017		−FASTOP2(cmp);
1018		−FASTOP2(test);
1019		−
1020		−FASTOP1SRC2(mul, mul_ex);
1021		−FASTOP1SRC2(imul, imul_ex);
1022		−FASTOP1SRC2EX(div, div_ex);
1023		−FASTOP1SRC2EX(idiv, idiv_ex);
1024		−
1025		−FASTOP3WCL(shld);
1026		−FASTOP3WCL(shrd);
1027		−
1028		−FASTOP2W(imul);
1029		−
1030		−FASTOP1(not);
1031		−FASTOP1(neg);
1032		−FASTOP1(inc);
1033		−FASTOP1(dec);
1034		−
1035		−FASTOP2CL(rol);
1036		−FASTOP2CL(ror);
1037		−FASTOP2CL(rcl);
1038		−FASTOP2CL(rcr);
1039		−FASTOP2CL(shl);
1040		−FASTOP2CL(shr);
1041		−FASTOP2CL(sar);
1042		−
1043		−FASTOP2W(bsf);
1044		−FASTOP2W(bsr);
1045		−FASTOP2W(bt);
1046		−FASTOP2W(bts);
1047		−FASTOP2W(btr);
1048		−FASTOP2W(btc);
1049		−
1050		−FASTOP2(xadd);
1051		−
1052		−FASTOP2R(cmp, cmp_r);
	953	+EM_ASM_2(add);
	954	+EM_ASM_2(or);
	955	+EM_ASM_2(adc);
	956	+EM_ASM_2(sbb);
	957	+EM_ASM_2(and);
	958	+EM_ASM_2(sub);
	959	+EM_ASM_2(xor);
	960	+EM_ASM_2(cmp);
	961	+EM_ASM_2(test);
	962	+EM_ASM_2(xadd);
	963	+
	964	+EM_ASM_1SRC2(mul, mul_ex);
	965	+EM_ASM_1SRC2(imul, imul_ex);
	966	+EM_ASM_1SRC2EX(div, div_ex);
	967	+EM_ASM_1SRC2EX(idiv, idiv_ex);
	968	+
	969	+EM_ASM_3WCL(shld);
	970	+EM_ASM_3WCL(shrd);
	971	+
	972	+EM_ASM_2W(imul);
	973	+
	974	+EM_ASM_1(not);
	975	+EM_ASM_1(neg);
	976	+EM_ASM_1(inc);
	977	+EM_ASM_1(dec);
	978	+
	979	+EM_ASM_2CL(rol);
	980	+EM_ASM_2CL(ror);
	981	+EM_ASM_2CL(rcl);
	982	+EM_ASM_2CL(rcr);
	983	+EM_ASM_2CL(shl);
	984	+EM_ASM_2CL(shr);
	985	+EM_ASM_2CL(sar);
	986	+
	987	+EM_ASM_2W(bsf);
	988	+EM_ASM_2W(bsr);
	989	+EM_ASM_2W(bt);
	990	+EM_ASM_2W(bts);
	991	+EM_ASM_2W(btr);
	992	+EM_ASM_2W(btc);
	993	+
	994	+EM_ASM_2R(cmp, cmp_r);
1053	995
1054	996	static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1055	997	{
1056	998	/* If src is zero, do not writeback, but update flags */
1057	999	if (ctxt->src.val == 0)
1058	1000	ctxt->dst.type = OP_NONE;
1059		− return fastop(ctxt, em_bsf);
	1001	+ return em_bsf(ctxt);
1060	1002	}
1061	1003
1062	1004	static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
@@ -1064,18 +1006,12 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt) · 1 unmodified line
1064	1006	/* If src is zero, do not writeback, but update flags */
1065	1007	if (ctxt->src.val == 0)
1066	1008	ctxt->dst.type = OP_NONE;
1067		− return fastop(ctxt, em_bsr);
	1009	+ return em_bsr(ctxt);
1068	1010	}
1069	1011
1070	1012	static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1071	1013	{
1072		− u8 rc;
1073		− void (fop)(void) = (void )em_setcc + FASTOP_SIZE * (condition & 0xf);
1074		−
1075		− flags = (flags & EFLAGS_MASK) \| X86_EFLAGS_IF;
1076		− asm("push %[flags]; popf; " CALL_NOSPEC
1077		− : "=a"(rc), ASM_CALL_CONSTRAINT : [thunk_target]"r"(fop), [flags]"r"(flags));
1078		− return rc;
	1014	+ return __emulate_cc(flags, condition & 0xf);
1079	1015	}
1080	1016
1081	1017	static void fetch_register_operand(struct operand *op)
@@ -2325,7 +2261,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) · 1243 unmodified lines
2325	2261	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2326	2262	ctxt->src.orig_val = ctxt->src.val;
2327	2263	ctxt->src.val = ctxt->dst.orig_val;
2328		− fastop(ctxt, em_cmp);
	2264	+ em_cmp(ctxt);
2329	2265
2330	2266	if (ctxt->eflags & X86_EFLAGS_ZF) {
2331	2267	/* Success: write back to memory; no update of EAX */
@@ -3090,7 +3026,7 @@ static int em_das(struct x86_emulate_ctxt *ctxt) · 758 unmodified lines
3090	3026	ctxt->src.type = OP_IMM;
3091	3027	ctxt->src.val = 0;
3092	3028	ctxt->src.bytes = 1;
3093		− fastop(ctxt, em_or);
	3029	+ em_or(ctxt);
3094	3030	ctxt->eflags &= ~(X86_EFLAGS_AF \| X86_EFLAGS_CF);
3095	3031	if (cf)
3096	3032	ctxt->eflags \|= X86_EFLAGS_CF;
@@ -3116,7 +3052,7 @@ static int em_aam(struct x86_emulate_ctxt *ctxt) · 19 unmodified lines
3116	3052	ctxt->src.type = OP_IMM;
3117	3053	ctxt->src.val = 0;
3118	3054	ctxt->src.bytes = 1;
3119		− fastop(ctxt, em_or);
	3055	+ em_or(ctxt);
3120	3056
3121	3057	return X86EMUL_CONTINUE;
3122	3058	}
@@ -3134,7 +3070,7 @@ static int em_aad(struct x86_emulate_ctxt *ctxt) · 11 unmodified lines
3134	3070	ctxt->src.type = OP_IMM;
3135	3071	ctxt->src.val = 0;
3136	3072	ctxt->src.bytes = 1;
3137		− fastop(ctxt, em_or);
	3073	+ em_or(ctxt);
3138	3074
3139	3075	return X86EMUL_CONTINUE;
3140	3076	}
@@ -3225,7 +3161,7 @@ static int em_xchg(struct x86_emulate_ctxt *ctxt) · 84 unmodified lines
3225	3161	static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3226	3162	{
3227	3163	ctxt->dst.val = ctxt->src2.val;
3228		− return fastop(ctxt, em_imul);
	3164	+ return em_imul(ctxt);
3229	3165	}
3230	3166
3231	3167	static int em_cwd(struct x86_emulate_ctxt *ctxt)
@@ -4004,7 +3940,6 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) · 772 unmodified lines
4004	3940	#define MD(_f, _m) { .flags = ((_f) \| ModeDual), .u.mdual = (_m) }
4005	3941	#define E(_f, _e) { .flags = ((_f) \| Escape \| ModRM), .u.esc = (_e) }
4006	3942	#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4007		−#define F(_f, _e) { .flags = (_f) \| Fastop, .u.fastop = (_e) }
4008	3943	#define II(_f, _e, _i) \
4009	3944	{ .flags = (_f)\|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4010	3945	#define IIP(_f, _e, _i, _p) \
@@ -4019,9 +3954,9 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) · 8 unmodified lines
4019	3954	#define I2bvIP(_f, _e, _i, _p) \
4020	3955	IIP((_f) \| ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4021	3956
4022		−#define F6ALU(_f, _e) F2bv((_f) \| DstMem \| SrcReg \| ModRM, _e), \
4023		− F2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \
4024		− F2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)
	3957	+#define I6ALU(_f, _e) I2bv((_f) \| DstMem \| SrcReg \| ModRM, _e), \
	3958	+ I2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \
	3959	+ I2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)
4025	3960
4026	3961	static const struct opcode group7_rm0[] = {
4027	3962	N,
@@ -4059,14 +3994,14 @@ static const struct opcode group7_rm7[] = { · 31 unmodified lines
4059	3994	};
4060	3995
4061	3996	static const struct opcode group1[] = {
4062		− F(Lock, em_add),
4063		− F(Lock \| PageTable, em_or),
4064		− F(Lock, em_adc),
4065		− F(Lock, em_sbb),
4066		− F(Lock \| PageTable, em_and),
4067		− F(Lock, em_sub),
4068		− F(Lock, em_xor),
4069		− F(NoWrite, em_cmp),
	3997	+ I(Lock, em_add),
	3998	+ I(Lock \| PageTable, em_or),
	3999	+ I(Lock, em_adc),
	4000	+ I(Lock, em_sbb),
	4001	+ I(Lock \| PageTable, em_and),
	4002	+ I(Lock, em_sub),
	4003	+ I(Lock, em_xor),
	4004	+ I(NoWrite, em_cmp),
4070	4005	};
4071	4006
4072	4007	static const struct opcode group1A[] = {
@@ -4074,36 +4009,36 @@ static const struct opcode group1A[] = { · 1 unmodified line
4074	4009	};
4075	4010
4076	4011	static const struct opcode group2[] = {
4077		− F(DstMem \| ModRM, em_rol),
4078		− F(DstMem \| ModRM, em_ror),
4079		− F(DstMem \| ModRM, em_rcl),
4080		− F(DstMem \| ModRM, em_rcr),
4081		− F(DstMem \| ModRM, em_shl),
4082		− F(DstMem \| ModRM, em_shr),
4083		− F(DstMem \| ModRM, em_shl),
4084		− F(DstMem \| ModRM, em_sar),
	4012	+ I(DstMem \| ModRM, em_rol),
	4013	+ I(DstMem \| ModRM, em_ror),
	4014	+ I(DstMem \| ModRM, em_rcl),
	4015	+ I(DstMem \| ModRM, em_rcr),
	4016	+ I(DstMem \| ModRM, em_shl),
	4017	+ I(DstMem \| ModRM, em_shr),
	4018	+ I(DstMem \| ModRM, em_shl),
	4019	+ I(DstMem \| ModRM, em_sar),
4085	4020	};
4086	4021
4087	4022	static const struct opcode group3[] = {
4088		− F(DstMem \| SrcImm \| NoWrite, em_test),
4089		− F(DstMem \| SrcImm \| NoWrite, em_test),
4090		− F(DstMem \| SrcNone \| Lock, em_not),
4091		− F(DstMem \| SrcNone \| Lock, em_neg),
4092		− F(DstXacc \| Src2Mem, em_mul_ex),
4093		− F(DstXacc \| Src2Mem, em_imul_ex),
4094		− F(DstXacc \| Src2Mem, em_div_ex),
4095		− F(DstXacc \| Src2Mem, em_idiv_ex),
	4023	+ I(DstMem \| SrcImm \| NoWrite, em_test),
	4024	+ I(DstMem \| SrcImm \| NoWrite, em_test),
	4025	+ I(DstMem \| SrcNone \| Lock, em_not),
	4026	+ I(DstMem \| SrcNone \| Lock, em_neg),
	4027	+ I(DstXacc \| Src2Mem, em_mul_ex),
	4028	+ I(DstXacc \| Src2Mem, em_imul_ex),
	4029	+ I(DstXacc \| Src2Mem, em_div_ex),
	4030	+ I(DstXacc \| Src2Mem, em_idiv_ex),
4096	4031	};
4097	4032
4098	4033	static const struct opcode group4[] = {
4099		− F(ByteOp \| DstMem \| SrcNone \| Lock, em_inc),
4100		− F(ByteOp \| DstMem \| SrcNone \| Lock, em_dec),
	4034	+ I(ByteOp \| DstMem \| SrcNone \| Lock, em_inc),
	4035	+ I(ByteOp \| DstMem \| SrcNone \| Lock, em_dec),
4101	4036	N, N, N, N, N, N,
4102	4037	};
4103	4038
4104	4039	static const struct opcode group5[] = {
4105		− F(DstMem \| SrcNone \| Lock, em_inc),
4106		− F(DstMem \| SrcNone \| Lock, em_dec),
	4040	+ I(DstMem \| SrcNone \| Lock, em_inc),
	4041	+ I(DstMem \| SrcNone \| Lock, em_dec),
4107	4042	I(SrcMem \| NearBranch \| IsBranch \| ShadowStack, em_call_near_abs),
4108	4043	I(SrcMemFAddr \| ImplicitOps \| IsBranch \| ShadowStack, em_call_far),
4109	4044	I(SrcMem \| NearBranch \| IsBranch, em_jmp_abs),
@@ -4139,10 +4074,10 @@ static const struct group_dual group7 = { { · 29 unmodified lines
4139	4074
4140	4075	static const struct opcode group8[] = {
4141	4076	N, N, N, N,
4142		− F(DstMem \| SrcImmByte \| NoWrite, em_bt),
4143		− F(DstMem \| SrcImmByte \| Lock \| PageTable, em_bts),
4144		− F(DstMem \| SrcImmByte \| Lock, em_btr),
4145		− F(DstMem \| SrcImmByte \| Lock \| PageTable, em_btc),
	4077	+ I(DstMem \| SrcImmByte \| NoWrite, em_bt),
	4078	+ I(DstMem \| SrcImmByte \| Lock \| PageTable, em_bts),
	4079	+ I(DstMem \| SrcImmByte \| Lock, em_btr),
	4080	+ I(DstMem \| SrcImmByte \| Lock \| PageTable, em_btc),
4146	4081	};
4147	4082
4148	4083	/*
@@ -4279,31 +4214,31 @@ static const struct instr_dual instr_dual_8d = { · 130 unmodified lines
4279	4214
4280	4215	static const struct opcode opcode_table[256] = {
4281	4216	/* 0x00 - 0x07 */
4282		− F6ALU(Lock, em_add),
	4217	+ I6ALU(Lock, em_add),
4283	4218	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_push_sreg),
4284	4219	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_pop_sreg),
4285	4220	/* 0x08 - 0x0F */
4286		− F6ALU(Lock \| PageTable, em_or),
	4221	+ I6ALU(Lock \| PageTable, em_or),
4287	4222	I(ImplicitOps \| Stack \| No64 \| Src2CS, em_push_sreg),
4288	4223	N,
4289	4224	/* 0x10 - 0x17 */
4290		− F6ALU(Lock, em_adc),
	4225	+ I6ALU(Lock, em_adc),
4291	4226	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_push_sreg),
4292	4227	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_pop_sreg),
4293	4228	/* 0x18 - 0x1F */
4294		− F6ALU(Lock, em_sbb),
	4229	+ I6ALU(Lock, em_sbb),
4295	4230	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_push_sreg),
4296	4231	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_pop_sreg),
4297	4232	/* 0x20 - 0x27 */
4298		− F6ALU(Lock \| PageTable, em_and), N, N,
	4233	+ I6ALU(Lock \| PageTable, em_and), N, N,
4299	4234	/* 0x28 - 0x2F */
4300		− F6ALU(Lock, em_sub), N, I(ByteOp \| DstAcc \| No64, em_das),
	4235	+ I6ALU(Lock, em_sub), N, I(ByteOp \| DstAcc \| No64, em_das),
4301	4236	/* 0x30 - 0x37 */
4302		− F6ALU(Lock, em_xor), N, N,
	4237	+ I6ALU(Lock, em_xor), N, N,
4303	4238	/* 0x38 - 0x3F */
4304		− F6ALU(NoWrite, em_cmp), N, N,
	4239	+ I6ALU(NoWrite, em_cmp), N, N,
4305	4240	/* 0x40 - 0x4F */
4306		− X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
	4241	+ X8(I(DstReg, em_inc)), X8(I(DstReg, em_dec)),
4307	4242	/* 0x50 - 0x57 */
4308	4243	X8(I(SrcReg \| Stack, em_push)),
4309	4244	/* 0x58 - 0x5F */
@@ -4327,7 +4262,7 @@ static const struct opcode opcode_table[256] = { · 17 unmodified lines
4327	4262	G(DstMem \| SrcImm, group1),
4328	4263	G(ByteOp \| DstMem \| SrcImm \| No64, group1),
4329	4264	G(DstMem \| SrcImmByte, group1),
4330		− F2bv(DstMem \| SrcReg \| ModRM \| NoWrite, em_test),
	4265	+ I2bv(DstMem \| SrcReg \| ModRM \| NoWrite, em_test),
4331	4266	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable, em_xchg),
4332	4267	/* 0x88 - 0x8F */
4333	4268	I2bv(DstMem \| SrcReg \| ModRM \| Mov \| PageTable, em_mov),
@@ -4348,12 +4283,12 @@ static const struct opcode opcode_table[256] = { · 14 unmodified lines
4348	4283	I2bv(DstAcc \| SrcMem \| Mov \| MemAbs, em_mov),
4349	4284	I2bv(DstMem \| SrcAcc \| Mov \| MemAbs \| PageTable, em_mov),
4350	4285	I2bv(SrcSI \| DstDI \| Mov \| String \| TwoMemOp, em_mov),
4351		− F2bv(SrcSI \| DstDI \| String \| NoWrite \| TwoMemOp, em_cmp_r),
	4286	+ I2bv(SrcSI \| DstDI \| String \| NoWrite \| TwoMemOp, em_cmp_r),
4352	4287	/* 0xA8 - 0xAF */
4353		− F2bv(DstAcc \| SrcImm \| NoWrite, em_test),
	4288	+ I2bv(DstAcc \| SrcImm \| NoWrite, em_test),
4354	4289	I2bv(SrcAcc \| DstDI \| Mov \| String, em_mov),
4355	4290	I2bv(SrcSI \| DstAcc \| Mov \| String, em_mov),
4356		− F2bv(SrcAcc \| DstDI \| String \| NoWrite, em_cmp_r),
	4291	+ I2bv(SrcAcc \| DstDI \| String \| NoWrite, em_cmp_r),
4357	4292	/* 0xB0 - 0xB7 */
4358	4293	X8(I(ByteOp \| DstReg \| SrcImm \| Mov, em_mov)),
4359	4294	/* 0xB8 - 0xBF */
@@ -4378,7 +4313,7 @@ static const struct opcode opcode_table[256] = { · 18 unmodified lines
4378	4313	G(Src2CL \| ByteOp, group2), G(Src2CL, group2),
4379	4314	I(DstAcc \| SrcImmUByte \| No64, em_aam),
4380	4315	I(DstAcc \| SrcImmUByte \| No64, em_aad),
4381		− F(DstAcc \| ByteOp \| No64, em_salc),
	4316	+ I(DstAcc \| ByteOp \| No64, em_salc),
4382	4317	I(DstAcc \| SrcXLat \| ByteOp, em_mov),
4383	4318	/* 0xD8 - 0xDF */
4384	4319	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
@@ -4463,32 +4398,32 @@ static const struct opcode twobyte_table[256] = { · 78 unmodified lines
4463	4398	/* 0xA0 - 0xA7 */
4464	4399	I(Stack \| Src2FS, em_push_sreg), I(Stack \| Src2FS, em_pop_sreg),
4465	4400	II(ImplicitOps, em_cpuid, cpuid),
4466		− F(DstMem \| SrcReg \| ModRM \| BitOp \| NoWrite, em_bt),
4467		− F(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shld),
4468		− F(DstMem \| SrcReg \| Src2CL \| ModRM, em_shld), N, N,
	4401	+ I(DstMem \| SrcReg \| ModRM \| BitOp \| NoWrite, em_bt),
	4402	+ I(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shld),
	4403	+ I(DstMem \| SrcReg \| Src2CL \| ModRM, em_shld), N, N,
4469	4404	/* 0xA8 - 0xAF */
4470	4405	I(Stack \| Src2GS, em_push_sreg), I(Stack \| Src2GS, em_pop_sreg),
4471	4406	II(EmulateOnUD \| ImplicitOps, em_rsm, rsm),
4472		− F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_bts),
4473		− F(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shrd),
4474		− F(DstMem \| SrcReg \| Src2CL \| ModRM, em_shrd),
4475		− GD(0, &group15), F(DstReg \| SrcMem \| ModRM, em_imul),
	4407	+ I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_bts),
	4408	+ I(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shrd),
	4409	+ I(DstMem \| SrcReg \| Src2CL \| ModRM, em_shrd),
	4410	+ GD(0, &group15), I(DstReg \| SrcMem \| ModRM, em_imul),
4476	4411	/* 0xB0 - 0xB7 */
4477	4412	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable \| SrcWrite, em_cmpxchg),
4478	4413	I(DstReg \| SrcMemFAddr \| ModRM \| Src2SS, em_lseg),
4479		− F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock, em_btr),
	4414	+ I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock, em_btr),
4480	4415	I(DstReg \| SrcMemFAddr \| ModRM \| Src2FS, em_lseg),
4481	4416	I(DstReg \| SrcMemFAddr \| ModRM \| Src2GS, em_lseg),
4482	4417	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),
4483	4418	/* 0xB8 - 0xBF */
4484	4419	N, N,
4485	4420	G(BitOp, group8),
4486		− F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_btc),
	4421	+ I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_btc),
4487	4422	I(DstReg \| SrcMem \| ModRM, em_bsf_c),
4488	4423	I(DstReg \| SrcMem \| ModRM, em_bsr_c),
4489	4424	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),
4490	4425	/* 0xC0 - 0xC7 */
4491		− F2bv(DstMem \| SrcReg \| ModRM \| SrcWrite \| Lock, em_xadd),
	4426	+ I2bv(DstMem \| SrcReg \| ModRM \| SrcWrite \| Lock, em_xadd),
4492	4427	N, ID(0, &instr_dual_0f_c3),
4493	4428	N, N, N, GD(0, &group9),
4494	4429	/* 0xC8 - 0xCF */
@@ -5198,24 +5133,6 @@ static void fetch_possible_mmx_operand(struct operand *op) · 703 unmodified lines
5198	5133	kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5199	5134	}
5200	5135
5201		−static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5202		−{
5203		− ulong flags = (ctxt->eflags & EFLAGS_MASK) \| X86_EFLAGS_IF;
5204		−
5205		− if (!(ctxt->d & ByteOp))
5206		− fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5207		−
5208		− asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5209		− : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5210		− [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5211		− : "c"(ctxt->src2.val));
5212		−
5213		− ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) \| (flags & EFLAGS_MASK);
5214		− if (!fop) /* exception is returned in fop variable */
5215		− return emulate_de(ctxt);
5216		− return X86EMUL_CONTINUE;
5217		−}
5218		−
5219	5136	void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5220	5137	{
5221	5138	/* Clear fields that are set conditionally but read without a guard. */
@@ -5379,10 +5296,7 @@ special_insn: · 157 unmodified lines
5379	5296	ctxt->eflags &= ~X86_EFLAGS_RF;
5380	5297
5381	5298	if (ctxt->execute) {
5382		− if (ctxt->d & Fastop)
5383		− rc = fastop(ctxt, ctxt->fop);
5384		− else
5385		− rc = ctxt->execute(ctxt);
	5299	+ rc = ctxt->execute(ctxt);
5386	5300	if (rc != X86EMUL_CONTINUE)
5387	5301	goto done;
5388	5302	goto writeback;

arch/x86/kvm/vmx/vmenter.S+4−0

@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) · 360 unmodified lines
361	361
362	362	.section .text, "ax"
363	363
	364	+#ifndef CONFIG_X86_FRED
	365	+
364	366	SYM_FUNC_START(vmx_do_interrupt_irqoff)
365	367	VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
366	368	SYM_FUNC_END(vmx_do_interrupt_irqoff)
	369	+
	370	+#endif

arch/x86/kvm/vmx/vmx.c+7−1

@@ -7021,8 +7021,14 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, · 7020 unmodified lines
7021	7021	"unexpected VM-Exit interrupt info: 0x%x", intr_info))
7022	7022	return;
7023	7023
	7024	+ /*
	7025	+ * Invoke the kernel's IRQ handler for the vector. Use the FRED path
	7026	+ * when it's available even if FRED isn't fully enabled, e.g. even if
	7027	+ * FRED isn't supported in hardware, in order to avoid the indirect
	7028	+ * CALL in the non-FRED path.
	7029	+ */
7024	7030	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
7025		− if (cpu_feature_enabled(X86_FEATURE_FRED))
	7031	+ if (IS_ENABLED(CONFIG_X86_FRED))
7026	7032	fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
7027	7033	else
7028	7034	vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));

arch/x86/lib/bhi.S+29−29

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	5	#include <asm/nospec-branch.h>
6	6
7	7	/*
8		− * Notably, the FineIBT preamble calling these will have ZF set and r10 zero.
	8	+ * Notably, the FineIBT preamble calling these will have ZF set and eax zero.
9	9	*
10	10	* The very last element is in fact larger than 32 bytes, but since its the
11	11	* last element, this does not matter,
@@ -36,7 +36,7 @@ SYM_INNER_LABEL(__bhi_args_1, SYM_L_LOCAL) · 24 unmodified lines
36	36	ANNOTATE_NOENDBR
37	37	UNWIND_HINT_FUNC
38	38	jne .Lud_1
39		− cmovne %r10, %rdi
	39	+ cmovne %rax, %rdi
40	40	ANNOTATE_UNRET_SAFE
41	41	ret
42	42	int3
@@ -53,8 +53,8 @@ SYM_INNER_LABEL(__bhi_args_2, SYM_L_LOCAL) · 10 unmodified lines
53	53	ANNOTATE_NOENDBR
54	54	UNWIND_HINT_FUNC
55	55	jne .Lud_1
56		− cmovne %r10, %rdi
57		− cmovne %r10, %rsi
	56	+ cmovne %rax, %rdi
	57	+ cmovne %rax, %rsi
58	58	ANNOTATE_UNRET_SAFE
59	59	ret
60	60	int3
@@ -64,9 +64,9 @@ SYM_INNER_LABEL(__bhi_args_3, SYM_L_LOCAL) · 3 unmodified lines
64	64	ANNOTATE_NOENDBR
65	65	UNWIND_HINT_FUNC
66	66	jne .Lud_1
67		− cmovne %r10, %rdi
68		− cmovne %r10, %rsi
69		− cmovne %r10, %rdx
	67	+ cmovne %rax, %rdi
	68	+ cmovne %rax, %rsi
	69	+ cmovne %rax, %rdx
70	70	ANNOTATE_UNRET_SAFE
71	71	ret
72	72	int3
@@ -76,10 +76,10 @@ SYM_INNER_LABEL(__bhi_args_4, SYM_L_LOCAL) · 3 unmodified lines
76	76	ANNOTATE_NOENDBR
77	77	UNWIND_HINT_FUNC
78	78	jne .Lud_2
79		− cmovne %r10, %rdi
80		− cmovne %r10, %rsi
81		− cmovne %r10, %rdx
82		− cmovne %r10, %rcx
	79	+ cmovne %rax, %rdi
	80	+ cmovne %rax, %rsi
	81	+ cmovne %rax, %rdx
	82	+ cmovne %rax, %rcx
83	83	ANNOTATE_UNRET_SAFE
84	84	ret
85	85	int3
@@ -89,11 +89,11 @@ SYM_INNER_LABEL(__bhi_args_5, SYM_L_LOCAL) · 3 unmodified lines
89	89	ANNOTATE_NOENDBR
90	90	UNWIND_HINT_FUNC
91	91	jne .Lud_2
92		− cmovne %r10, %rdi
93		− cmovne %r10, %rsi
94		− cmovne %r10, %rdx
95		− cmovne %r10, %rcx
96		− cmovne %r10, %r8
	92	+ cmovne %rax, %rdi
	93	+ cmovne %rax, %rsi
	94	+ cmovne %rax, %rdx
	95	+ cmovne %rax, %rcx
	96	+ cmovne %rax, %r8
97	97	ANNOTATE_UNRET_SAFE
98	98	ret
99	99	int3
@@ -110,12 +110,12 @@ SYM_INNER_LABEL(__bhi_args_6, SYM_L_LOCAL) · 10 unmodified lines
110	110	ANNOTATE_NOENDBR
111	111	UNWIND_HINT_FUNC
112	112	jne .Lud_2
113		− cmovne %r10, %rdi
114		− cmovne %r10, %rsi
115		− cmovne %r10, %rdx
116		− cmovne %r10, %rcx
117		− cmovne %r10, %r8
118		− cmovne %r10, %r9
	113	+ cmovne %rax, %rdi
	114	+ cmovne %rax, %rsi
	115	+ cmovne %rax, %rdx
	116	+ cmovne %rax, %rcx
	117	+ cmovne %rax, %r8
	118	+ cmovne %rax, %r9
119	119	ANNOTATE_UNRET_SAFE
120	120	ret
121	121	int3
@@ -125,13 +125,13 @@ SYM_INNER_LABEL(__bhi_args_7, SYM_L_LOCAL) · 3 unmodified lines
125	125	ANNOTATE_NOENDBR
126	126	UNWIND_HINT_FUNC
127	127	jne .Lud_2
128		− cmovne %r10, %rdi
129		− cmovne %r10, %rsi
130		− cmovne %r10, %rdx
131		− cmovne %r10, %rcx
132		− cmovne %r10, %r8
133		− cmovne %r10, %r9
134		− cmovne %r10, %rsp
	128	+ cmovne %rax, %rdi
	129	+ cmovne %rax, %rsi
	130	+ cmovne %rax, %rdx
	131	+ cmovne %rax, %rcx
	132	+ cmovne %rax, %r8
	133	+ cmovne %rax, %r9
	134	+ cmovne %rax, %rsp
135	135	ANNOTATE_UNRET_SAFE
136	136	ret
137	137	int3

arch/x86/lib/retpoline.S+2−2

@@ -134,10 +134,10 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array) · 133 unmodified lines
134	134	.macro ITS_THUNK reg
135	135
136	136	/*
137		− * If CFI paranoid is used then the ITS thunk starts with opcodes (0xea; jne 1b)
	137	+ * If CFI paranoid is used then the ITS thunk starts with opcodes (1: udb; jne 1b)
138	138	* that complete the fineibt_paranoid caller sequence.
139	139	*/
140		−1: .byte 0xea
	140	+1: ASM_UDB
141	141	SYM_INNER_LABEL(__x86_indirect_paranoid_thunk_\reg, SYM_L_GLOBAL)
142	142	UNWIND_HINT_UNDEFINED
143	143	ANNOTATE_NOENDBR

arch/x86/net/bpf_jit_comp.c+3−3

@@ -420,12 +420,12 @@ static void emit_fineibt(u8 *pprog, u8 ip, u32 hash, int arity) · 419 unmodified lines
420	420	u8 prog = pprog;
421	421
422	422	EMIT_ENDBR();
423		− EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */
	423	+ EMIT1_off32(0x2d, hash); /* subl $hash, %eax */
424	424	if (cfi_bhi) {
	425	+ EMIT2(0x2e, 0x2e); /* cs cs */
425	426	emit_call(&prog, __bhi_args[arity], ip + 11);
426	427	} else {
427		− EMIT2(0x75, 0xf9); /* jne.d8 .-7 */
428		− EMIT3(0x0f, 0x1f, 0x00); /* nop3 */
	428	+ EMIT3_off32(0x2e, 0x0f, 0x85, 3); /* jne.d32,pn 3 */
429	429	}
430	430	EMIT_ENDBR_POISON();
431	431

arch/x86/platform/efi/efi_stub_64.S+4−0

@@ -11,6 +11,10 @@ · 10 unmodified lines
11	11	#include <asm/nospec-branch.h>
12	12
13	13	SYM_FUNC_START(__efi_call)
	14	+ /*
	15	+ * The EFI code doesn't have any CFI, annotate away the CFI violation.
	16	+ */
	17	+ ANNOTATE_NOCFI_SYM
14	18	pushq %rbp
15	19	movq %rsp, %rbp
16	20	and $~0xf, %rsp

drivers/misc/lkdtm/perms.c+5−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/vmalloc.h>
10	10	#include <linux/mman.h>
11	11	#include <linux/uaccess.h>
	12	+#include <linux/objtool.h>
12	13	#include <asm/cacheflush.h>
13	14	#include <asm/sections.h>
14	15
@@ -86,6 +87,10 @@ static noinline __nocfi void execute_location(void *dst, bool write) · 71 unmodified lines
86	87	func();
87	88	pr_err("FAIL: func returned\n");
88	89	}
	90	+/*
	91	+ * Explicitly doing the wrong thing for testing.
	92	+ */
	93	+ANNOTATE_NOCFI_SYM(execute_location);
89	94
90	95	static void execute_user_location(void *dst)
91	96	{

include/linux/compiler-clang.h+0−5

@@ -115,11 +115,6 @@ · 114 unmodified lines
115	115	# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
116	116	#endif
117	117
118		−#if __has_feature(kcfi)
119		−/* Disable CFI checking inside a function. */
120		−#define __nocfi __attribute__((__no_sanitize__("kcfi")))
121		−#endif
122		−
123	118	/*
124	119	* Turn individual warnings and errors on and off locally, depending
125	120	* on version.

include/linux/compiler-gcc.h+0−4

@@ -35,10 +35,6 @@ · 34 unmodified lines
35	35	(typeof(ptr)) (__ptr + (off)); \
36	36	})
37	37
38		−#ifdef CONFIG_MITIGATION_RETPOLINE
39		−#define __noretpoline __attribute__((__indirect_branch__("keep")))
40		−#endif
41		−
42	38	#if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
43	39	#define __latent_entropy __attribute__((latent_entropy))
44	40	#endif

include/linux/compiler_types.h+3−1

@@ -455,7 +455,9 @@ struct ftrace_likely_data { · 454 unmodified lines
455	455	# define __noscs
456	456	#endif
457	457
458		−#ifndef __nocfi
	458	+#if defined(CONFIG_CFI)
	459	+# define __nocfi __attribute__((__no_sanitize__("kcfi")))
	460	+#else
459	461	# define __nocfi
460	462	#endif
461	463

include/linux/init.h+0−8

@@ -7,13 +7,6 @@ · 6 unmodified lines
7	7	#include <linux/stringify.h>
8	8	#include <linux/types.h>
9	9
10		−/* Built-in __init functions needn't be compiled with retpoline */
11		−#if defined(__noretpoline) && !defined(MODULE)
12		−#define __noinitretpoline __noretpoline
13		−#else
14		−#define __noinitretpoline
15		−#endif
16		−
17	10	/* These macros are used to mark some functions or
18	11	* initialized data (doesn't apply to uninitialized data)
19	12	* as `initialization' functions. The kernel can take this
@@ -50,7 +43,6 @@ · 30 unmodified lines
50	43	/* These are for everybody (although not all archs will actually
51	44	discard it in modules) */
52	45	#define __init __section(".init.text") __cold __latent_entropy \
53		− __noinitretpoline \
54	46	__no_kstack_erase
55	47	#define __initdata __section(".init.data")
56	48	#define __initconst __section(".init.rodata")

include/linux/objtool.h+10−0

@@ -184,6 +184,15 @@ · 183 unmodified lines
184	184	* WARN using UD2.
185	185	*/
186	186	#define ANNOTATE_REACHABLE(label) __ASM_ANNOTATE(label, ANNOTYPE_REACHABLE)
	187	+/*
	188	+ * This should not be used; it annotates away CFI violations. There are a few
	189	+ * valid use cases like kexec handover to the next kernel image, and there is
	190	+ * no security concern there.
	191	+ *
	192	+ * There are also a few real issues annotated away, like EFI because we can't
	193	+ * control the EFI code.
	194	+ */
	195	+#define ANNOTATE_NOCFI_SYM(sym) asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI))
187	196
188	197	#else
189	198	#define ANNOTATE_NOENDBR ANNOTATE type=ANNOTYPE_NOENDBR
@@ -194,6 +203,7 @@ · 4 unmodified lines
194	203	#define ANNOTATE_INTRA_FUNCTION_CALL ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL
195	204	#define ANNOTATE_UNRET_BEGIN ANNOTATE type=ANNOTYPE_UNRET_BEGIN
196	205	#define ANNOTATE_REACHABLE ANNOTATE type=ANNOTYPE_REACHABLE
	206	+#define ANNOTATE_NOCFI_SYM ANNOTATE type=ANNOTYPE_NOCFI
197	207	#endif
198	208
199	209	#if defined(CONFIG_NOINSTR_VALIDATION) && \

include/linux/objtool_types.h+1−0

@@ -65,5 +65,6 @@ struct unwind_hint { · 64 unmodified lines
65	65	#define ANNOTYPE_IGNORE_ALTS 6
66	66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7
67	67	#define ANNOTYPE_REACHABLE 8
	68	+#define ANNOTYPE_NOCFI 9
68	69
69	70	#endif /* _LINUX_OBJTOOL_TYPES_H */

tools/include/linux/objtool_types.h+1−0

@@ -65,5 +65,6 @@ struct unwind_hint { · 64 unmodified lines
65	65	#define ANNOTYPE_IGNORE_ALTS 6
66	66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7
67	67	#define ANNOTYPE_REACHABLE 8
	68	+#define ANNOTYPE_NOCFI 9
68	69
69	70	#endif /* _LINUX_OBJTOOL_TYPES_H */

tools/objtool/check.c+42−0

@@ -2392,6 +2392,8 @@ static int __annotate_ifc(struct objtool_file *file, int type, struct instructio · 2391 unmodified lines
2392	2392
2393	2393	static int __annotate_late(struct objtool_file file, int type, struct instruction insn)
2394	2394	{
	2395	+ struct symbol *sym;
	2396	+
2395	2397	switch (type) {
2396	2398	case ANNOTYPE_NOENDBR:
2397	2399	/* early */
@@ -2433,6 +2435,15 @@ static int __annotate_late(struct objtool_file *file, int type, struct instructi · 35 unmodified lines
2433	2435	insn->dead_end = false;
2434	2436	break;
2435	2437
	2438	+ case ANNOTYPE_NOCFI:
	2439	+ sym = insn->sym;
	2440	+ if (!sym) {
	2441	+ ERROR_INSN(insn, "dodgy NOCFI annotation");
	2442	+ return -1;
	2443	+ }
	2444	+ insn->sym->nocfi = 1;
	2445	+ break;
	2446	+
2436	2447	default:
2437	2448	ERROR_INSN(insn, "Unknown annotation type: %d", type);
2438	2449	return -1;
@@ -3994,6 +4005,37 @@ static int validate_retpoline(struct objtool_file *file) · 1555 unmodified lines
3994	4005	warnings++;
3995	4006	}
3996	4007
	4008	+ if (!opts.cfi)
	4009	+ return warnings;
	4010	+
	4011	+ /*
	4012	+ * kCFI call sites look like:
	4013	+ *
	4014	+ * movl $(-0x12345678), %r10d
	4015	+ * addl -4(%r11), %r10d
	4016	+ * jz 1f
	4017	+ * ud2
	4018	+ * 1: cs call __x86_indirect_thunk_r11
	4019	+ *
	4020	+ * Verify all indirect calls are kCFI adorned by checking for the
	4021	+ * UD2. Notably, doing __nocfi calls to regular (cfi) functions is
	4022	+ * broken.
	4023	+ */
	4024	+ list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
	4025	+ struct symbol *sym = insn->sym;
	4026	+
	4027	+ if (sym && (sym->type == STT_NOTYPE \|\|
	4028	+ sym->type == STT_FUNC) && !sym->nocfi) {
	4029	+ struct instruction *prev =
	4030	+ prev_insn_same_sym(file, insn);
	4031	+
	4032	+ if (!prev \|\| prev->type != INSN_BUG) {
	4033	+ WARN_INSN(insn, "no-cfi indirect call!");
	4034	+ warnings++;
	4035	+ }
	4036	+ }
	4037	+ }
	4038	+
3997	4039	return warnings;
3998	4040	}
3999	4041

tools/objtool/include/objtool/elf.h+1−0

@@ -70,6 +70,7 @@ struct symbol { · 69 unmodified lines
70	70	u8 local_label : 1;
71	71	u8 frame_pointer : 1;
72	72	u8 ignore : 1;
	73	+ u8 nocfi : 1;
73	74	struct list_head pv_target;
74	75	struct reloc *relocs;
75	76	struct section *group_sec;

Merge tag 'x86_core_for_v6.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code