lael-labs/linux

Documentation/admin-guide/kernel-parameters.txt+18−0

@@ -608,6 +608,24 @@ · 607 unmodified lines
608	ccw_timeout_log [S390]	608	ccw_timeout_log [S390]
609	See Documentation/arch/s390/common_io.rst for details.	609	See Documentation/arch/s390/common_io.rst for details.
610		610
		611	cfi= [X86-64] Set Control Flow Integrity checking features
		612	when CONFIG_FINEIBT is enabled.
		613	Format: feature[,feature...]
		614	Default: auto
		615
		616	auto: Use FineIBT if IBT available, otherwise kCFI.
		617	Under FineIBT, enable "paranoid" mode when
		618	FRED is not available.
		619	off: Turn off CFI checking.
		620	kcfi: Use kCFI (disable FineIBT).
		621	fineibt: Use FineIBT (even if IBT not available).
		622	norand: Do not re-randomize CFI hashes.
		623	paranoid: Add caller hash checking under FineIBT.
		624	bhi: Enable register poisoning to stop speculation
		625	across FineIBT. (Disabled by default.)
		626	warn: Do not enforce CFI checking: warn only.
		627	debug: Report CFI initialization details.
		628
611	cgroup_disable= [KNL] Disable a particular controller or optional feature	629	cgroup_disable= [KNL] Disable a particular controller or optional feature
612	Format: {name of the controller(s) or feature(s) to disable}	630	Format: {name of the controller(s) or feature(s) to disable}
613	The effects of cgroup_disable=foo are:	631	The effects of cgroup_disable=foo are:

arch/x86/entry/calling.h+5−6

@@ -99,7 +99,7 @@ For 32-bit we have the following conventions - kernel is built with · 98 unmodified lines
99	.endif	99	.endif
100	.endm	100	.endm
101		101
102	.macro CLEAR_REGS clear_bp=1	102	.macro CLEAR_REGS clear_callee=1
103	/*	103	/*
104	* Sanitize registers of values that a speculation attack might	104	* Sanitize registers of values that a speculation attack might
105	* otherwise want to exploit. The lower registers are likely clobbered	105	* otherwise want to exploit. The lower registers are likely clobbered
@@ -113,20 +113,19 @@ For 32-bit we have the following conventions - kernel is built with · 7 unmodified lines
113	xorl %r9d, %r9d /* nospec r9 */	113	xorl %r9d, %r9d /* nospec r9 */
114	xorl %r10d, %r10d /* nospec r10 */	114	xorl %r10d, %r10d /* nospec r10 */
115	xorl %r11d, %r11d /* nospec r11 */	115	xorl %r11d, %r11d /* nospec r11 */
		116	.if \clear_callee
116	xorl %ebx, %ebx /* nospec rbx */	117	xorl %ebx, %ebx /* nospec rbx */
117	.if \clear_bp
118	xorl %ebp, %ebp /* nospec rbp */	118	xorl %ebp, %ebp /* nospec rbp */
119	.endif
120	xorl %r12d, %r12d /* nospec r12 */	119	xorl %r12d, %r12d /* nospec r12 */
121	xorl %r13d, %r13d /* nospec r13 */	120	xorl %r13d, %r13d /* nospec r13 */
122	xorl %r14d, %r14d /* nospec r14 */	121	xorl %r14d, %r14d /* nospec r14 */
123	xorl %r15d, %r15d /* nospec r15 */	122	xorl %r15d, %r15d /* nospec r15 */
124		123	.endif
125	.endm	124	.endm
126		125
127	.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_bp=1 unwind_hint=1	126	.macro PUSH_AND_CLEAR_REGS rdx=%rdx rcx=%rcx rax=%rax save_ret=0 clear_callee=1 unwind_hint=1
128	PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint	127	PUSH_REGS rdx=\rdx, rcx=\rcx, rax=\rax, save_ret=\save_ret unwind_hint=\unwind_hint
129	CLEAR_REGS clear_bp=\clear_bp	128	CLEAR_REGS clear_callee=\clear_callee
130	.endm	129	.endm
131		130
132	.macro POP_REGS pop_rdi=1	131	.macro POP_REGS pop_rdi=1

arch/x86/entry/entry_64_fred.S+26−7

@@ -111,18 +111,37 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) · 110 unmodified lines
111	push %rax /* Return RIP */	111	push %rax /* Return RIP */
112	push $0 /* Error code, 0 for IRQ/NMI */	112	push $0 /* Error code, 0 for IRQ/NMI */
113		113
114	PUSH_AND_CLEAR_REGS clear_bp=0 unwind_hint=0	114	PUSH_AND_CLEAR_REGS clear_callee=0 unwind_hint=0
		115
115	movq %rsp, %rdi /* %rdi -> pt_regs */	116	movq %rsp, %rdi /* %rdi -> pt_regs */
		117	/*
		118	* At this point: {rdi, rsi, rdx, rcx, r8, r9}, {r10, r11}, {rax, rdx}
		119	* are clobbered, which corresponds to: arguments, extra caller-saved
		120	* and return. All registers a C function is allowed to clobber.
		121	*
		122	* Notably, the callee-saved registers: {rbx, r12, r13, r14, r15}
		123	* are untouched, with the exception of rbp, which carries the stack
		124	* frame and will be restored before exit.
		125	*
		126	* Further calling another C function will not alter this state.
		127	*/
116	call __fred_entry_from_kvm /* Call the C entry point */	128	call __fred_entry_from_kvm /* Call the C entry point */
117	POP_REGS	129
118	ERETS
119	1:
120	/*	130	/*
121	* Objtool doesn't understand what ERETS does, this hint tells it that	131	* When FRED, use ERETS to potentially clear NMIs, otherwise simply
122	* yes, we'll reach here and with what stack state. A save/restore pair	132	* restore the stack pointer.
123	* isn't strictly needed, but it's the simplest form.	133	*/
		134	ALTERNATIVE "nop; nop; mov %rbp, %rsp", \
		135	__stringify(add $C_PTREGS_SIZE, %rsp; ERETS), \
		136	X86_FEATURE_FRED
		137
		138	1: /*
		139	* Objtool doesn't understand ERETS, and the cfi register state is
		140	* different from initial_func_cfi due to PUSH_REGS. Tell it the state
		141	* is similar to where UNWIND_HINT_SAVE is.
124	*/	142	*/
125	UNWIND_HINT_RESTORE	143	UNWIND_HINT_RESTORE
		144
126	pop %rbp	145	pop %rbp
127	RET	146	RET
128		147

arch/x86/hyperv/hv_init.c+44−25

 #include <asm/desc.h>
 #include <asm/e820/api.h>
 #include <asm/sev.h>
-#include <asm/ibt.h>
 #include <asm/hypervisor.h>
 #include <hyperv/hvhdk.h>
 #include <asm/mshyperv.h>
 #include <linux/export.h>
 void *hv_hypercall_pg;
+#ifdef CONFIG_X86_64
+static u64 __hv_hyperfail(u64 control, u64 param1, u64 param2)
+{
+	return U64_MAX;
+}
+DEFINE_STATIC_CALL(__hv_hypercall, __hv_hyperfail);
+u64 hv_std_hypercall(u64 control, u64 param1, u64 param2)
+{
+	u64 hv_status;
+	register u64 __r8 asm("r8") = param2;
+	asm volatile ("call " STATIC_CALL_TRAMP_STR(__hv_hypercall)
+		      : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+		        "+c" (control), "+d" (param1), "+r" (__r8)
+		      : : "cc", "memory", "r9", "r10", "r11");
+	return hv_status;
+}
+typedef u64 (*hv_hypercall_f)(u64 control, u64 param1, u64 param2);
+static inline void hv_set_hypercall_pg(void *ptr)
+{
+	hv_hypercall_pg = ptr;
+	if (!ptr)
+		ptr = &__hv_hyperfail;
+	static_call_update(__hv_hypercall, (hv_hypercall_f)ptr);
+}
+#else
+static inline void hv_set_hypercall_pg(void *ptr)
+{
+	hv_hypercall_pg = ptr;
+}
 EXPORT_SYMBOL_GPL(hv_hypercall_pg);
+#endif
 union hv_ghcb * __percpu *hv_ghcb_pg;
 	 * pointer is restored on resume.
 	 */
 	hv_hypercall_pg_saved = hv_hypercall_pg;
-	hv_hypercall_pg = NULL;
+	hv_set_hypercall_pg(NULL);
 	/* Disable the hypercall page in the hypervisor */
 	rdmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 		vmalloc_to_pfn(hv_hypercall_pg_saved);
 	wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
-	hv_hypercall_pg = hv_hypercall_pg_saved;
+	hv_set_hypercall_pg(hv_hypercall_pg_saved);
 	hv_hypercall_pg_saved = NULL;
 	/*
 	if (hv_isolation_type_tdx() && !ms_hyperv.paravisor_present)
 		goto skip_hypercall_pg_init;
-	hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START,
+	hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR,
-			VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX,
+			MODULES_END, GFP_KERNEL, PAGE_KERNEL_ROX,
 			VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
 			__builtin_return_address(0));
 	if (hv_hypercall_pg == NULL)
 		wrmsrq(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 	}
-skip_hypercall_pg_init:
+	hv_set_hypercall_pg(hv_hypercall_pg);
-	/*
-	 * Some versions of Hyper-V that provide IBT in guest VMs have a bug
-	 * in that there's no ENDBR64 instruction at the entry to the
-	 * hypercall page. Because hypercalls are invoked via an indirect call
-	 * to the hypercall page, all hypercall attempts fail when IBT is
-	 * enabled, and Linux panics. For such buggy versions, disable IBT.
-	 *
-	 * Fixed versions of Hyper-V always provide ENDBR64 on the hypercall
-	 * page, so if future Linux kernel versions enable IBT for 32-bit
-	 * builds, additional hypercall page hackery will be required here
-	 * to provide an ENDBR32.
-	 */
-#ifdef CONFIG_X86_KERNEL_IBT
-	if (cpu_feature_enabled(X86_FEATURE_IBT) &&
-	    *(u32 *)hv_hypercall_pg != gen_endbr()) {
-		setup_clear_cpu_cap(X86_FEATURE_IBT);
-		pr_warn("Disabling IBT because of Hyper-V bug\n");
-	}
-#endif
+skip_hypercall_pg_init:
 	/*
 	 * hyperv_init() is called before LAPIC is initialized: see
 	 * apic_intr_mode_init() -> x86_platform.apic_post_init() and

arch/x86/hyperv/ivm.c+15−0

 	return ret;
 }
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2)
+{
+	u64 hv_status;
+	register u64 __r8 asm("r8") = param2;
+	asm volatile("vmmcall"
+		     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+		       "+c" (control), "+d" (param1), "+r" (__r8)
+		     : : "cc", "memory", "r9", "r10", "r11");
+	return hv_status;
+}
 #else
 static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
 static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_AMD_MEM_ENCRYPT */
 #ifdef CONFIG_INTEL_TDX_GUEST
 #else
 static inline void hv_tdx_msr_write(u64 msr, u64 value) {}
 static inline void hv_tdx_msr_read(u64 msr, u64 *value) {}
+u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_INTEL_TDX_GUEST */
 #if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST)

arch/x86/include/asm/bug.h+7−2

 #include <linux/stringify.h>
 #include <linux/instrumentation.h>
 #include <linux/objtool.h>
+#include <asm/asm.h>
 /*
  * Despite that some emulators terminate on UD2, we use it for WARN().
  */
-#define ASM_UD2		".byte 0x0f, 0x0b"
+#define ASM_UD2		_ASM_BYTES(0x0f, 0x0b)
 #define INSN_UD2	0x0b0f
 #define LEN_UD2		2
+#define ASM_UDB		_ASM_BYTES(0xd6)
+#define INSN_UDB	0xd6
+#define LEN_UDB		1
 /*
  * In clang we have UD1s reporting UBSAN failures on X86, 64 and 32bit.
  */
 #define BUG_UD2			0xfffe
 #define BUG_UD1			0xfffd
 #define BUG_UD1_UBSAN		0xfffc
-#define BUG_EA			0xffea
+#define BUG_UDB			0xffd6
 #define BUG_LOCK		0xfff0
 #ifdef CONFIG_GENERIC_BUG

arch/x86/include/asm/cfi.h+6−8

  *
  * __cfi_foo:
  *   endbr64
- *   subl 0x12345678, %r10d
+ *   subl 0x12345678, %eax
- *   jz   foo
+ *   jne.32,pn foo+3
- *   ud2
- *   nop
  * foo:
- *   osp nop3			# was endbr64
+ *   nopl -42(%rax)		# was endbr64
  *   ... code here ...
  *   ret
  *
  * indirect caller:
  *   lea foo(%rip), %r11
  *   ...
- *   movl $0x12345678, %r10d
+ *   movl $0x12345678, %eax
- *   subl $16, %r11
+ *   lea  -0x10(%r11), %r11
- *   nop4
+ *   nop5
  *   call *%r11
  *
  */

arch/x86/include/asm/ibt.h+3−7

 static __always_inline __attribute_const__ u32 gen_endbr_poison(void)
 {
 	/*
-	 * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
+	 * 4 byte NOP that isn't NOP4, such that it will be unique to (former)
-	 * will be unique to (former) ENDBR sites.
+	 * ENDBR sites. Additionally it carries UDB as immediate.
 	 */
-	return 0x001f0f66; /* osp nopl (%rax) */
+	return 0xd6401f0f; /* nopl -42(%rax) */
 }
 static inline bool __is_endbr(u32 val)
 	if (val == gen_endbr_poison())
 		return true;
-	/* See cfi_fineibt_bhi_preamble() */
-	if (IS_ENABLED(CONFIG_FINEIBT_BHI) && val == 0x001f0ff5)
-		return true;
 	val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
 	return val == gen_endbr();
 }

arch/x86/include/asm/idtentry.h+2−7

 #endif
 void idt_install_sysvec(unsigned int n, const void *function);
-#ifdef CONFIG_X86_FRED
 void fred_install_sysvec(unsigned int vector, const idtentry_t function);
-#else
-static inline void fred_install_sysvec(unsigned int vector, const idtentry_t function) { }
-#endif
 #define sysvec_install(vector, function) {				\
-	if (cpu_feature_enabled(X86_FEATURE_FRED))			\
+	if (IS_ENABLED(CONFIG_X86_FRED))				\
 		fred_install_sysvec(vector, function);			\
-	else								\
+	if (!cpu_feature_enabled(X86_FEATURE_FRED))			\
 		idt_install_sysvec(vector, asm_##function);		\
 }

arch/x86/include/asm/mshyperv.h+41−96

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#include <linux/nmi.h>	6	#include <linux/nmi.h>
7	#include <linux/msi.h>	7	#include <linux/msi.h>
8	#include <linux/io.h>	8	#include <linux/io.h>
		9	#include <linux/static_call.h>
9	#include <asm/nospec-branch.h>	10	#include <asm/nospec-branch.h>
10	#include <asm/paravirt.h>	11	#include <asm/paravirt.h>
11	#include <asm/msr.h>	12	#include <asm/msr.h>
@@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_reason(void) · 27 unmodified lines
39	return 0;	40	return 0;
40	}	41	}
41		42
42	#if IS_ENABLED(CONFIG_HYPERV)	43	extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
43	extern bool hyperv_paravisor_present;	44	extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2);
		45	extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2);
44		46
		47	#if IS_ENABLED(CONFIG_HYPERV)
45	extern void *hv_hypercall_pg;	48	extern void *hv_hypercall_pg;
46		49
47	extern union hv_ghcb * __percpu *hv_ghcb_pg;	50	extern union hv_ghcb * __percpu *hv_ghcb_pg;
48		51
49	bool hv_isolation_type_snp(void);	52	bool hv_isolation_type_snp(void);
50	bool hv_isolation_type_tdx(void);	53	bool hv_isolation_type_tdx(void);
51	u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);	54
		55	#ifdef CONFIG_X86_64
		56	DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
		57	#endif
52		58
53	/*	59	/*
54	* DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA	60	* DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA
@@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 control, void input, void output) · 10 unmodified lines
65	{	71	{
66	u64 input_address = input ? virt_to_phys(input) : 0;	72	u64 input_address = input ? virt_to_phys(input) : 0;
67	u64 output_address = output ? virt_to_phys(output) : 0;	73	u64 output_address = output ? virt_to_phys(output) : 0;
68	u64 hv_status;
69		74
70	#ifdef CONFIG_X86_64	75	#ifdef CONFIG_X86_64
71	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)	76	return static_call_mod(hv_hypercall)(control, input_address, output_address);
72	return hv_tdx_hypercall(control, input_address, output_address);
73
74	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
75	__asm__ __volatile__("mov %[output_address], %%r8\n"
76	"vmmcall"
77	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
78	"+c" (control), "+d" (input_address)
79	: [output_address] "r" (output_address)
80	: "cc", "memory", "r8", "r9", "r10", "r11");
81	return hv_status;
82	}
83
84	if (!hv_hypercall_pg)
85	return U64_MAX;
86
87	__asm__ __volatile__("mov %[output_address], %%r8\n"
88	CALL_NOSPEC
89	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
90	"+c" (control), "+d" (input_address)
91	: [output_address] "r" (output_address),
92	THUNK_TARGET(hv_hypercall_pg)
93	: "cc", "memory", "r8", "r9", "r10", "r11");
94	#else	77	#else
95	u32 input_address_hi = upper_32_bits(input_address);	78	u32 input_address_hi = upper_32_bits(input_address);
96	u32 input_address_lo = lower_32_bits(input_address);	79	u32 input_address_lo = lower_32_bits(input_address);
97	u32 output_address_hi = upper_32_bits(output_address);	80	u32 output_address_hi = upper_32_bits(output_address);
98	u32 output_address_lo = lower_32_bits(output_address);	81	u32 output_address_lo = lower_32_bits(output_address);
		82	u64 hv_status;
99		83
100	if (!hv_hypercall_pg)	84	if (!hv_hypercall_pg)
101	return U64_MAX;	85	return U64_MAX;
@@ -108,48 +92,30 @@ static inline u64 hv_do_hypercall(u64 control, void input, void output) · 6 unmodified lines
108	"D"(output_address_hi), "S"(output_address_lo),	92	"D"(output_address_hi), "S"(output_address_lo),
109	THUNK_TARGET(hv_hypercall_pg)	93	THUNK_TARGET(hv_hypercall_pg)
110	: "cc", "memory");	94	: "cc", "memory");
111	#endif /* !x86_64 */
112	return hv_status;	95	return hv_status;
		96	#endif /* !x86_64 */
113	}	97	}
114		98
115	/* Fast hypercall with 8 bytes of input and no output */	99	/* Fast hypercall with 8 bytes of input and no output */
116	static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)	100	static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)
117	{	101	{
118	u64 hv_status;
119
120	#ifdef CONFIG_X86_64	102	#ifdef CONFIG_X86_64
121	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)	103	return static_call_mod(hv_hypercall)(control, input1, 0);
122	return hv_tdx_hypercall(control, input1, 0);
123
124	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
125	__asm__ __volatile__(
126	"vmmcall"
127	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
128	"+c" (control), "+d" (input1)
129	:: "cc", "r8", "r9", "r10", "r11");
130	} else {
131	__asm__ __volatile__(CALL_NOSPEC
132	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
133	"+c" (control), "+d" (input1)
134	: THUNK_TARGET(hv_hypercall_pg)
135	: "cc", "r8", "r9", "r10", "r11");
136	}
137	#else	104	#else
138	{	105	u32 input1_hi = upper_32_bits(input1);
139	u32 input1_hi = upper_32_bits(input1);	106	u32 input1_lo = lower_32_bits(input1);
140	u32 input1_lo = lower_32_bits(input1);	107	u64 hv_status;
141		108
142	__asm__ __volatile__ (CALL_NOSPEC	109	__asm__ __volatile__ (CALL_NOSPEC
143	: "=A"(hv_status),	110	: "=A"(hv_status),
144	"+c"(input1_lo),	111	"+c"(input1_lo),
145	ASM_CALL_CONSTRAINT	112	ASM_CALL_CONSTRAINT
146	: "A" (control),	113	: "A" (control),
147	"b" (input1_hi),	114	"b" (input1_hi),
148	THUNK_TARGET(hv_hypercall_pg)	115	THUNK_TARGET(hv_hypercall_pg)
149	: "cc", "edi", "esi");	116	: "cc", "edi", "esi");
150	}
151	#endif
152	return hv_status;	117	return hv_status;
		118	#endif
153	}	119	}
154		120
155	static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)	121	static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -162,45 +128,24 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) · 6 unmodified lines
162	/* Fast hypercall with 16 bytes of input */	128	/* Fast hypercall with 16 bytes of input */
163	static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)	129	static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)
164	{	130	{
165	u64 hv_status;
166
167	#ifdef CONFIG_X86_64	131	#ifdef CONFIG_X86_64
168	if (hv_isolation_type_tdx() && !hyperv_paravisor_present)	132	return static_call_mod(hv_hypercall)(control, input1, input2);
169	return hv_tdx_hypercall(control, input1, input2);
170
171	if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
172	__asm__ __volatile__("mov %[input2], %%r8\n"
173	"vmmcall"
174	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
175	"+c" (control), "+d" (input1)
176	: [input2] "r" (input2)
177	: "cc", "r8", "r9", "r10", "r11");
178	} else {
179	__asm__ __volatile__("mov %[input2], %%r8\n"
180	CALL_NOSPEC
181	: "=a" (hv_status), ASM_CALL_CONSTRAINT,
182	"+c" (control), "+d" (input1)
183	: [input2] "r" (input2),
184	THUNK_TARGET(hv_hypercall_pg)
185	: "cc", "r8", "r9", "r10", "r11");
186	}
187	#else	133	#else
188	{	134	u32 input1_hi = upper_32_bits(input1);
189	u32 input1_hi = upper_32_bits(input1);	135	u32 input1_lo = lower_32_bits(input1);
190	u32 input1_lo = lower_32_bits(input1);	136	u32 input2_hi = upper_32_bits(input2);
191	u32 input2_hi = upper_32_bits(input2);	137	u32 input2_lo = lower_32_bits(input2);
192	u32 input2_lo = lower_32_bits(input2);	138	u64 hv_status;
193		139
194	__asm__ __volatile__ (CALL_NOSPEC	140	__asm__ __volatile__ (CALL_NOSPEC
195	: "=A"(hv_status),	141	: "=A"(hv_status),
196	"+c"(input1_lo), ASM_CALL_CONSTRAINT	142	"+c"(input1_lo), ASM_CALL_CONSTRAINT
197	: "A" (control), "b" (input1_hi),	143	: "A" (control), "b" (input1_hi),
198	"D"(input2_hi), "S"(input2_lo),	144	"D"(input2_hi), "S"(input2_lo),
199	THUNK_TARGET(hv_hypercall_pg)	145	THUNK_TARGET(hv_hypercall_pg)
200	: "cc");	146	: "cc");
201	}
202	#endif
203	return hv_status;	147	return hv_status;
		148	#endif
204	}	149	}
205		150
206	static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)	151	static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)

arch/x86/include/asm/text-patching.h+13−7

 }
 static __always_inline
-void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
+bool __emulate_cc(unsigned long flags, u8 cc)
 {
-	static const unsigned long jcc_mask[6] = {
+	static const unsigned long cc_mask[6] = {
 		[0] = X86_EFLAGS_OF,
 		[1] = X86_EFLAGS_CF,
 		[2] = X86_EFLAGS_ZF,
 	bool match;
 	if (cc < 0xc) {
-		match = regs->flags & jcc_mask[cc >> 1];
+		match = flags & cc_mask[cc >> 1];
 	} else {
-		match = ((regs->flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
+		match = ((flags & X86_EFLAGS_SF) >> X86_EFLAGS_SF_BIT) ^
-			((regs->flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
+			((flags & X86_EFLAGS_OF) >> X86_EFLAGS_OF_BIT);
 		if (cc >= 0xe)
-			match = match || (regs->flags & X86_EFLAGS_ZF);
+			match = match || (flags & X86_EFLAGS_ZF);
 	}
-	if ((match && !invert) || (!match && invert))
+	return (match && !invert) || (!match && invert);
+}
+static __always_inline
+void int3_emulate_jcc(struct pt_regs *regs, u8 cc, unsigned long ip, unsigned long disp)
+{
+	if (__emulate_cc(regs->flags, cc))
 		ip += disp;
 	int3_emulate_jmp(regs, ip);

arch/x86/kernel/alternative.c+176−116

 		/*
 		 * When ITS uses indirect branch thunk the fineibt_paranoid
 		 * caller sequence doesn't fit in the caller site. So put the
-		 * remaining part of the sequence (<ea> + JNE) into the ITS
+		 * remaining part of the sequence (UDB + JNE) into the ITS
 		 * thunk.
 		 */
-		bytes[i++] = 0xea; /* invalid instruction */
+		bytes[i++] = 0xd6; /* UDB */
 		bytes[i++] = 0x75; /* JNE */
 		bytes[i++] = 0xfd;
 		reg -= 8;
 	}
 	bytes[i++] = 0xff;
-	bytes[i++] = 0xe0 + reg; /* jmp *reg */
+	bytes[i++] = 0xe0 + reg; /* JMP *reg */
 	bytes[i++] = 0xcc;
 	return thunk + offset;
 #if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL)
 /*
- * CALL/JMP *%\reg
+ * [CS]{,3} CALL/JMP *%\reg [INT3]*
  */
-static int emit_indirect(int op, int reg, u8 *bytes)
+static int emit_indirect(int op, int reg, u8 *bytes, int len)
 {
+	int cs = 0, bp = 0;
 	int i = 0;
 	u8 modrm;
+	/*
+	 * Set @len to the excess bytes after writing the instruction.
+	 */
+	len -= 2 + (reg >= 8);
+	WARN_ON_ONCE(len < 0);
 	switch (op) {
 	case CALL_INSN_OPCODE:
 		modrm = 0x10; /* Reg = 2; CALL r/m */
+		/*
+		 * Additional NOP is better than prefix decode penalty.
+		 */
+		if (len <= 3)
+			cs = len;
 		break;
 	case JMP32_INSN_OPCODE:
 		modrm = 0x20; /* Reg = 4; JMP r/m */
+		bp = len;
 		break;
 	default:
 		return -1;
 	}
+	while (cs--)
+		bytes[i++] = 0x2e; /* CS-prefix */
 	if (reg >= 8) {
 		bytes[i++] = 0x41; /* REX.B prefix */
 		reg -= 8;
 	bytes[i++] = 0xff; /* opcode */
 	bytes[i++] = modrm;
+	while (bp--)
+		bytes[i++] = 0xcc; /* INT3 */
 	return i;
 }
 		return emit_its_trampoline(addr, insn, reg, bytes);
 #endif
-	ret = emit_indirect(op, reg, bytes + i);
+	ret = emit_indirect(op, reg, bytes + i, insn->length - i);
 	if (ret < 0)
 		return ret;
 	i += ret;
-	/*
-	 * The compiler is supposed to EMIT an INT3 after every unconditional
-	 * JMP instruction due to AMD BTC. However, if the compiler is too old
-	 * or MITIGATION_SLS isn't enabled, we still need an INT3 after
-	 * indirect JMPs even on Intel.
-	 */
-	if (op == JMP32_INSN_OPCODE && i < insn->length)
-		bytes[i++] = INT3_INSN_OPCODE;
 	for (; i < insn->length;)
 		bytes[i++] = BYTES_NOP1;
 		case JMP32_INSN_OPCODE:
 			/* Check for cfi_paranoid + ITS */
 			dest = addr + insn.length + insn.immediate.value;
-			if (dest[-1] == 0xea && (dest[0] & 0xf0) == 0x70) {
+			if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {
 				WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
 				continue;
 			}
 #endif
 enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
+static bool cfi_debug __ro_after_init;
 #ifdef CONFIG_FINEIBT_BHI
 bool cfi_bhi __ro_after_init = false;
 		} else if (!strcmp(str, "off")) {
 			cfi_mode = CFI_OFF;
 			cfi_rand = false;
+		} else if (!strcmp(str, "debug")) {
+			cfi_debug = true;
 		} else if (!strcmp(str, "kcfi")) {
 			cfi_mode = CFI_KCFI;
 		} else if (!strcmp(str, "fineibt")) {
 		} else if (!strcmp(str, "norand")) {
 			cfi_rand = false;
 		} else if (!strcmp(str, "warn")) {
-			pr_alert("CFI mismatch non-fatal!\n");
+			pr_alert("CFI: mismatch non-fatal!\n");
 			cfi_warn = true;
 		} else if (!strcmp(str, "paranoid")) {
 			if (cfi_mode == CFI_FINEIBT) {
 				cfi_paranoid = true;
 			} else {
-				pr_err("Ignoring paranoid; depends on fineibt.\n");
+				pr_err("CFI: ignoring paranoid; depends on fineibt.\n");
 			}
 		} else if (!strcmp(str, "bhi")) {
 #ifdef CONFIG_FINEIBT_BHI
 			if (cfi_mode == CFI_FINEIBT) {
 				cfi_bhi = true;
 			} else {
-				pr_err("Ignoring bhi; depends on fineibt.\n");
+				pr_err("CFI: ignoring bhi; depends on fineibt.\n");
 			}
 #else
-			pr_err("Ignoring bhi; depends on FINEIBT_BHI=y.\n");
+			pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n");
 #endif
 		} else {
-			pr_err("Ignoring unknown cfi option (%s).", str);
+			pr_err("CFI: Ignoring unknown option (%s).", str);
 		}
 		str = next;
  *
  * __cfi_\func:					__cfi_\func:
  *	movl   $0x12345678,%eax		// 5	     endbr64			// 4
- *	nop					     subl   $0x12345678,%r10d   // 7
+ *	nop					     subl   $0x12345678,%eax    // 5
- *	nop					     jne    __cfi_\func+6	// 2
+ *	nop					     jne.d32,pn \func+3		// 7
- *	nop					     nop3			// 3
+ *	nop
  *	nop
  *	nop
  *	nop
  *	nop
  *	nop
  *	nop
+ * \func:					\func:
+ *	endbr64					     nopl -42(%rax)
  *
  *
  * caller:					caller:
- *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%r10d	// 6
+ *	movl	$(-0x12345678),%r10d	 // 6	     movl   $0x12345678,%eax	// 5
  *	addl	$-15(%r11),%r10d	 // 4	     lea    -0x10(%r11),%r11	// 4
- *	je	1f			 // 2	     nop4			// 4
+ *	je	1f			 // 2	     nop5			// 5
  *	ud2				 // 2
  * 1:	cs call	__x86_indirect_thunk_r11 // 6	     call   *%r11; nop3;	// 6
  *
+ *
+ * Notably, the FineIBT sequences are crafted such that branches are presumed
+ * non-taken. This is based on Agner Fog's optimization manual, which states:
+ *
+ *  "Make conditional jumps most often not taken: The efficiency and throughput
+ *   for not-taken branches is better than for taken branches on most
+ *   processors. Therefore, it is good to place the most frequent branch first"
  */
 /*
  * <fineibt_preamble_start>:
  *  0:   f3 0f 1e fa             endbr64
- *  4:   41 81 <ea> 78 56 34 12  sub    $0x12345678, %r10d
+ *  4:   2d 78 56 34 12          sub    $0x12345678, %eax
- *  b:   75 f9                   jne    6 <fineibt_preamble_start+0x6>
+ *  9:   2e 0f 85 03 00 00 00    jne,pn 13 <fineibt_preamble_start+0x13>
- *  d:   0f 1f 00                nopl   (%rax)
+ * 10:   0f 1f 40 d6             nopl   -0x2a(%rax)
  *
- * Note that the JNE target is the 0xEA byte inside the SUB, this decodes as
+ * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
- * (bad) on x86_64 and raises #UD.
+ * UDB on x86_64 and raises #UD.
  */
 asm(	".pushsection .rodata				\n"
 	"fineibt_preamble_start:			\n"
 	"	endbr64					\n"
-	"	subl	$0x12345678, %r10d		\n"
+	"	subl	$0x12345678, %eax		\n"
 	"fineibt_preamble_bhi:				\n"
-	"	jne	fineibt_preamble_start+6	\n"
+	"	cs jne.d32 fineibt_preamble_start+0x13	\n"
-	ASM_NOP3
+	"#fineibt_func:					\n"
+	"	nopl	-42(%rax)			\n"
 	"fineibt_preamble_end:				\n"
 	".popsection\n"
 );
 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
 #define fineibt_preamble_bhi  (fineibt_preamble_bhi - fineibt_preamble_start)
-#define fineibt_preamble_ud   6
+#define fineibt_preamble_ud   0x13
-#define fineibt_preamble_hash 7
+#define fineibt_preamble_hash 5
 /*
  * <fineibt_caller_start>:
- *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d
+ *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
- *  6:   4d 8d 5b f0             lea    -0x10(%r11), %r11
+ *  5:   4d 8d 5b f0             lea    -0x10(%r11), %r11
- *  a:   0f 1f 40 00             nopl   0x0(%rax)
+ *  9:   0f 1f 44 00 00          nopl   0x0(%rax,%rax,1)
  */
 asm(	".pushsection .rodata			\n"
 	"fineibt_caller_start:			\n"
-	"	movl	$0x12345678, %r10d	\n"
+	"	movl	$0x12345678, %eax	\n"
 	"	lea	-0x10(%r11), %r11	\n"
-	ASM_NOP4
+	ASM_NOP5
 	"fineibt_caller_end:			\n"
 	".popsection				\n"
 );
 extern u8 fineibt_caller_end[];
 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
-#define fineibt_caller_hash 2
+#define fineibt_caller_hash 1
 #define fineibt_caller_jmp (fineibt_caller_size - 2)
  * of adding a load.
  *
  * <fineibt_paranoid_start>:
- *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d
+ *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
- *  6:   45 3b 53 f7             cmp    -0x9(%r11), %r10d
+ *  5:   41 3b 43 f5             cmp    -0x11(%r11), %eax
- *  a:   4d 8d 5b <f0>           lea    -0x10(%r11), %r11
+ *  9:   2e 4d 8d 5b <f0>        cs lea -0x10(%r11), %r11
  *  e:   75 fd                   jne    d <fineibt_paranoid_start+0xd>
  * 10:   41 ff d3                call   *%r11
  * 13:   90                      nop
  */
 asm(	".pushsection .rodata				\n"
 	"fineibt_paranoid_start:			\n"
-	"	movl	$0x12345678, %r10d		\n"
+	"	mov	$0x12345678, %eax		\n"
-	"	cmpl	-9(%r11), %r10d			\n"
+	"	cmpl	-11(%r11), %eax			\n"
-	"	lea	-0x10(%r11), %r11		\n"
+	"	cs lea	-0x10(%r11), %r11		\n"
+	"#fineibt_caller_size:                          \n"
 	"	jne	fineibt_paranoid_start+0xd	\n"
 	"fineibt_paranoid_ind:				\n"
-	"	call	*%r11				\n"
+	"	cs call	*%r11				\n"
-	"	nop					\n"
 	"fineibt_paranoid_end:				\n"
 	".popsection					\n"
 );
 	return 0;
 }
+/*
+ * Inline the bhi-arity 1 case:
+ *
+ * __cfi_foo:
+ *  0: f3 0f 1e fa             endbr64
+ *  4: 2d 78 56 34 12          sub    $0x12345678, %eax
+ *  9: 49 0f 45 fa             cmovne %rax, %rdi
+ *  d: 2e 75 03                jne,pn    foo+0x3
+ *
+ * foo:
+ * 10: 0f 1f 40 <d6>           nopl -42(%rax)
+ *
+ * Notably, this scheme is incompatible with permissive CFI
+ * because the CMOVcc is unconditional and RDI will have been
+ * clobbered.
+ */
+asm(	".pushsection .rodata				\n"
+	"fineibt_bhi1_start:				\n"
+	"	cmovne %rax, %rdi			\n"
+	"	cs jne fineibt_bhi1_func + 0x3		\n"
+	"fineibt_bhi1_func:				\n"
+	"	nopl -42(%rax)				\n"
+	"fineibt_bhi1_end:				\n"
+	".popsection					\n"
+);
+extern u8 fineibt_bhi1_start[];
+extern u8 fineibt_bhi1_end[];
+#define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
 static void cfi_fineibt_bhi_preamble(void *addr, int arity)
 {
+	u8 bytes[MAX_INSN_SIZE];
 	if (!arity)
 		return;
 	if (!cfi_warn && arity == 1) {
-		/*
+		text_poke_early(addr + fineibt_preamble_bhi,
-		 * Crazy scheme to allow arity-1 inline:
+				fineibt_bhi1_start, fineibt_bhi1_size);
-		 *
-		 * __cfi_foo:
-		 *  0: f3 0f 1e fa             endbr64
-		 *  4: 41 81 <ea> 78 56 34 12  sub     0x12345678, %r10d
-		 *  b: 49 0f 45 fa             cmovne  %r10, %rdi
-		 *  f: 75 f5                   jne     __cfi_foo+6
-		 * 11: 0f 1f 00                nopl    (%rax)
-		 *
-		 * Code that direct calls to foo()+0, decodes the tail end as:
-		 *
-		 * foo:
-		 *  0: f5                      cmc
-		 *  1: 0f 1f 00                nopl    (%rax)
-		 *
-		 * which clobbers CF, but does not affect anything ABI
-		 * wise.
-		 *
-		 * Notably, this scheme is incompatible with permissive CFI
-		 * because the CMOVcc is unconditional and RDI will have been
-		 * clobbered.
-		 */
-		const u8 magic[9] = {
-			0x49, 0x0f, 0x45, 0xfa,
-			0x75, 0xf5,
-			BYTES_NOP3,
-		};
-		text_poke_early(addr + fineibt_preamble_bhi, magic, 9);
 		return;
 	}
-	text_poke_early(addr + fineibt_preamble_bhi,
+	/*
-			text_gen_insn(CALL_INSN_OPCODE,
+	 * Replace the bytes at fineibt_preamble_bhi with a CALL instruction
-				      addr + fineibt_preamble_bhi,
+	 * that lines up exactly with the end of the preamble, such that the
-				      __bhi_args[arity]),
+	 * return address will be foo+0.
-			CALL_INSN_SIZE);
+	 *
+	 * __cfi_foo:
+	 *  0: f3 0f 1e fa             endbr64
+	 *  4: 2d 78 56 34 12          sub    $0x12345678, %eax
+	 *  9: 2e 2e e8 DD DD DD DD    cs cs call __bhi_args[arity]
+	 */
+	bytes[0] = 0x2e;
+	bytes[1] = 0x2e;
+	__text_gen_insn(bytes + 2, CALL_INSN_OPCODE,
+			addr + fineibt_preamble_bhi + 2,
+			__bhi_args[arity], CALL_INSN_SIZE);
+	text_poke_early(addr + fineibt_preamble_bhi, bytes, 7);
 }
 static int cfi_rewrite_preamble(s32 *start, s32 *end)
 {
 	s32 *s;
-	BUG_ON(fineibt_paranoid_size != 20);
 	for (s = start; s < end; s++) {
 		void *addr = (void *)s + *s;
 		struct insn insn;
 			emit_paranoid_trampoline(addr + fineibt_caller_size,
 						 &insn, 11, bytes + fineibt_caller_size);
 		} else {
-			ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind);
+			int len = fineibt_paranoid_size - fineibt_paranoid_ind;
-			if (WARN_ON_ONCE(ret != 3))
+			ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len);
+			if (WARN_ON_ONCE(ret != len))
 				continue;
 		}
 	return 0;
 }
+#define pr_cfi_debug(X...) if (cfi_debug) pr_info(X)
+#define FINEIBT_WARN(_f, _v) \
+	WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 			    s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
 	int ret;
-	if (WARN_ONCE(fineibt_preamble_size != 16,
+	if (FINEIBT_WARN(fineibt_preamble_size, 20)			||
-		      "FineIBT preamble wrong size: %ld", fineibt_preamble_size))
+	    FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20)	||
+	    FINEIBT_WARN(fineibt_caller_size, 14)			||
+	    FINEIBT_WARN(fineibt_paranoid_size, 20))
 		return;
 	if (cfi_mode == CFI_AUTO) {
 	 * rewrite them. This disables all CFI. If this succeeds but any of the
 	 * later stages fails, we're without CFI.
 	 */
+	pr_cfi_debug("CFI: disabling all indirect call checking\n");
 	ret = cfi_disable_callers(start_retpoline, end_retpoline);
 	if (ret)
 		goto err;
 			cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
 			cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
 		}
+		pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed);
+		pr_cfi_debug("CFI: rehashing all preambles\n");
 		ret = cfi_rand_preamble(start_cfi, end_cfi);
 		if (ret)
 			goto err;
+		pr_cfi_debug("CFI: rehashing all indirect calls\n");
 		ret = cfi_rand_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
+	} else {
+		pr_cfi_debug("CFI: rehashing disabled\n");
 	}
 	switch (cfi_mode) {
 	case CFI_OFF:
 		if (builtin)
-			pr_info("Disabling CFI\n");
+			pr_info("CFI: disabled\n");
 		return;
 	case CFI_KCFI:
+		pr_cfi_debug("CFI: re-enabling all indirect call checking\n");
 		ret = cfi_enable_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
 		if (builtin)
-			pr_info("Using kCFI\n");
+			pr_info("CFI: Using %sretpoline kCFI\n",
+				cfi_rand ? "rehashed " : "");
 		return;
 	case CFI_FINEIBT:
+		pr_cfi_debug("CFI: adding FineIBT to all preambles\n");
 		/* place the FineIBT preamble at func()-16 */
 		ret = cfi_rewrite_preamble(start_cfi, end_cfi);
 		if (ret)
 			goto err;
 		/* rewrite the callers to target func()-16 */
+		pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n");
 		ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
 		if (ret)
 			goto err;
 		/* now that nobody targets func()+0, remove ENDBR there */
+		pr_cfi_debug("CFI: removing old endbr insns\n");
 		cfi_rewrite_endbr(start_cfi, end_cfi);
 		if (builtin) {
 		/*
 		 * __cfi_\func:
-		 *	osp nopl (%rax)
+		 *	nopl	-42(%rax)
-		 *	subl	$0, %r10d
+		 *	sub	$0, %eax
-		 *	jz	1f
+		 *	jne	\func+3
-		 *	ud2
+		 * \func:
-		 * 1:	nop
+		 *	nopl	-42(%rax)
 		 */
 		poison_endbr(addr);
 		poison_hash(addr + fineibt_preamble_hash);
 	}
 }
+#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
 /*
- * When regs->ip points to a 0xEA byte in the FineIBT preamble,
+ * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
  * return true and fill out target and type.
  *
  * We check the preamble by checking for the ENDBR instruction relative to the
- * 0xEA instruction.
+ * UDB instruction.
  */
 static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type)
 {
 	if (!exact_endbr((void *)addr))
 		return false;
-	*target = addr + fineibt_preamble_size;
+	*target = addr + fineibt_prefix_size;
 	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
-	*type = (u32)regs->r10 + hash;
+	*type = (u32)regs->ax + hash;
 	/*
 	 * Since regs->ip points to the middle of an instruction; it cannot
 	__get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);
 	*target = addr;
-	addr -= fineibt_preamble_size;
+	addr -= fineibt_prefix_size;
 	if (!exact_endbr((void *)addr))
 		return false;
 	__get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
-	*type = (u32)regs->r10 + hash;
+	*type = (u32)regs->ax + hash;
 	/*
 	 * The UD2 sites are constructed with a RET immediately following,
 	u32 thunk;
 	__get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault);
-	return (thunk & 0x00FFFFFF) == 0xfd75ea;
+	return (thunk & 0x00FFFFFF) == 0xfd75d6;
 Efault:
 	return false;
 /*
  * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
- * sequence, or to an invalid instruction (0xea) + Jcc.d8 for cfi_paranoid + ITS
+ * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk.
- * thunk.
  */
 static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type)
 {
 		return false;
 	if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {
-		*target = regs->r11 + fineibt_preamble_size;
+		*target = regs->r11 + fineibt_prefix_size;
-		*type = regs->r10;
+		*type = regs->ax;
 		/*
 		 * Since the trapping instruction is the exact, but LOCK prefixed,
 	/*
 	 * The cfi_paranoid + ITS thunk combination results in:
 	 *
-	 *  0:   41 ba 78 56 34 12       mov    $0x12345678, %r10d
+	 *  0:   b8 78 56 34 12          mov    $0x12345678, %eax
-	 *  6:   45 3b 53 f7             cmp    -0x9(%r11), %r10d
+	 *  5:   41 3b 43 f7             cmp    -11(%r11), %eax
-	 *  a:   4d 8d 5b f0             lea    -0x10(%r11), %r11
+	 *  a:   2e 3d 8d 5b f0          cs lea -0x10(%r11), %r11
 	 *  e:   2e e8 XX XX XX XX	 cs call __x86_indirect_paranoid_thunk_r11
 	 *
 	 * Where the paranoid_thunk looks like:
 	 *
-	 *  1d:  <ea>                    (bad)
+	 *  1d:  <d6>                    udb
 	 *  __x86_indirect_paranoid_thunk_r11:
 	 *  1e:  75 fd                   jne 1d
 	 *  __x86_indirect_its_thunk_r11:
 	 *
 	 */
 	if (is_paranoid_thunk(regs->ip)) {
-		*target = regs->r11 + fineibt_preamble_size;
+		*target = regs->r11 + fineibt_prefix_size;
-		*type = regs->r10;
+		*type = regs->ax;
 		regs->ip = *target;
 		return true;
 static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
 			    s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
+	if (IS_ENABLED(CONFIG_CFI) && builtin)
+		pr_info("CFI: Using standard kCFI\n");
 }
 #ifdef CONFIG_X86_KERNEL_IBT
 	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,
 			__cfi_sites, __cfi_sites_end, true);
+	cfi_debug = false;
 	/*
 	 * Rewrite the retpolines, must be done before alternatives since

arch/x86/kernel/asm-offsets.c+1−0

@@ -102,6 +102,7 @@ static void __used common(void) · 101 unmodified lines
102		102
103	BLANK();	103	BLANK();
104	DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));	104	DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
		105	OFFSET(C_PTREGS_SIZE, pt_regs, orig_ax);
105		106
106	/* TLB state for the entry code */	107	/* TLB state for the entry code */
107	OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);	108	OFFSET(TLB_STATE_user_pcid_flush_mask, tlb_state, user_pcid_flush_mask);

arch/x86/kernel/cfi.c+1−1

@@ -27,7 +27,7 @@ static bool decode_cfi_insn(struct pt_regs regs, unsigned long target, · 26 unmodified lines
27	* for indirect call checks:	27	* for indirect call checks:
28	*	28	*
29	* movl -<id>, %r10d ; 6 bytes	29	* movl -<id>, %r10d ; 6 bytes
30	* addl -4(%reg), %r10d ; 4 bytes	30	* addl -<pos>(%reg), %r10d; 4 bytes
31	* je .Ltmp1 ; 2 bytes	31	* je .Ltmp1 ; 2 bytes
32	* ud2 ; <- regs->ip	32	* ud2 ; <- regs->ip
33	* .Ltmp1:	33	* .Ltmp1:

arch/x86/kernel/cpu/mshyperv.c+13−6

 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
-/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */
-bool hyperv_paravisor_present __ro_after_init;
-EXPORT_SYMBOL_GPL(hyperv_paravisor_present);
 #if IS_ENABLED(CONFIG_HYPERV)
 static inline unsigned int hv_get_nested_msr(unsigned int reg)
 {
 	old_restore_sched_clock_state = x86_platform.restore_sched_clock_state;
 	x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state;
 }
+#ifdef CONFIG_X86_64
+DEFINE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
+EXPORT_STATIC_CALL_TRAMP_GPL(hv_hypercall);
+#define hypercall_update(hc) static_call_update(hv_hypercall, hc)
+#endif
 #endif /* CONFIG_HYPERV */
+#ifndef hypercall_update
+#define hypercall_update(hc) (void)hc
+#endif
 static uint32_t  __init ms_hyperv_platform(void)
 {
 	u32 eax;
 			ms_hyperv.shared_gpa_boundary =
 				BIT_ULL(ms_hyperv.shared_gpa_boundary_bits);
-		hyperv_paravisor_present = !!ms_hyperv.paravisor_present;
 		pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n",
 			ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);
 		if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) {
 			static_branch_enable(&isolation_type_snp);
+			if (!ms_hyperv.paravisor_present)
+				hypercall_update(hv_snp_hypercall);
 		} else if (hv_get_isolation_type() == HV_ISOLATION_TYPE_TDX) {
 			static_branch_enable(&isolation_type_tdx);
 			ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED;
 			if (!ms_hyperv.paravisor_present) {
+				hypercall_update(hv_tdx_hypercall);
 				/*
 				 * Mark the Hyper-V TSC page feature as disabled
 				 * in a TDX VM without paravisor so that the

arch/x86/kernel/irqinit.c+4−2

@@ -97,9 +97,11 @@ void __init native_init_IRQ(void) · 96 unmodified lines
97	/* Execute any quirks before the call gates are initialised: */	97	/* Execute any quirks before the call gates are initialised: */
98	x86_init.irqs.pre_vector_init();	98	x86_init.irqs.pre_vector_init();
99		99
100	if (cpu_feature_enabled(X86_FEATURE_FRED))	100	/* FRED's IRQ path may be used even if FRED isn't fully enabled. */
		101	if (IS_ENABLED(CONFIG_X86_FRED))
101	fred_complete_exception_setup();	102	fred_complete_exception_setup();
102	else	103
		104	if (!cpu_feature_enabled(X86_FEATURE_FRED))
103	idt_setup_apic_and_irq_gates();	105	idt_setup_apic_and_irq_gates();
104		106
105	lapic_assign_system_vectors();	107	lapic_assign_system_vectors();

arch/x86/kernel/machine_kexec_64.c+4−0

@@ -479,6 +479,10 @@ void __nocfi machine_kexec(struct kimage *image) · 478 unmodified lines
479		479
480	__ftrace_enabled_restore(save_ftrace_enabled);	480	__ftrace_enabled_restore(save_ftrace_enabled);
481	}	481	}
		482	/*
		483	* Handover to the next kernel, no CFI concern.
		484	*/
		485	ANNOTATE_NOCFI_SYM(machine_kexec);
482		486
483	/* arch-dependent functionality related to kexec file-based syscall */	487	/* arch-dependent functionality related to kexec file-based syscall */
484		488

arch/x86/kernel/traps.c+4−4

@@ -97,7 +97,7 @@ __always_inline int is_valid_bugaddr(unsigned long addr) · 96 unmodified lines
97	* Check for UD1 or UD2, accounting for Address Size Override Prefixes.	97	* Check for UD1 or UD2, accounting for Address Size Override Prefixes.
98	* If it's a UD1, further decode to determine its use:	98	* If it's a UD1, further decode to determine its use:
99	*	99	*
100	* FineIBT: ea (bad)	100	* FineIBT: d6 udb
101	* FineIBT: f0 75 f9 lock jne . - 6	101	* FineIBT: f0 75 f9 lock jne . - 6
102	* UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax	102	* UBSan{0}: 67 0f b9 00 ud1 (%eax),%eax
103	* UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax	103	* UBSan{10}: 67 0f b9 40 10 ud1 0x10(%eax),%eax
@@ -130,9 +130,9 @@ __always_inline int decode_bug(unsigned long addr, s32 imm, int len) · 26 unmodified lines
130	WARN_ON_ONCE(!lock);	130	WARN_ON_ONCE(!lock);
131	return BUG_LOCK;	131	return BUG_LOCK;
132		132
133	case 0xea:	133	case 0xd6:
134	*len = addr - start;	134	*len = addr - start;
135	return BUG_EA;	135	return BUG_UDB;
136		136
137	case OPCODE_ESCAPE:	137	case OPCODE_ESCAPE:
138	break;	138	break;
@@ -341,7 +341,7 @@ static noinstr bool handle_bug(struct pt_regs *regs) · 202 unmodified lines
341	}	341	}
342	fallthrough;	342	fallthrough;
343		343
344	case BUG_EA:	344	case BUG_UDB:
345	case BUG_LOCK:	345	case BUG_LOCK:
346	if (handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {	346	if (handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
347	handled = true;	347	handled = true;

arch/x86/kvm/Kconfig+1−0

@@ -96,6 +96,7 @@ config KVM_SW_PROTECTED_VM · 95 unmodified lines
96	config KVM_INTEL	96	config KVM_INTEL
97	tristate "KVM for Intel (and compatible) processors support"	97	tristate "KVM for Intel (and compatible) processors support"
98	depends on KVM && IA32_FEAT_CTL	98	depends on KVM && IA32_FEAT_CTL
		99	select X86_FRED if X86_64
99	help	100	help
100	Provides support for KVM on processors equipped with Intel's VT	101	Provides support for KVM on processors equipped with Intel's VT
101	extensions, a.k.a. Virtual Machine Extensions (VMX).	102	extensions, a.k.a. Virtual Machine Extensions (VMX).

arch/x86/kvm/emulate.c+232−318

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	#include <asm/debugreg.h>	26	#include <asm/debugreg.h>
27	#include <asm/nospec-branch.h>	27	#include <asm/nospec-branch.h>
28	#include <asm/ibt.h>	28	#include <asm/ibt.h>
		29	#include <asm/text-patching.h>
29		30
30	#include "x86.h"	31	#include "x86.h"
31	#include "tss.h"	32	#include "tss.h"
@@ -166,7 +167,6 @@ · 134 unmodified lines
166	#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */	167	#define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
167	#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */	168	#define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
168	#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */	169	#define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
169	#define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
170	#define NoWrite ((u64)1 << 45) /* No writeback */	170	#define NoWrite ((u64)1 << 45) /* No writeback */
171	#define SrcWrite ((u64)1 << 46) /* Write back src operand */	171	#define SrcWrite ((u64)1 << 46) /* Write back src operand */
172	#define NoMod ((u64)1 << 47) /* Mod field is ignored */	172	#define NoMod ((u64)1 << 47) /* Mod field is ignored */
@@ -203,7 +203,6 @@ struct opcode { · 30 unmodified lines
203	const struct escape *esc;	203	const struct escape *esc;
204	const struct instr_dual *idual;	204	const struct instr_dual *idual;
205	const struct mode_dual *mdual;	205	const struct mode_dual *mdual;
206	void (fastop)(struct fastop fake);
207	} u;	206	} u;
208	int (check_perm)(struct x86_emulate_ctxt ctxt);	207	int (check_perm)(struct x86_emulate_ctxt ctxt);
209	};	208	};
@@ -267,186 +266,130 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt) · 57 unmodified lines
267	X86_EFLAGS_PF\|X86_EFLAGS_CF)	266	X86_EFLAGS_PF\|X86_EFLAGS_CF)
268		267
269	#ifdef CONFIG_X86_64	268	#ifdef CONFIG_X86_64
270	#define ON64(x) x	269	#define ON64(x...) x
271	#else	270	#else
272	#define ON64(x)	271	#define ON64(x...)
273	#endif	272	#endif
274		273
275	/*	274	#define EM_ASM_START(op) \
276	* fastop functions have a special calling convention:	275	static int em_##op(struct x86_emulate_ctxt *ctxt) \
277	*	276	{ \
278	* dst: rax (in/out)	277	unsigned long flags = (ctxt->eflags & EFLAGS_MASK) \| X86_EFLAGS_IF; \
279	* src: rdx (in/out)	278	int bytes = 1, ok = 1; \
280	* src2: rcx (in)	279	if (!(ctxt->d & ByteOp)) \
281	* flags: rflags (in/out)	280	bytes = ctxt->dst.bytes; \
282	* ex: rsi (in:fastop pointer, out:zero if exception)	281	switch (bytes) {
283	*	282
284	* Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for	283	#define __EM_ASM(str) \
285	* different operand sizes can be reached by calculation, rather than a jump	284	asm("push %[flags]; popf \n\t" \
286	* table (which would be bigger than the code).	285	"10: " str \
287	*	286	"pushf; pop %[flags] \n\t" \
288	* The 16 byte alignment, considering 5 bytes for the RET thunk, 3 for ENDBR	287	"11: \n\t" \
289	* and 1 for the straight line speculation INT3, leaves 7 bytes for the	288	: "+a" (ctxt->dst.val), \
290	* body of the function. Currently none is larger than 4.	289	"+d" (ctxt->src.val), \
291	*/	290	[flags] "+D" (flags), \
292	static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);	291	"+S" (ok) \
293		292	: "c" (ctxt->src2.val))
294	#define FASTOP_SIZE 16	293
295		294	#define __EM_ASM_1(op, dst) \
296	#define __FOP_FUNC(name) \	295	__EM_ASM(#op " %%" #dst " \n\t")
297	".align " __stringify(FASTOP_SIZE) " \n\t" \	296
298	".type " name ", @function \n\t" \	297	#define __EM_ASM_1_EX(op, dst) \
299	name ":\n\t" \	298	__EM_ASM(#op " %%" #dst " \n\t" \
300	ASM_ENDBR \	299	_ASM_EXTABLE_TYPE_REG(10b, 11f, EX_TYPE_ZERO_REG, %%esi))
301	IBT_NOSEAL(name)	300
302		301	#define __EM_ASM_2(op, dst, src) \
303	#define FOP_FUNC(name) \	302	__EM_ASM(#op " %%" #src ", %%" #dst " \n\t")
304	__FOP_FUNC(#name)	303
305		304	#define __EM_ASM_3(op, dst, src, src2) \
306	#define __FOP_RET(name) \	305	__EM_ASM(#op " %%" #src2 ", %%" #src ", %%" #dst " \n\t")
307	"11: " ASM_RET \	306
308	".size " name ", .-" name "\n\t"	307	#define EM_ASM_END \
309		308	} \
310	#define FOP_RET(name) \	309	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) \| (flags & EFLAGS_MASK); \
311	__FOP_RET(#name)	310	return !ok ? emulate_de(ctxt) : X86EMUL_CONTINUE; \
312		311	}
313	#define __FOP_START(op, align) \	312
314	extern void em_##op(struct fastop *fake); \	313	/* 1-operand, using "a" (dst) */
315	asm(".pushsection .text, \"ax\" \n\t" \	314	#define EM_ASM_1(op) \
316	".global em_" #op " \n\t" \	315	EM_ASM_START(op) \
317	".align " __stringify(align) " \n\t" \	316	case 1: __EM_ASM_1(op##b, al); break; \
318	"em_" #op ":\n\t"	317	case 2: __EM_ASM_1(op##w, ax); break; \
319		318	case 4: __EM_ASM_1(op##l, eax); break; \
320	#define FOP_START(op) __FOP_START(op, FASTOP_SIZE)	319	ON64(case 8: __EM_ASM_1(op##q, rax); break;) \
321		320	EM_ASM_END
322	#define FOP_END \	321
323	".popsection")	322	/* 1-operand, using "c" (src2) */
324		323	#define EM_ASM_1SRC2(op, name) \
325	#define __FOPNOP(name) \	324	EM_ASM_START(name) \
326	__FOP_FUNC(name) \	325	case 1: __EM_ASM_1(op##b, cl); break; \
327	__FOP_RET(name)	326	case 2: __EM_ASM_1(op##w, cx); break; \
328		327	case 4: __EM_ASM_1(op##l, ecx); break; \
329	#define FOPNOP() \	328	ON64(case 8: __EM_ASM_1(op##q, rcx); break;) \
330	__FOPNOP(__stringify(__UNIQUE_ID(nop)))	329	EM_ASM_END
331		330
332	#define FOP1E(op, dst) \	331	/* 1-operand, using "c" (src2) with exception */
333	__FOP_FUNC(#op "_" #dst) \	332	#define EM_ASM_1SRC2EX(op, name) \
334	"10: " #op " %" #dst " \n\t" \	333	EM_ASM_START(name) \
335	__FOP_RET(#op "_" #dst)	334	case 1: __EM_ASM_1_EX(op##b, cl); break; \
336		335	case 2: __EM_ASM_1_EX(op##w, cx); break; \
337	#define FOP1EEX(op, dst) \	336	case 4: __EM_ASM_1_EX(op##l, ecx); break; \
338	FOP1E(op, dst) _ASM_EXTABLE_TYPE_REG(10b, 11b, EX_TYPE_ZERO_REG, %%esi)	337	ON64(case 8: __EM_ASM_1_EX(op##q, rcx); break;) \
339		338	EM_ASM_END
340	#define FASTOP1(op) \	339
341	FOP_START(op) \	340	/* 2-operand, using "a" (dst), "d" (src) */
342	FOP1E(op##b, al) \	341	#define EM_ASM_2(op) \
343	FOP1E(op##w, ax) \	342	EM_ASM_START(op) \
344	FOP1E(op##l, eax) \	343	case 1: __EM_ASM_2(op##b, al, dl); break; \
345	ON64(FOP1E(op##q, rax)) \	344	case 2: __EM_ASM_2(op##w, ax, dx); break; \
346	FOP_END	345	case 4: __EM_ASM_2(op##l, eax, edx); break; \
347		346	ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \
348	/* 1-operand, using src2 (for MUL/DIV r/m) */	347	EM_ASM_END
349	#define FASTOP1SRC2(op, name) \	348
350	FOP_START(name) \	349	/* 2-operand, reversed */
351	FOP1E(op, cl) \	350	#define EM_ASM_2R(op, name) \
352	FOP1E(op, cx) \	351	EM_ASM_START(name) \
353	FOP1E(op, ecx) \	352	case 1: __EM_ASM_2(op##b, dl, al); break; \
354	ON64(FOP1E(op, rcx)) \	353	case 2: __EM_ASM_2(op##w, dx, ax); break; \
355	FOP_END	354	case 4: __EM_ASM_2(op##l, edx, eax); break; \
356		355	ON64(case 8: __EM_ASM_2(op##q, rdx, rax); break;) \
357	/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */	356	EM_ASM_END
358	#define FASTOP1SRC2EX(op, name) \	357
359	FOP_START(name) \	358	/* 2-operand, word only (no byte op) */
360	FOP1EEX(op, cl) \	359	#define EM_ASM_2W(op) \
361	FOP1EEX(op, cx) \	360	EM_ASM_START(op) \
362	FOP1EEX(op, ecx) \	361	case 1: break; \
363	ON64(FOP1EEX(op, rcx)) \	362	case 2: __EM_ASM_2(op##w, ax, dx); break; \
364	FOP_END	363	case 4: __EM_ASM_2(op##l, eax, edx); break; \
365		364	ON64(case 8: __EM_ASM_2(op##q, rax, rdx); break;) \
366	#define FOP2E(op, dst, src) \	365	EM_ASM_END
367	__FOP_FUNC(#op "_" #dst "_" #src) \	366
368	#op " %" #src ", %" #dst " \n\t" \	367	/* 2-operand, using "a" (dst) and CL (src2) */
369	__FOP_RET(#op "_" #dst "_" #src)	368	#define EM_ASM_2CL(op) \
370		369	EM_ASM_START(op) \
371	#define FASTOP2(op) \	370	case 1: __EM_ASM_2(op##b, al, cl); break; \
372	FOP_START(op) \	371	case 2: __EM_ASM_2(op##w, ax, cl); break; \
373	FOP2E(op##b, al, dl) \	372	case 4: __EM_ASM_2(op##l, eax, cl); break; \
374	FOP2E(op##w, ax, dx) \	373	ON64(case 8: __EM_ASM_2(op##q, rax, cl); break;) \
375	FOP2E(op##l, eax, edx) \	374	EM_ASM_END
376	ON64(FOP2E(op##q, rax, rdx)) \	375
377	FOP_END	376	/* 3-operand, using "a" (dst), "d" (src) and CL (src2) */
378		377	#define EM_ASM_3WCL(op) \
379	/* 2 operand, word only */	378	EM_ASM_START(op) \
380	#define FASTOP2W(op) \	379	case 1: break; \
381	FOP_START(op) \	380	case 2: __EM_ASM_3(op##w, ax, dx, cl); break; \
382	FOPNOP() \	381	case 4: __EM_ASM_3(op##l, eax, edx, cl); break; \
383	FOP2E(op##w, ax, dx) \	382	ON64(case 8: __EM_ASM_3(op##q, rax, rdx, cl); break;) \
384	FOP2E(op##l, eax, edx) \	383	EM_ASM_END
385	ON64(FOP2E(op##q, rax, rdx)) \	384
386	FOP_END	385	static int em_salc(struct x86_emulate_ctxt *ctxt)
387		386	{
388	/* 2 operand, src is CL */	387	/*
389	#define FASTOP2CL(op) \	388	* Set AL 0xFF if CF is set, or 0x00 when clear.
390	FOP_START(op) \	389	*/
391	FOP2E(op##b, al, cl) \	390	ctxt->dst.val = 0xFF * !!(ctxt->eflags & X86_EFLAGS_CF);
392	FOP2E(op##w, ax, cl) \	391	return X86EMUL_CONTINUE;
393	FOP2E(op##l, eax, cl) \	392	}
394	ON64(FOP2E(op##q, rax, cl)) \
395	FOP_END
396
397	/* 2 operand, src and dest are reversed */
398	#define FASTOP2R(op, name) \
399	FOP_START(name) \
400	FOP2E(op##b, dl, al) \
401	FOP2E(op##w, dx, ax) \
402	FOP2E(op##l, edx, eax) \
403	ON64(FOP2E(op##q, rdx, rax)) \
404	FOP_END
405
406	#define FOP3E(op, dst, src, src2) \
407	__FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408	#op " %" #src2 ", %" #src ", %" #dst " \n\t"\
409	__FOP_RET(#op "_" #dst "_" #src "_" #src2)
410
411	/* 3-operand, word-only, src2=cl */
412	#define FASTOP3WCL(op) \
413	FOP_START(op) \
414	FOPNOP() \
415	FOP3E(op##w, ax, dx, cl) \
416	FOP3E(op##l, eax, edx, cl) \
417	ON64(FOP3E(op##q, rax, rdx, cl)) \
418	FOP_END
419
420	/* Special case for SETcc - 1 instruction per cc */
421	#define FOP_SETCC(op) \
422	FOP_FUNC(op) \
423	#op " %al \n\t" \
424	FOP_RET(op)
425
426	FOP_START(setcc)
427	FOP_SETCC(seto)
428	FOP_SETCC(setno)
429	FOP_SETCC(setc)
430	FOP_SETCC(setnc)
431	FOP_SETCC(setz)
432	FOP_SETCC(setnz)
433	FOP_SETCC(setbe)
434	FOP_SETCC(setnbe)
435	FOP_SETCC(sets)
436	FOP_SETCC(setns)
437	FOP_SETCC(setp)
438	FOP_SETCC(setnp)
439	FOP_SETCC(setl)
440	FOP_SETCC(setnl)
441	FOP_SETCC(setle)
442	FOP_SETCC(setnle)
443	FOP_END;
444
445	FOP_START(salc)
446	FOP_FUNC(salc)
447	"pushf; sbb %al, %al; popf \n\t"
448	FOP_RET(salc)
449	FOP_END;
450		393
451	/*	394	/*
452	* XXX: inoutclob user must know where the argument is being expanded.	395	* XXX: inoutclob user must know where the argument is being expanded.
@@ -1007,56 +950,55 @@ static int read_descriptor(struct x86_emulate_ctxt *ctxt, · 554 unmodified lines
1007	return rc;	950	return rc;
1008	}	951	}
1009		952
1010	FASTOP2(add);	953	EM_ASM_2(add);
1011	FASTOP2(or);	954	EM_ASM_2(or);
1012	FASTOP2(adc);	955	EM_ASM_2(adc);
1013	FASTOP2(sbb);	956	EM_ASM_2(sbb);
1014	FASTOP2(and);	957	EM_ASM_2(and);
1015	FASTOP2(sub);	958	EM_ASM_2(sub);
1016	FASTOP2(xor);	959	EM_ASM_2(xor);
1017	FASTOP2(cmp);	960	EM_ASM_2(cmp);
1018	FASTOP2(test);	961	EM_ASM_2(test);
1019		962	EM_ASM_2(xadd);
1020	FASTOP1SRC2(mul, mul_ex);	963
1021	FASTOP1SRC2(imul, imul_ex);	964	EM_ASM_1SRC2(mul, mul_ex);
1022	FASTOP1SRC2EX(div, div_ex);	965	EM_ASM_1SRC2(imul, imul_ex);
1023	FASTOP1SRC2EX(idiv, idiv_ex);	966	EM_ASM_1SRC2EX(div, div_ex);
1024		967	EM_ASM_1SRC2EX(idiv, idiv_ex);
1025	FASTOP3WCL(shld);	968
1026	FASTOP3WCL(shrd);	969	EM_ASM_3WCL(shld);
1027		970	EM_ASM_3WCL(shrd);
1028	FASTOP2W(imul);	971
1029		972	EM_ASM_2W(imul);
1030	FASTOP1(not);	973
1031	FASTOP1(neg);	974	EM_ASM_1(not);
1032	FASTOP1(inc);	975	EM_ASM_1(neg);
1033	FASTOP1(dec);	976	EM_ASM_1(inc);
1034		977	EM_ASM_1(dec);
1035	FASTOP2CL(rol);	978
1036	FASTOP2CL(ror);	979	EM_ASM_2CL(rol);
1037	FASTOP2CL(rcl);	980	EM_ASM_2CL(ror);
1038	FASTOP2CL(rcr);	981	EM_ASM_2CL(rcl);
1039	FASTOP2CL(shl);	982	EM_ASM_2CL(rcr);
1040	FASTOP2CL(shr);	983	EM_ASM_2CL(shl);
1041	FASTOP2CL(sar);	984	EM_ASM_2CL(shr);
1042		985	EM_ASM_2CL(sar);
1043	FASTOP2W(bsf);	986
1044	FASTOP2W(bsr);	987	EM_ASM_2W(bsf);
1045	FASTOP2W(bt);	988	EM_ASM_2W(bsr);
1046	FASTOP2W(bts);	989	EM_ASM_2W(bt);
1047	FASTOP2W(btr);	990	EM_ASM_2W(bts);
1048	FASTOP2W(btc);	991	EM_ASM_2W(btr);
1049		992	EM_ASM_2W(btc);
1050	FASTOP2(xadd);	993
1051		994	EM_ASM_2R(cmp, cmp_r);
1052	FASTOP2R(cmp, cmp_r);
1053		995
1054	static int em_bsf_c(struct x86_emulate_ctxt *ctxt)	996	static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1055	{	997	{
1056	/* If src is zero, do not writeback, but update flags */	998	/* If src is zero, do not writeback, but update flags */
1057	if (ctxt->src.val == 0)	999	if (ctxt->src.val == 0)
1058	ctxt->dst.type = OP_NONE;	1000	ctxt->dst.type = OP_NONE;
1059	return fastop(ctxt, em_bsf);	1001	return em_bsf(ctxt);
1060	}	1002	}
1061		1003
1062	static int em_bsr_c(struct x86_emulate_ctxt *ctxt)	1004	static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
@@ -1064,18 +1006,12 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt) · 1 unmodified line
1064	/* If src is zero, do not writeback, but update flags */	1006	/* If src is zero, do not writeback, but update flags */
1065	if (ctxt->src.val == 0)	1007	if (ctxt->src.val == 0)
1066	ctxt->dst.type = OP_NONE;	1008	ctxt->dst.type = OP_NONE;
1067	return fastop(ctxt, em_bsr);	1009	return em_bsr(ctxt);
1068	}	1010	}
1069		1011
1070	static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)	1012	static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1071	{	1013	{
1072	u8 rc;	1014	return __emulate_cc(flags, condition & 0xf);
1073	void (fop)(void) = (void )em_setcc + FASTOP_SIZE * (condition & 0xf);
1074
1075	flags = (flags & EFLAGS_MASK) \| X86_EFLAGS_IF;
1076	asm("push %[flags]; popf; " CALL_NOSPEC
1077	: "=a"(rc), ASM_CALL_CONSTRAINT : [thunk_target]"r"(fop), [flags]"r"(flags));
1078	return rc;
1079	}	1015	}
1080		1016
1081	static void fetch_register_operand(struct operand *op)	1017	static void fetch_register_operand(struct operand *op)
@@ -2325,7 +2261,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt) · 1243 unmodified lines
2325	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);	2261	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2326	ctxt->src.orig_val = ctxt->src.val;	2262	ctxt->src.orig_val = ctxt->src.val;
2327	ctxt->src.val = ctxt->dst.orig_val;	2263	ctxt->src.val = ctxt->dst.orig_val;
2328	fastop(ctxt, em_cmp);	2264	em_cmp(ctxt);
2329		2265
2330	if (ctxt->eflags & X86_EFLAGS_ZF) {	2266	if (ctxt->eflags & X86_EFLAGS_ZF) {
2331	/* Success: write back to memory; no update of EAX */	2267	/* Success: write back to memory; no update of EAX */
@@ -3090,7 +3026,7 @@ static int em_das(struct x86_emulate_ctxt *ctxt) · 758 unmodified lines
3090	ctxt->src.type = OP_IMM;	3026	ctxt->src.type = OP_IMM;
3091	ctxt->src.val = 0;	3027	ctxt->src.val = 0;
3092	ctxt->src.bytes = 1;	3028	ctxt->src.bytes = 1;
3093	fastop(ctxt, em_or);	3029	em_or(ctxt);
3094	ctxt->eflags &= ~(X86_EFLAGS_AF \| X86_EFLAGS_CF);	3030	ctxt->eflags &= ~(X86_EFLAGS_AF \| X86_EFLAGS_CF);
3095	if (cf)	3031	if (cf)
3096	ctxt->eflags \|= X86_EFLAGS_CF;	3032	ctxt->eflags \|= X86_EFLAGS_CF;
@@ -3116,7 +3052,7 @@ static int em_aam(struct x86_emulate_ctxt *ctxt) · 19 unmodified lines
3116	ctxt->src.type = OP_IMM;	3052	ctxt->src.type = OP_IMM;
3117	ctxt->src.val = 0;	3053	ctxt->src.val = 0;
3118	ctxt->src.bytes = 1;	3054	ctxt->src.bytes = 1;
3119	fastop(ctxt, em_or);	3055	em_or(ctxt);
3120		3056
3121	return X86EMUL_CONTINUE;	3057	return X86EMUL_CONTINUE;
3122	}	3058	}
@@ -3134,7 +3070,7 @@ static int em_aad(struct x86_emulate_ctxt *ctxt) · 11 unmodified lines
3134	ctxt->src.type = OP_IMM;	3070	ctxt->src.type = OP_IMM;
3135	ctxt->src.val = 0;	3071	ctxt->src.val = 0;
3136	ctxt->src.bytes = 1;	3072	ctxt->src.bytes = 1;
3137	fastop(ctxt, em_or);	3073	em_or(ctxt);
3138		3074
3139	return X86EMUL_CONTINUE;	3075	return X86EMUL_CONTINUE;
3140	}	3076	}
@@ -3225,7 +3161,7 @@ static int em_xchg(struct x86_emulate_ctxt *ctxt) · 84 unmodified lines
3225	static int em_imul_3op(struct x86_emulate_ctxt *ctxt)	3161	static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3226	{	3162	{
3227	ctxt->dst.val = ctxt->src2.val;	3163	ctxt->dst.val = ctxt->src2.val;
3228	return fastop(ctxt, em_imul);	3164	return em_imul(ctxt);
3229	}	3165	}
3230		3166
3231	static int em_cwd(struct x86_emulate_ctxt *ctxt)	3167	static int em_cwd(struct x86_emulate_ctxt *ctxt)
@@ -4004,7 +3940,6 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) · 772 unmodified lines
4004	#define MD(_f, _m) { .flags = ((_f) \| ModeDual), .u.mdual = (_m) }	3940	#define MD(_f, _m) { .flags = ((_f) \| ModeDual), .u.mdual = (_m) }
4005	#define E(_f, _e) { .flags = ((_f) \| Escape \| ModRM), .u.esc = (_e) }	3941	#define E(_f, _e) { .flags = ((_f) \| Escape \| ModRM), .u.esc = (_e) }
4006	#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }	3942	#define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4007	#define F(_f, _e) { .flags = (_f) \| Fastop, .u.fastop = (_e) }
4008	#define II(_f, _e, _i) \	3943	#define II(_f, _e, _i) \
4009	{ .flags = (_f)\|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }	3944	{ .flags = (_f)\|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4010	#define IIP(_f, _e, _i, _p) \	3945	#define IIP(_f, _e, _i, _p) \
@@ -4019,9 +3954,9 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt) · 8 unmodified lines
4019	#define I2bvIP(_f, _e, _i, _p) \	3954	#define I2bvIP(_f, _e, _i, _p) \
4020	IIP((_f) \| ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)	3955	IIP((_f) \| ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4021		3956
4022	#define F6ALU(_f, _e) F2bv((_f) \| DstMem \| SrcReg \| ModRM, _e), \	3957	#define I6ALU(_f, _e) I2bv((_f) \| DstMem \| SrcReg \| ModRM, _e), \
4023	F2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \	3958	I2bv(((_f) \| DstReg \| SrcMem \| ModRM) & ~Lock, _e), \
4024	F2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)	3959	I2bv(((_f) & ~Lock) \| DstAcc \| SrcImm, _e)
4025		3960
4026	static const struct opcode group7_rm0[] = {	3961	static const struct opcode group7_rm0[] = {
4027	N,	3962	N,
@@ -4059,14 +3994,14 @@ static const struct opcode group7_rm7[] = { · 31 unmodified lines
4059	};	3994	};
4060		3995
4061	static const struct opcode group1[] = {	3996	static const struct opcode group1[] = {
4062	F(Lock, em_add),	3997	I(Lock, em_add),
4063	F(Lock \| PageTable, em_or),	3998	I(Lock \| PageTable, em_or),
4064	F(Lock, em_adc),	3999	I(Lock, em_adc),
4065	F(Lock, em_sbb),	4000	I(Lock, em_sbb),
4066	F(Lock \| PageTable, em_and),	4001	I(Lock \| PageTable, em_and),
4067	F(Lock, em_sub),	4002	I(Lock, em_sub),
4068	F(Lock, em_xor),	4003	I(Lock, em_xor),
4069	F(NoWrite, em_cmp),	4004	I(NoWrite, em_cmp),
4070	};	4005	};
4071		4006
4072	static const struct opcode group1A[] = {	4007	static const struct opcode group1A[] = {
@@ -4074,36 +4009,36 @@ static const struct opcode group1A[] = { · 1 unmodified line
4074	};	4009	};
4075		4010
4076	static const struct opcode group2[] = {	4011	static const struct opcode group2[] = {
4077	F(DstMem \| ModRM, em_rol),	4012	I(DstMem \| ModRM, em_rol),
4078	F(DstMem \| ModRM, em_ror),	4013	I(DstMem \| ModRM, em_ror),
4079	F(DstMem \| ModRM, em_rcl),	4014	I(DstMem \| ModRM, em_rcl),
4080	F(DstMem \| ModRM, em_rcr),	4015	I(DstMem \| ModRM, em_rcr),
4081	F(DstMem \| ModRM, em_shl),	4016	I(DstMem \| ModRM, em_shl),
4082	F(DstMem \| ModRM, em_shr),	4017	I(DstMem \| ModRM, em_shr),
4083	F(DstMem \| ModRM, em_shl),	4018	I(DstMem \| ModRM, em_shl),
4084	F(DstMem \| ModRM, em_sar),	4019	I(DstMem \| ModRM, em_sar),
4085	};	4020	};
4086		4021
4087	static const struct opcode group3[] = {	4022	static const struct opcode group3[] = {
4088	F(DstMem \| SrcImm \| NoWrite, em_test),	4023	I(DstMem \| SrcImm \| NoWrite, em_test),
4089	F(DstMem \| SrcImm \| NoWrite, em_test),	4024	I(DstMem \| SrcImm \| NoWrite, em_test),
4090	F(DstMem \| SrcNone \| Lock, em_not),	4025	I(DstMem \| SrcNone \| Lock, em_not),
4091	F(DstMem \| SrcNone \| Lock, em_neg),	4026	I(DstMem \| SrcNone \| Lock, em_neg),
4092	F(DstXacc \| Src2Mem, em_mul_ex),	4027	I(DstXacc \| Src2Mem, em_mul_ex),
4093	F(DstXacc \| Src2Mem, em_imul_ex),	4028	I(DstXacc \| Src2Mem, em_imul_ex),
4094	F(DstXacc \| Src2Mem, em_div_ex),	4029	I(DstXacc \| Src2Mem, em_div_ex),
4095	F(DstXacc \| Src2Mem, em_idiv_ex),	4030	I(DstXacc \| Src2Mem, em_idiv_ex),
4096	};	4031	};
4097		4032
4098	static const struct opcode group4[] = {	4033	static const struct opcode group4[] = {
4099	F(ByteOp \| DstMem \| SrcNone \| Lock, em_inc),	4034	I(ByteOp \| DstMem \| SrcNone \| Lock, em_inc),
4100	F(ByteOp \| DstMem \| SrcNone \| Lock, em_dec),	4035	I(ByteOp \| DstMem \| SrcNone \| Lock, em_dec),
4101	N, N, N, N, N, N,	4036	N, N, N, N, N, N,
4102	};	4037	};
4103		4038
4104	static const struct opcode group5[] = {	4039	static const struct opcode group5[] = {
4105	F(DstMem \| SrcNone \| Lock, em_inc),	4040	I(DstMem \| SrcNone \| Lock, em_inc),
4106	F(DstMem \| SrcNone \| Lock, em_dec),	4041	I(DstMem \| SrcNone \| Lock, em_dec),
4107	I(SrcMem \| NearBranch \| IsBranch \| ShadowStack, em_call_near_abs),	4042	I(SrcMem \| NearBranch \| IsBranch \| ShadowStack, em_call_near_abs),
4108	I(SrcMemFAddr \| ImplicitOps \| IsBranch \| ShadowStack, em_call_far),	4043	I(SrcMemFAddr \| ImplicitOps \| IsBranch \| ShadowStack, em_call_far),
4109	I(SrcMem \| NearBranch \| IsBranch, em_jmp_abs),	4044	I(SrcMem \| NearBranch \| IsBranch, em_jmp_abs),
@@ -4139,10 +4074,10 @@ static const struct group_dual group7 = { { · 29 unmodified lines
4139		4074
4140	static const struct opcode group8[] = {	4075	static const struct opcode group8[] = {
4141	N, N, N, N,	4076	N, N, N, N,
4142	F(DstMem \| SrcImmByte \| NoWrite, em_bt),	4077	I(DstMem \| SrcImmByte \| NoWrite, em_bt),
4143	F(DstMem \| SrcImmByte \| Lock \| PageTable, em_bts),	4078	I(DstMem \| SrcImmByte \| Lock \| PageTable, em_bts),
4144	F(DstMem \| SrcImmByte \| Lock, em_btr),	4079	I(DstMem \| SrcImmByte \| Lock, em_btr),
4145	F(DstMem \| SrcImmByte \| Lock \| PageTable, em_btc),	4080	I(DstMem \| SrcImmByte \| Lock \| PageTable, em_btc),
4146	};	4081	};
4147		4082
4148	/*	4083	/*
@@ -4279,31 +4214,31 @@ static const struct instr_dual instr_dual_8d = { · 130 unmodified lines
4279		4214
4280	static const struct opcode opcode_table[256] = {	4215	static const struct opcode opcode_table[256] = {
4281	/* 0x00 - 0x07 */	4216	/* 0x00 - 0x07 */
4282	F6ALU(Lock, em_add),	4217	I6ALU(Lock, em_add),
4283	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_push_sreg),	4218	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_push_sreg),
4284	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_pop_sreg),	4219	I(ImplicitOps \| Stack \| No64 \| Src2ES, em_pop_sreg),
4285	/* 0x08 - 0x0F */	4220	/* 0x08 - 0x0F */
4286	F6ALU(Lock \| PageTable, em_or),	4221	I6ALU(Lock \| PageTable, em_or),
4287	I(ImplicitOps \| Stack \| No64 \| Src2CS, em_push_sreg),	4222	I(ImplicitOps \| Stack \| No64 \| Src2CS, em_push_sreg),
4288	N,	4223	N,
4289	/* 0x10 - 0x17 */	4224	/* 0x10 - 0x17 */
4290	F6ALU(Lock, em_adc),	4225	I6ALU(Lock, em_adc),
4291	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_push_sreg),	4226	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_push_sreg),
4292	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_pop_sreg),	4227	I(ImplicitOps \| Stack \| No64 \| Src2SS, em_pop_sreg),
4293	/* 0x18 - 0x1F */	4228	/* 0x18 - 0x1F */
4294	F6ALU(Lock, em_sbb),	4229	I6ALU(Lock, em_sbb),
4295	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_push_sreg),	4230	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_push_sreg),
4296	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_pop_sreg),	4231	I(ImplicitOps \| Stack \| No64 \| Src2DS, em_pop_sreg),
4297	/* 0x20 - 0x27 */	4232	/* 0x20 - 0x27 */
4298	F6ALU(Lock \| PageTable, em_and), N, N,	4233	I6ALU(Lock \| PageTable, em_and), N, N,
4299	/* 0x28 - 0x2F */	4234	/* 0x28 - 0x2F */
4300	F6ALU(Lock, em_sub), N, I(ByteOp \| DstAcc \| No64, em_das),	4235	I6ALU(Lock, em_sub), N, I(ByteOp \| DstAcc \| No64, em_das),
4301	/* 0x30 - 0x37 */	4236	/* 0x30 - 0x37 */
4302	F6ALU(Lock, em_xor), N, N,	4237	I6ALU(Lock, em_xor), N, N,
4303	/* 0x38 - 0x3F */	4238	/* 0x38 - 0x3F */
4304	F6ALU(NoWrite, em_cmp), N, N,	4239	I6ALU(NoWrite, em_cmp), N, N,
4305	/* 0x40 - 0x4F */	4240	/* 0x40 - 0x4F */
4306	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),	4241	X8(I(DstReg, em_inc)), X8(I(DstReg, em_dec)),
4307	/* 0x50 - 0x57 */	4242	/* 0x50 - 0x57 */
4308	X8(I(SrcReg \| Stack, em_push)),	4243	X8(I(SrcReg \| Stack, em_push)),
4309	/* 0x58 - 0x5F */	4244	/* 0x58 - 0x5F */
@@ -4327,7 +4262,7 @@ static const struct opcode opcode_table[256] = { · 17 unmodified lines
4327	G(DstMem \| SrcImm, group1),	4262	G(DstMem \| SrcImm, group1),
4328	G(ByteOp \| DstMem \| SrcImm \| No64, group1),	4263	G(ByteOp \| DstMem \| SrcImm \| No64, group1),
4329	G(DstMem \| SrcImmByte, group1),	4264	G(DstMem \| SrcImmByte, group1),
4330	F2bv(DstMem \| SrcReg \| ModRM \| NoWrite, em_test),	4265	I2bv(DstMem \| SrcReg \| ModRM \| NoWrite, em_test),
4331	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable, em_xchg),	4266	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable, em_xchg),
4332	/* 0x88 - 0x8F */	4267	/* 0x88 - 0x8F */
4333	I2bv(DstMem \| SrcReg \| ModRM \| Mov \| PageTable, em_mov),	4268	I2bv(DstMem \| SrcReg \| ModRM \| Mov \| PageTable, em_mov),
@@ -4348,12 +4283,12 @@ static const struct opcode opcode_table[256] = { · 14 unmodified lines
4348	I2bv(DstAcc \| SrcMem \| Mov \| MemAbs, em_mov),	4283	I2bv(DstAcc \| SrcMem \| Mov \| MemAbs, em_mov),
4349	I2bv(DstMem \| SrcAcc \| Mov \| MemAbs \| PageTable, em_mov),	4284	I2bv(DstMem \| SrcAcc \| Mov \| MemAbs \| PageTable, em_mov),
4350	I2bv(SrcSI \| DstDI \| Mov \| String \| TwoMemOp, em_mov),	4285	I2bv(SrcSI \| DstDI \| Mov \| String \| TwoMemOp, em_mov),
4351	F2bv(SrcSI \| DstDI \| String \| NoWrite \| TwoMemOp, em_cmp_r),	4286	I2bv(SrcSI \| DstDI \| String \| NoWrite \| TwoMemOp, em_cmp_r),
4352	/* 0xA8 - 0xAF */	4287	/* 0xA8 - 0xAF */
4353	F2bv(DstAcc \| SrcImm \| NoWrite, em_test),	4288	I2bv(DstAcc \| SrcImm \| NoWrite, em_test),
4354	I2bv(SrcAcc \| DstDI \| Mov \| String, em_mov),	4289	I2bv(SrcAcc \| DstDI \| Mov \| String, em_mov),
4355	I2bv(SrcSI \| DstAcc \| Mov \| String, em_mov),	4290	I2bv(SrcSI \| DstAcc \| Mov \| String, em_mov),
4356	F2bv(SrcAcc \| DstDI \| String \| NoWrite, em_cmp_r),	4291	I2bv(SrcAcc \| DstDI \| String \| NoWrite, em_cmp_r),
4357	/* 0xB0 - 0xB7 */	4292	/* 0xB0 - 0xB7 */
4358	X8(I(ByteOp \| DstReg \| SrcImm \| Mov, em_mov)),	4293	X8(I(ByteOp \| DstReg \| SrcImm \| Mov, em_mov)),
4359	/* 0xB8 - 0xBF */	4294	/* 0xB8 - 0xBF */
@@ -4378,7 +4313,7 @@ static const struct opcode opcode_table[256] = { · 18 unmodified lines
4378	G(Src2CL \| ByteOp, group2), G(Src2CL, group2),	4313	G(Src2CL \| ByteOp, group2), G(Src2CL, group2),
4379	I(DstAcc \| SrcImmUByte \| No64, em_aam),	4314	I(DstAcc \| SrcImmUByte \| No64, em_aam),
4380	I(DstAcc \| SrcImmUByte \| No64, em_aad),	4315	I(DstAcc \| SrcImmUByte \| No64, em_aad),
4381	F(DstAcc \| ByteOp \| No64, em_salc),	4316	I(DstAcc \| ByteOp \| No64, em_salc),
4382	I(DstAcc \| SrcXLat \| ByteOp, em_mov),	4317	I(DstAcc \| SrcXLat \| ByteOp, em_mov),
4383	/* 0xD8 - 0xDF */	4318	/* 0xD8 - 0xDF */
4384	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,	4319	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
@@ -4463,32 +4398,32 @@ static const struct opcode twobyte_table[256] = { · 78 unmodified lines
4463	/* 0xA0 - 0xA7 */	4398	/* 0xA0 - 0xA7 */
4464	I(Stack \| Src2FS, em_push_sreg), I(Stack \| Src2FS, em_pop_sreg),	4399	I(Stack \| Src2FS, em_push_sreg), I(Stack \| Src2FS, em_pop_sreg),
4465	II(ImplicitOps, em_cpuid, cpuid),	4400	II(ImplicitOps, em_cpuid, cpuid),
4466	F(DstMem \| SrcReg \| ModRM \| BitOp \| NoWrite, em_bt),	4401	I(DstMem \| SrcReg \| ModRM \| BitOp \| NoWrite, em_bt),
4467	F(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shld),	4402	I(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shld),
4468	F(DstMem \| SrcReg \| Src2CL \| ModRM, em_shld), N, N,	4403	I(DstMem \| SrcReg \| Src2CL \| ModRM, em_shld), N, N,
4469	/* 0xA8 - 0xAF */	4404	/* 0xA8 - 0xAF */
4470	I(Stack \| Src2GS, em_push_sreg), I(Stack \| Src2GS, em_pop_sreg),	4405	I(Stack \| Src2GS, em_push_sreg), I(Stack \| Src2GS, em_pop_sreg),
4471	II(EmulateOnUD \| ImplicitOps, em_rsm, rsm),	4406	II(EmulateOnUD \| ImplicitOps, em_rsm, rsm),
4472	F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_bts),	4407	I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_bts),
4473	F(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shrd),	4408	I(DstMem \| SrcReg \| Src2ImmByte \| ModRM, em_shrd),
4474	F(DstMem \| SrcReg \| Src2CL \| ModRM, em_shrd),	4409	I(DstMem \| SrcReg \| Src2CL \| ModRM, em_shrd),
4475	GD(0, &group15), F(DstReg \| SrcMem \| ModRM, em_imul),	4410	GD(0, &group15), I(DstReg \| SrcMem \| ModRM, em_imul),
4476	/* 0xB0 - 0xB7 */	4411	/* 0xB0 - 0xB7 */
4477	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable \| SrcWrite, em_cmpxchg),	4412	I2bv(DstMem \| SrcReg \| ModRM \| Lock \| PageTable \| SrcWrite, em_cmpxchg),
4478	I(DstReg \| SrcMemFAddr \| ModRM \| Src2SS, em_lseg),	4413	I(DstReg \| SrcMemFAddr \| ModRM \| Src2SS, em_lseg),
4479	F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock, em_btr),	4414	I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock, em_btr),
4480	I(DstReg \| SrcMemFAddr \| ModRM \| Src2FS, em_lseg),	4415	I(DstReg \| SrcMemFAddr \| ModRM \| Src2FS, em_lseg),
4481	I(DstReg \| SrcMemFAddr \| ModRM \| Src2GS, em_lseg),	4416	I(DstReg \| SrcMemFAddr \| ModRM \| Src2GS, em_lseg),
4482	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),	4417	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),
4483	/* 0xB8 - 0xBF */	4418	/* 0xB8 - 0xBF */
4484	N, N,	4419	N, N,
4485	G(BitOp, group8),	4420	G(BitOp, group8),
4486	F(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_btc),	4421	I(DstMem \| SrcReg \| ModRM \| BitOp \| Lock \| PageTable, em_btc),
4487	I(DstReg \| SrcMem \| ModRM, em_bsf_c),	4422	I(DstReg \| SrcMem \| ModRM, em_bsf_c),
4488	I(DstReg \| SrcMem \| ModRM, em_bsr_c),	4423	I(DstReg \| SrcMem \| ModRM, em_bsr_c),
4489	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),	4424	D(DstReg \| SrcMem8 \| ModRM \| Mov), D(DstReg \| SrcMem16 \| ModRM \| Mov),
4490	/* 0xC0 - 0xC7 */	4425	/* 0xC0 - 0xC7 */
4491	F2bv(DstMem \| SrcReg \| ModRM \| SrcWrite \| Lock, em_xadd),	4426	I2bv(DstMem \| SrcReg \| ModRM \| SrcWrite \| Lock, em_xadd),
4492	N, ID(0, &instr_dual_0f_c3),	4427	N, ID(0, &instr_dual_0f_c3),
4493	N, N, N, GD(0, &group9),	4428	N, N, N, GD(0, &group9),
4494	/* 0xC8 - 0xCF */	4429	/* 0xC8 - 0xCF */
@@ -5198,24 +5133,6 @@ static void fetch_possible_mmx_operand(struct operand *op) · 703 unmodified lines
5198	kvm_read_mmx_reg(op->addr.mm, &op->mm_val);	5133	kvm_read_mmx_reg(op->addr.mm, &op->mm_val);
5199	}	5134	}
5200		5135
5201	static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop)
5202	{
5203	ulong flags = (ctxt->eflags & EFLAGS_MASK) \| X86_EFLAGS_IF;
5204
5205	if (!(ctxt->d & ByteOp))
5206	fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5207
5208	asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5209	: "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5210	[thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5211	: "c"(ctxt->src2.val));
5212
5213	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) \| (flags & EFLAGS_MASK);
5214	if (!fop) /* exception is returned in fop variable */
5215	return emulate_de(ctxt);
5216	return X86EMUL_CONTINUE;
5217	}
5218
5219	void init_decode_cache(struct x86_emulate_ctxt *ctxt)	5136	void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5220	{	5137	{
5221	/* Clear fields that are set conditionally but read without a guard. */	5138	/* Clear fields that are set conditionally but read without a guard. */
@@ -5379,10 +5296,7 @@ special_insn: · 157 unmodified lines
5379	ctxt->eflags &= ~X86_EFLAGS_RF;	5296	ctxt->eflags &= ~X86_EFLAGS_RF;
5380		5297
5381	if (ctxt->execute) {	5298	if (ctxt->execute) {
5382	if (ctxt->d & Fastop)	5299	rc = ctxt->execute(ctxt);
5383	rc = fastop(ctxt, ctxt->fop);
5384	else
5385	rc = ctxt->execute(ctxt);
5386	if (rc != X86EMUL_CONTINUE)	5300	if (rc != X86EMUL_CONTINUE)
5387	goto done;	5301	goto done;
5388	goto writeback;	5302	goto writeback;

arch/x86/kvm/vmx/vmenter.S+4−0

@@ -361,6 +361,10 @@ SYM_FUNC_END(vmread_error_trampoline) · 360 unmodified lines
361		361
362	.section .text, "ax"	362	.section .text, "ax"
363		363
		364	#ifndef CONFIG_X86_FRED
		365
364	SYM_FUNC_START(vmx_do_interrupt_irqoff)	366	SYM_FUNC_START(vmx_do_interrupt_irqoff)
365	VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1	367	VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
366	SYM_FUNC_END(vmx_do_interrupt_irqoff)	368	SYM_FUNC_END(vmx_do_interrupt_irqoff)
		369
		370	#endif

arch/x86/kvm/vmx/vmx.c+7−1

 	    "unexpected VM-Exit interrupt info: 0x%x", intr_info))
 		return;
+	/*
+	 * Invoke the kernel's IRQ handler for the vector.  Use the FRED path
+	 * when it's available even if FRED isn't fully enabled, e.g. even if
+	 * FRED isn't supported in hardware, in order to avoid the indirect
+	 * CALL in the non-FRED path.
+	 */
 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
-	if (cpu_feature_enabled(X86_FEATURE_FRED))
+	if (IS_ENABLED(CONFIG_X86_FRED))
 		fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
 	else
 		vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));

arch/x86/lib/bhi.S+29−29

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	#include <asm/nospec-branch.h>	5	#include <asm/nospec-branch.h>
6		6
7	/*	7	/*
8	* Notably, the FineIBT preamble calling these will have ZF set and r10 zero.	8	* Notably, the FineIBT preamble calling these will have ZF set and eax zero.
9	*	9	*
10	* The very last element is in fact larger than 32 bytes, but since its the	10	* The very last element is in fact larger than 32 bytes, but since its the
11	* last element, this does not matter,	11	* last element, this does not matter,
@@ -36,7 +36,7 @@ SYM_INNER_LABEL(__bhi_args_1, SYM_L_LOCAL) · 24 unmodified lines
36	ANNOTATE_NOENDBR	36	ANNOTATE_NOENDBR
37	UNWIND_HINT_FUNC	37	UNWIND_HINT_FUNC
38	jne .Lud_1	38	jne .Lud_1
39	cmovne %r10, %rdi	39	cmovne %rax, %rdi
40	ANNOTATE_UNRET_SAFE	40	ANNOTATE_UNRET_SAFE
41	ret	41	ret
42	int3	42	int3
@@ -53,8 +53,8 @@ SYM_INNER_LABEL(__bhi_args_2, SYM_L_LOCAL) · 10 unmodified lines
53	ANNOTATE_NOENDBR	53	ANNOTATE_NOENDBR
54	UNWIND_HINT_FUNC	54	UNWIND_HINT_FUNC
55	jne .Lud_1	55	jne .Lud_1
56	cmovne %r10, %rdi	56	cmovne %rax, %rdi
57	cmovne %r10, %rsi	57	cmovne %rax, %rsi
58	ANNOTATE_UNRET_SAFE	58	ANNOTATE_UNRET_SAFE
59	ret	59	ret
60	int3	60	int3
@@ -64,9 +64,9 @@ SYM_INNER_LABEL(__bhi_args_3, SYM_L_LOCAL) · 3 unmodified lines
64	ANNOTATE_NOENDBR	64	ANNOTATE_NOENDBR
65	UNWIND_HINT_FUNC	65	UNWIND_HINT_FUNC
66	jne .Lud_1	66	jne .Lud_1
67	cmovne %r10, %rdi	67	cmovne %rax, %rdi
68	cmovne %r10, %rsi	68	cmovne %rax, %rsi
69	cmovne %r10, %rdx	69	cmovne %rax, %rdx
70	ANNOTATE_UNRET_SAFE	70	ANNOTATE_UNRET_SAFE
71	ret	71	ret
72	int3	72	int3
@@ -76,10 +76,10 @@ SYM_INNER_LABEL(__bhi_args_4, SYM_L_LOCAL) · 3 unmodified lines
76	ANNOTATE_NOENDBR	76	ANNOTATE_NOENDBR
77	UNWIND_HINT_FUNC	77	UNWIND_HINT_FUNC
78	jne .Lud_2	78	jne .Lud_2
79	cmovne %r10, %rdi	79	cmovne %rax, %rdi
80	cmovne %r10, %rsi	80	cmovne %rax, %rsi
81	cmovne %r10, %rdx	81	cmovne %rax, %rdx
82	cmovne %r10, %rcx	82	cmovne %rax, %rcx
83	ANNOTATE_UNRET_SAFE	83	ANNOTATE_UNRET_SAFE
84	ret	84	ret
85	int3	85	int3
@@ -89,11 +89,11 @@ SYM_INNER_LABEL(__bhi_args_5, SYM_L_LOCAL) · 3 unmodified lines
89	ANNOTATE_NOENDBR	89	ANNOTATE_NOENDBR
90	UNWIND_HINT_FUNC	90	UNWIND_HINT_FUNC
91	jne .Lud_2	91	jne .Lud_2
92	cmovne %r10, %rdi	92	cmovne %rax, %rdi
93	cmovne %r10, %rsi	93	cmovne %rax, %rsi
94	cmovne %r10, %rdx	94	cmovne %rax, %rdx
95	cmovne %r10, %rcx	95	cmovne %rax, %rcx
96	cmovne %r10, %r8	96	cmovne %rax, %r8
97	ANNOTATE_UNRET_SAFE	97	ANNOTATE_UNRET_SAFE
98	ret	98	ret
99	int3	99	int3
@@ -110,12 +110,12 @@ SYM_INNER_LABEL(__bhi_args_6, SYM_L_LOCAL) · 10 unmodified lines
110	ANNOTATE_NOENDBR	110	ANNOTATE_NOENDBR
111	UNWIND_HINT_FUNC	111	UNWIND_HINT_FUNC
112	jne .Lud_2	112	jne .Lud_2
113	cmovne %r10, %rdi	113	cmovne %rax, %rdi
114	cmovne %r10, %rsi	114	cmovne %rax, %rsi
115	cmovne %r10, %rdx	115	cmovne %rax, %rdx
116	cmovne %r10, %rcx	116	cmovne %rax, %rcx
117	cmovne %r10, %r8	117	cmovne %rax, %r8
118	cmovne %r10, %r9	118	cmovne %rax, %r9
119	ANNOTATE_UNRET_SAFE	119	ANNOTATE_UNRET_SAFE
120	ret	120	ret
121	int3	121	int3
@@ -125,13 +125,13 @@ SYM_INNER_LABEL(__bhi_args_7, SYM_L_LOCAL) · 3 unmodified lines
125	ANNOTATE_NOENDBR	125	ANNOTATE_NOENDBR
126	UNWIND_HINT_FUNC	126	UNWIND_HINT_FUNC
127	jne .Lud_2	127	jne .Lud_2
128	cmovne %r10, %rdi	128	cmovne %rax, %rdi
129	cmovne %r10, %rsi	129	cmovne %rax, %rsi
130	cmovne %r10, %rdx	130	cmovne %rax, %rdx
131	cmovne %r10, %rcx	131	cmovne %rax, %rcx
132	cmovne %r10, %r8	132	cmovne %rax, %r8
133	cmovne %r10, %r9	133	cmovne %rax, %r9
134	cmovne %r10, %rsp	134	cmovne %rax, %rsp
135	ANNOTATE_UNRET_SAFE	135	ANNOTATE_UNRET_SAFE
136	ret	136	ret
137	int3	137	int3

arch/x86/lib/retpoline.S+2−2

@@ -134,10 +134,10 @@ SYM_CODE_END(__x86_indirect_jump_thunk_array) · 133 unmodified lines
134	.macro ITS_THUNK reg	134	.macro ITS_THUNK reg
135		135
136	/*	136	/*
137	* If CFI paranoid is used then the ITS thunk starts with opcodes (0xea; jne 1b)	137	* If CFI paranoid is used then the ITS thunk starts with opcodes (1: udb; jne 1b)
138	* that complete the fineibt_paranoid caller sequence.	138	* that complete the fineibt_paranoid caller sequence.
139	*/	139	*/
140	1: .byte 0xea	140	1: ASM_UDB
141	SYM_INNER_LABEL(__x86_indirect_paranoid_thunk_\reg, SYM_L_GLOBAL)	141	SYM_INNER_LABEL(__x86_indirect_paranoid_thunk_\reg, SYM_L_GLOBAL)
142	UNWIND_HINT_UNDEFINED	142	UNWIND_HINT_UNDEFINED
143	ANNOTATE_NOENDBR	143	ANNOTATE_NOENDBR

arch/x86/net/bpf_jit_comp.c+3−3

 	u8 *prog = *pprog;
 	EMIT_ENDBR();
-	EMIT3_off32(0x41, 0x81, 0xea, hash);		/* subl $hash, %r10d	*/
+	EMIT1_off32(0x2d, hash);			/* subl $hash, %eax	*/
 	if (cfi_bhi) {
+		EMIT2(0x2e, 0x2e);			/* cs cs */
 		emit_call(&prog, __bhi_args[arity], ip + 11);
 	} else {
-		EMIT2(0x75, 0xf9);			/* jne.d8 .-7		*/
+		EMIT3_off32(0x2e, 0x0f, 0x85, 3);	/* jne.d32,pn 3		*/
-		EMIT3(0x0f, 0x1f, 0x00);		/* nop3			*/
 	}
 	EMIT_ENDBR_POISON();

arch/x86/platform/efi/efi_stub_64.S+4−0

 #include <asm/nospec-branch.h>
 SYM_FUNC_START(__efi_call)
+	/*
+	 * The EFI code doesn't have any CFI, annotate away the CFI violation.
+	 */
+	ANNOTATE_NOCFI_SYM
 	pushq %rbp
 	movq %rsp, %rbp
 	and $~0xf, %rsp

drivers/misc/lkdtm/perms.c+5−0

 #include <linux/vmalloc.h>
 #include <linux/mman.h>
 #include <linux/uaccess.h>
+#include <linux/objtool.h>
 #include <asm/cacheflush.h>
 #include <asm/sections.h>
 	func();
 	pr_err("FAIL: func returned\n");
 }
+/*
+ * Explicitly doing the wrong thing for testing.
+ */
+ANNOTATE_NOCFI_SYM(execute_location);
 static void execute_user_location(void *dst)
 {

include/linux/compiler-clang.h+0−5

 # define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
 #endif
-#if __has_feature(kcfi)
-/* Disable CFI checking inside a function. */
-#define __nocfi		__attribute__((__no_sanitize__("kcfi")))
-#endif
 /*
  * Turn individual warnings and errors on and off locally, depending
  * on version.

include/linux/compiler-gcc.h+0−4

 	(typeof(ptr)) (__ptr + (off));					\
 })
-#ifdef CONFIG_MITIGATION_RETPOLINE
-#define __noretpoline __attribute__((__indirect_branch__("keep")))
-#endif
 #if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
 #define __latent_entropy __attribute__((latent_entropy))
 #endif

include/linux/compiler_types.h+3−1

@@ -455,7 +455,9 @@ struct ftrace_likely_data { · 454 unmodified lines
455	# define __noscs	455	# define __noscs
456	#endif	456	#endif
457		457
458	#ifndef __nocfi	458	#if defined(CONFIG_CFI)
		459	# define __nocfi __attribute__((__no_sanitize__("kcfi")))
		460	#else
459	# define __nocfi	461	# define __nocfi
460	#endif	462	#endif
461		463

include/linux/init.h+0−8

 #include <linux/stringify.h>
 #include <linux/types.h>
-/* Built-in __init functions needn't be compiled with retpoline */
-#if defined(__noretpoline) && !defined(MODULE)
-#define __noinitretpoline __noretpoline
-#else
-#define __noinitretpoline
-#endif
 /* These macros are used to mark some functions or
  * initialized data (doesn't apply to uninitialized data)
  * as `initialization' functions. The kernel can take this
 /* These are for everybody (although not all archs will actually
    discard it in modules) */
 #define __init		__section(".init.text") __cold __latent_entropy	\
-						__noinitretpoline	\
 						__no_kstack_erase
 #define __initdata	__section(".init.data")
 #define __initconst	__section(".init.rodata")

include/linux/objtool.h+10−0

  * WARN using UD2.
  */
 #define ANNOTATE_REACHABLE(label)	__ASM_ANNOTATE(label, ANNOTYPE_REACHABLE)
+/*
+ * This should not be used; it annotates away CFI violations. There are a few
+ * valid use cases like kexec handover to the next kernel image, and there is
+ * no security concern there.
+ *
+ * There are also a few real issues annotated away, like EFI because we can't
+ * control the EFI code.
+ */
+#define ANNOTATE_NOCFI_SYM(sym)		asm(__ASM_ANNOTATE(sym, ANNOTYPE_NOCFI))
 #else
 #define ANNOTATE_NOENDBR		ANNOTATE type=ANNOTYPE_NOENDBR
 #define ANNOTATE_INTRA_FUNCTION_CALL	ANNOTATE type=ANNOTYPE_INTRA_FUNCTION_CALL
 #define ANNOTATE_UNRET_BEGIN		ANNOTATE type=ANNOTYPE_UNRET_BEGIN
 #define ANNOTATE_REACHABLE		ANNOTATE type=ANNOTYPE_REACHABLE
+#define ANNOTATE_NOCFI_SYM		ANNOTATE type=ANNOTYPE_NOCFI
 #endif
 #if defined(CONFIG_NOINSTR_VALIDATION) && \

include/linux/objtool_types.h+1−0

@@ -65,5 +65,6 @@ struct unwind_hint { · 64 unmodified lines
65	#define ANNOTYPE_IGNORE_ALTS 6	65	#define ANNOTYPE_IGNORE_ALTS 6
66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7	66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7
67	#define ANNOTYPE_REACHABLE 8	67	#define ANNOTYPE_REACHABLE 8
		68	#define ANNOTYPE_NOCFI 9
68		69
69	#endif /* _LINUX_OBJTOOL_TYPES_H */	70	#endif /* _LINUX_OBJTOOL_TYPES_H */

tools/include/linux/objtool_types.h+1−0

@@ -65,5 +65,6 @@ struct unwind_hint { · 64 unmodified lines
65	#define ANNOTYPE_IGNORE_ALTS 6	65	#define ANNOTYPE_IGNORE_ALTS 6
66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7	66	#define ANNOTYPE_INTRA_FUNCTION_CALL 7
67	#define ANNOTYPE_REACHABLE 8	67	#define ANNOTYPE_REACHABLE 8
		68	#define ANNOTYPE_NOCFI 9
68		69
69	#endif /* _LINUX_OBJTOOL_TYPES_H */	70	#endif /* _LINUX_OBJTOOL_TYPES_H */

tools/objtool/check.c+42−0

@@ -2392,6 +2392,8 @@ static int __annotate_ifc(struct objtool_file *file, int type, struct instructio · 2391 unmodified lines
2392		2392
2393	static int __annotate_late(struct objtool_file file, int type, struct instruction insn)	2393	static int __annotate_late(struct objtool_file file, int type, struct instruction insn)
2394	{	2394	{
		2395	struct symbol *sym;
		2396
2395	switch (type) {	2397	switch (type) {
2396	case ANNOTYPE_NOENDBR:	2398	case ANNOTYPE_NOENDBR:
2397	/* early */	2399	/* early */
@@ -2433,6 +2435,15 @@ static int __annotate_late(struct objtool_file *file, int type, struct instructi · 35 unmodified lines
2433	insn->dead_end = false;	2435	insn->dead_end = false;
2434	break;	2436	break;
2435		2437
		2438	case ANNOTYPE_NOCFI:
		2439	sym = insn->sym;
		2440	if (!sym) {
		2441	ERROR_INSN(insn, "dodgy NOCFI annotation");
		2442	return -1;
		2443	}
		2444	insn->sym->nocfi = 1;
		2445	break;
		2446
2436	default:	2447	default:
2437	ERROR_INSN(insn, "Unknown annotation type: %d", type);	2448	ERROR_INSN(insn, "Unknown annotation type: %d", type);
2438	return -1;	2449	return -1;
@@ -3994,6 +4005,37 @@ static int validate_retpoline(struct objtool_file *file) · 1555 unmodified lines
3994	warnings++;	4005	warnings++;
3995	}	4006	}
3996		4007
		4008	if (!opts.cfi)
		4009	return warnings;
		4010
		4011	/*
		4012	* kCFI call sites look like:
		4013	*
		4014	* movl $(-0x12345678), %r10d
		4015	* addl -4(%r11), %r10d
		4016	* jz 1f
		4017	* ud2
		4018	* 1: cs call __x86_indirect_thunk_r11
		4019	*
		4020	* Verify all indirect calls are kCFI adorned by checking for the
		4021	* UD2. Notably, doing __nocfi calls to regular (cfi) functions is
		4022	* broken.
		4023	*/
		4024	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
		4025	struct symbol *sym = insn->sym;
		4026
		4027	if (sym && (sym->type == STT_NOTYPE \|\|
		4028	sym->type == STT_FUNC) && !sym->nocfi) {
		4029	struct instruction *prev =
		4030	prev_insn_same_sym(file, insn);
		4031
		4032	if (!prev \|\| prev->type != INSN_BUG) {
		4033	WARN_INSN(insn, "no-cfi indirect call!");
		4034	warnings++;
		4035	}
		4036	}
		4037	}
		4038
3997	return warnings;	4039	return warnings;
3998	}	4040	}
3999		4041

tools/objtool/include/objtool/elf.h+1−0

@@ -70,6 +70,7 @@ struct symbol { · 69 unmodified lines
70	u8 local_label : 1;	70	u8 local_label : 1;
71	u8 frame_pointer : 1;	71	u8 frame_pointer : 1;
72	u8 ignore : 1;	72	u8 ignore : 1;
		73	u8 nocfi : 1;
73	struct list_head pv_target;	74	struct list_head pv_target;
74	struct reloc *relocs;	75	struct reloc *relocs;
75	struct section *group_sec;	76	struct section *group_sec;

Merge tag 'x86_core_for_v6.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code