lael-labs/linux

arch/powerpc/include/asm/livepatch.h+0−11

@@ -17,17 +17,6 @@ static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip) · 16 unmodified lines
17	ftrace_instruction_pointer_set(fregs, ip);	17	ftrace_instruction_pointer_set(fregs, ip);
18	}	18	}
19		19
20	#define klp_get_ftrace_location klp_get_ftrace_location
21	static inline unsigned long klp_get_ftrace_location(unsigned long faddr)
22	{
23	/*
24	* Live patch works on PPC32 and only with -mprofile-kernel on PPC64. In
25	* both cases, the ftrace location is always within the first 16 bytes.
26	*/
27	return ftrace_location_range(faddr, faddr + 16);
28	}
29	#endif /* CONFIG_LIVEPATCH */
30
31	#ifdef CONFIG_LIVEPATCH_64	20	#ifdef CONFIG_LIVEPATCH_64
32	static inline void klp_init_thread_info(struct task_struct *p)	21	static inline void klp_init_thread_info(struct task_struct *p)
33	{	22	{

arch/powerpc/kernel/kprobes.c+21−13

 	return addr;
 }
+static bool arch_kprobe_on_func_entry(unsigned long offset)
+{
+#ifdef PPC64_ELF_ABI_v2
+#ifdef CONFIG_KPROBES_ON_FTRACE
+	return offset <= 16;
+#else
+	return offset <= 8;
+#endif
+#else
+	return !offset;
+#endif
+}
+/* XXX try and fold the magic of kprobe_lookup_name() in this */
+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
+					 bool *on_func_entry)
+{
+	*on_func_entry = arch_kprobe_on_func_entry(offset);
+	return (kprobe_opcode_t *)(addr + offset);
+}
 void *alloc_insn_page(void)
 {
 	void *page;
 	kcb->kprobe_saved_msr = regs->msr;
 }
-bool arch_kprobe_on_func_entry(unsigned long offset)
-{
-#ifdef PPC64_ELF_ABI_v2
-#ifdef CONFIG_KPROBES_ON_FTRACE
-	return offset <= 16;
-#else
-	return offset <= 8;
-#endif
-#else
-	return !offset;
-#endif
-}
 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
 {
 	ri->ret_addr = (kprobe_opcode_t *)regs->link;

arch/um/kernel/um_arch.c+4−0

 	os_check_bugs();
 }
+void apply_ibt_endbr(s32 *start, s32 *end)
+{
+}
 void apply_retpolines(s32 *start, s32 *end)
 {
 }

arch/x86/Kconfig+37−6

@@ -1842,6 +1842,36 @@ config X86_UMIP · 1841 unmodified lines
1842	specific cases in protected and virtual-8086 modes. Emulated	1842	specific cases in protected and virtual-8086 modes. Emulated
1843	results are dummy.	1843	results are dummy.
1844		1844
		1845	config CC_HAS_IBT
		1846	# GCC >= 9 and binutils >= 2.29
		1847	# Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654
		1848	# Clang/LLVM >= 14
		1849	# https://github.com/llvm/llvm-project/commit/e0b89df2e0f0130881bf6c39bf31d7f6aac00e0f
		1850	# https://github.com/llvm/llvm-project/commit/dfcf69770bc522b9e411c66454934a37c1f35332
		1851	def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) \|\| \
		1852	(CC_IS_CLANG && CLANG_VERSION >= 140000)) && \
		1853	$(as-instr,endbr64)
		1854
		1855	config X86_KERNEL_IBT
		1856	prompt "Indirect Branch Tracking"
		1857	bool
		1858	depends on X86_64 && CC_HAS_IBT && STACK_VALIDATION
		1859	# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
		1860	depends on !LD_IS_LLD \|\| LLD_VERSION >= 140000
		1861	help
		1862	Build the kernel with support for Indirect Branch Tracking, a
		1863	hardware support course-grain forward-edge Control Flow Integrity
		1864	protection. It enforces that all indirect calls must land on
		1865	an ENDBR instruction, as such, the compiler will instrument the
		1866	code with them to make this happen.
		1867
		1868	In addition to building the kernel with IBT, seal all functions that
		1869	are not indirect call targets, avoiding them ever becomming one.
		1870
		1871	This requires LTO like objtool runs and will slow down the build. It
		1872	does significantly reduce the number of ENDBR instructions in the
		1873	kernel image.
		1874
1845	config X86_INTEL_MEMORY_PROTECTION_KEYS	1875	config X86_INTEL_MEMORY_PROTECTION_KEYS
1846	prompt "Memory Protection Keys"	1876	prompt "Memory Protection Keys"
1847	def_bool y	1877	def_bool y
@@ -2815,19 +2845,20 @@ config IA32_AOUT · 967 unmodified lines
2815	help	2845	help
2816	Support old a.out binaries in the 32bit emulation.	2846	Support old a.out binaries in the 32bit emulation.
2817		2847
2818	config X86_X32	2848	config X86_X32_ABI
2819	bool "x32 ABI for 64-bit mode"	2849	bool "x32 ABI for 64-bit mode"
2820	depends on X86_64	2850	depends on X86_64
		2851	# llvm-objcopy does not convert x86_64 .note.gnu.property or
		2852	# compressed debug sections to x86_x32 properly:
		2853	# https://github.com/ClangBuiltLinux/linux/issues/514
		2854	# https://github.com/ClangBuiltLinux/linux/issues/1141
		2855	depends on $(success,$(OBJCOPY) --version \| head -n1 \| grep -qv llvm)
2821	help	2856	help
2822	Include code to run binaries for the x32 native 32-bit ABI	2857	Include code to run binaries for the x32 native 32-bit ABI
2823	for 64-bit processors. An x32 process gets access to the	2858	for 64-bit processors. An x32 process gets access to the
2824	full 64-bit register file and wide data path while leaving	2859	full 64-bit register file and wide data path while leaving
2825	pointers at 32 bits for smaller memory footprint.	2860	pointers at 32 bits for smaller memory footprint.
2826		2861
2827	You will need a recent binutils (2.22 or later) with
2828	elf32_x86_64 support enabled to compile a kernel with this
2829	option set.
2830
2831	config COMPAT_32	2862	config COMPAT_32
2832	def_bool y	2863	def_bool y
2833	depends on IA32_EMULATION \|\| X86_32	2864	depends on IA32_EMULATION \|\| X86_32
@@ -2836,7 +2867,7 @@ config COMPAT_32 · 2 unmodified lines
2836		2867
2837	config COMPAT	2868	config COMPAT
2838	def_bool y	2869	def_bool y
2839	depends on IA32_EMULATION \|\| X86_X32	2870	depends on IA32_EMULATION \|\| X86_X32_ABI
2840		2871
2841	if COMPAT	2872	if COMPAT
2842	config COMPAT_FOR_U64_ALIGNMENT	2873	config COMPAT_FOR_U64_ALIGNMENT

arch/x86/Makefile+14−18

 # How to compile the 16-bit code.  Note we always compile for -march=i386;
 # that way we can complain to the user if the CPU is insufficient.
-REALMODE_CFLAGS	:= -m16 -g -Os -DDISABLE_BRANCH_PROFILING \
+REALMODE_CFLAGS	:= -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
 		   -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \
 		   -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
 		   -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
 #
 KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
-# Intel CET isn't enabled in the kernel
+ifeq ($(CONFIG_X86_KERNEL_IBT),y)
+#
+# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate
+# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK
+# for jump-tables, as such, disable jump-tables for now.
+#
+# (jump-tables are implicitly disabled by RETPOLINE)
+#
+#   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
+#
+KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables)
+else
 KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
+endif
 ifeq ($(CONFIG_X86_32),y)
         BITS := 32
         KBUILD_CFLAGS += -mcmodel=kernel
 endif
-ifdef CONFIG_X86_X32
-	x32_ld_ok := $(call try-run,\
-			/bin/echo -e '1: .quad 1b' | \
-			$(CC) $(KBUILD_AFLAGS) -c -x assembler -o "$$TMP" - && \
-			$(OBJCOPY) -O elf32-x86-64 "$$TMP" "$$TMP.o" && \
-			$(LD) -m elf32_x86_64 "$$TMP.o" -o "$$TMP",y,n)
-        ifeq ($(x32_ld_ok),y)
-                CONFIG_X86_X32_ABI := y
-                KBUILD_AFLAGS += -DCONFIG_X86_X32_ABI
-                KBUILD_CFLAGS += -DCONFIG_X86_X32_ABI
-        else
-                $(warning CONFIG_X86_X32 enabled but no binutils support)
-        endif
-endif
-export CONFIG_X86_X32_ABI
 #
 # If the function graph tracer is used with mcount instead of fentry,
 # '-maccumulate-outgoing-args' is needed to prevent a GCC bug

arch/x86/crypto/crc32c-pcl-intel-asm_64.S+3−0

 .altmacro
 LABEL crc_ %i
 .noaltmacro
+	ENDBR
 	crc32q   -i*8(block_0), crc_init
 	crc32q   -i*8(block_1), crc1
 	crc32q   -i*8(block_2), crc2
 .altmacro
 LABEL crc_ %i
 .noaltmacro
+	ENDBR
 	crc32q   -i*8(block_0), crc_init
 	crc32q   -i*8(block_1), crc1
 # SKIP  crc32  -i*8(block_2), crc2 ; Don't do this one yet
 	################################################################
 LABEL crc_ 0
+	ENDBR
 	mov     tmp, len
 	cmp     $128*24, tmp
 	jae     full_block

arch/x86/entry/entry_64.S+27−4

 SYM_CODE_START(entry_SYSCALL_64)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	swapgs
 	/* tss.sp2 is scratch space. */
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS				/* pt_regs->ss */
 .pushsection .text, "ax"
 SYM_CODE_START(ret_from_fork)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR // copy_thread
 	movq	%rax, %rdi
 	call	schedule_tail			/* rdi: 'prev' task parameter */
 .macro idtentry vector asmsym cfunc has_error_code:req
 SYM_CODE_START(\asmsym)
 	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
+	ENDBR
 	ASM_CLAC
 	.if \has_error_code == 0
 .macro idtentry_mce_db vector asmsym cfunc
 SYM_CODE_START(\asmsym)
 	UNWIND_HINT_IRET_REGS
+	ENDBR
 	ASM_CLAC
 	pushq	$-1			/* ORIG_RAX: no syscall to restart */
 .macro idtentry_vc vector asmsym cfunc
 SYM_CODE_START(\asmsym)
 	UNWIND_HINT_IRET_REGS
+	ENDBR
 	ASM_CLAC
 	/*
 .macro idtentry_df vector asmsym cfunc
 SYM_CODE_START(\asmsym)
 	UNWIND_HINT_IRET_REGS offset=8
+	ENDBR
 	ASM_CLAC
 	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
 	movq	$-1, ORIG_RAX(%rsp)	/* no syscall to restart */
 	call	\cfunc
+	/* For some configurations \cfunc ends up being a noreturn. */
+	REACHABLE
 	jmp	paranoid_exit
 _ASM_NOKPROBE(\asmsym)
 	.align 16
 	.globl __irqentry_text_end
 __irqentry_text_end:
+	ANNOTATE_NOENDBR
 SYM_CODE_START_LOCAL(common_interrupt_return)
 SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
 	/* Restore RDI. */
 	popq	%rdi
-	SWAPGS
+	swapgs
-	INTERRUPT_RETURN
+	jmp	.Lnative_iret
 SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
 	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
 	 * when returning from IPI handler.
 	 */
-	INTERRUPT_RETURN
+#ifdef CONFIG_XEN_PV
+SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
+	.byte 0xe9
+	.long .Lnative_iret - (. + 4)
+#endif
-SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
+.Lnative_iret:
 	UNWIND_HINT_IRET_REGS
 	/*
 	 * Are we returning to a stack segment from the LDT?  Note: in
 #endif
 SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR // exc_double_fault
 	/*
 	 * This may fault.  Non-paranoid faults on return to userspace are
 	 * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
 	FRAME_BEGIN
 	swapgs
 .Lgs_change:
+	ANNOTATE_NOENDBR // error_entry
 	movl	%edi, %gs
 2:	ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
 	swapgs
  */
 SYM_CODE_START(xen_failsafe_callback)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	movl	%ds, %ecx
 	cmpw	%cx, 0x10(%rsp)
 	jne	1f
  */
 SYM_CODE_START(asm_exc_nmi)
 	UNWIND_HINT_IRET_REGS
+	ENDBR
 	/*
 	 * We allow breakpoints in NMIs. If a breakpoint occurs, then
 #endif
 repeat_nmi:
+	ANNOTATE_NOENDBR // this code
 	/*
 	 * If there was a nested NMI, the first NMI's iret will return
 	 * here. But NMIs are still enabled and we can take another
 	.endr
 	subq	$(5*8), %rsp
 end_repeat_nmi:
+	ANNOTATE_NOENDBR // this code
 	/*
 	 * Everything below this point can be preempted by a nested NMI.
  */
 SYM_CODE_START(ignore_sysret)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	mov	$-ENOSYS, %eax
 	sysretl
 SYM_CODE_END(ignore_sysret)

arch/x86/entry/entry_64_compat.S+5−0

  */
 SYM_CODE_START(entry_SYSENTER_compat)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	/* Interrupts are off on entry. */
 	SWAPGS
 	popfq
 	jmp	.Lsysenter_flags_fixed
 SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR // is_sysenter_singlestep
 SYM_CODE_END(entry_SYSENTER_compat)
 /*
  */
 SYM_CODE_START(entry_SYSCALL_compat)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	/* Interrupts are off on entry. */
 	swapgs
 	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
+	ANNOTATE_NOENDBR
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER32_DS		/* pt_regs->ss */
  */
 SYM_CODE_START(entry_INT80_compat)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	/*
 	 * Interrupts are off on entry.
 	 */

arch/x86/entry/syscalls/Makefile+1−1

@@ -67,7 +67,7 @@ uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h · 66 unmodified lines
67	syshdr-y += syscalls_32.h	67	syshdr-y += syscalls_32.h
68	syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h	68	syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h
69	syshdr-$(CONFIG_X86_64) += syscalls_64.h	69	syshdr-$(CONFIG_X86_64) += syscalls_64.h
70	syshdr-$(CONFIG_X86_X32) += syscalls_x32.h	70	syshdr-$(CONFIG_X86_X32_ABI) += syscalls_x32.h
71	syshdr-$(CONFIG_XEN) += xen-hypercalls.h	71	syshdr-$(CONFIG_XEN) += xen-hypercalls.h
72		72
73	uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))	73	uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))

arch/x86/include/asm/alternative.h+1−0

@@ -76,6 +76,7 @@ extern int alternatives_patched; · 75 unmodified lines
76	extern void alternative_instructions(void);	76	extern void alternative_instructions(void);
77	extern void apply_alternatives(struct alt_instr start, struct alt_instr end);	77	extern void apply_alternatives(struct alt_instr start, struct alt_instr end);
78	extern void apply_retpolines(s32 start, s32 end);	78	extern void apply_retpolines(s32 start, s32 end);
		79	extern void apply_ibt_endbr(s32 start, s32 end);
79		80
80	struct module;	81	struct module;
81		82

arch/x86/include/asm/bug.h+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4		4
5	#include <linux/stringify.h>	5	#include <linux/stringify.h>
6	#include <linux/instrumentation.h>	6	#include <linux/instrumentation.h>
		7	#include <linux/objtool.h>
7		8
8	/*	9	/*
9	* Despite that some emulators terminate on UD2, we use it for WARN().	10	* Despite that some emulators terminate on UD2, we use it for WARN().

arch/x86/include/asm/cpu.h+4−0

 #include <linux/topology.h>
 #include <linux/nodemask.h>
 #include <linux/percpu.h>
+#include <asm/ibt.h>
 #ifdef CONFIG_SMP
 #else
 static inline void init_ia32_feat_ctl(struct cpuinfo_x86 *c) {}
 #endif
+extern __noendbr void cet_disable(void);
 #endif /* _ASM_X86_CPU_H */

arch/x86/include/asm/cpufeatures.h+1−0

@@ -388,6 +388,7 @@ · 387 unmodified lines
388	#define X86_FEATURE_TSXLDTRK (1832+16) / TSX Suspend Load Address Tracking */	388	#define X86_FEATURE_TSXLDTRK (1832+16) / TSX Suspend Load Address Tracking */
389	#define X86_FEATURE_PCONFIG (1832+18) / Intel PCONFIG */	389	#define X86_FEATURE_PCONFIG (1832+18) / Intel PCONFIG */
390	#define X86_FEATURE_ARCH_LBR (1832+19) / Intel ARCH LBR */	390	#define X86_FEATURE_ARCH_LBR (1832+19) / Intel ARCH LBR */
		391	#define X86_FEATURE_IBT (1832+20) / Indirect Branch Tracking */
391	#define X86_FEATURE_AMX_BF16 (1832+22) / AMX bf16 Support */	392	#define X86_FEATURE_AMX_BF16 (1832+22) / AMX bf16 Support */
392	#define X86_FEATURE_AVX512_FP16 (1832+23) / AVX512 FP16 */	393	#define X86_FEATURE_AVX512_FP16 (1832+23) / AVX512 FP16 */
393	#define X86_FEATURE_AMX_TILE (1832+24) / AMX tile Support */	394	#define X86_FEATURE_AMX_TILE (1832+24) / AMX tile Support */

arch/x86/include/asm/efi.h+7−2

 #include <asm/tlb.h>
 #include <asm/nospec-branch.h>
 #include <asm/mmu_context.h>
+#include <asm/ibt.h>
 #include <linux/build_bug.h>
 #include <linux/kernel.h>
 #include <linux/pgtable.h>
 	efi_enter_mm();							\
 })
-#define arch_efi_call_virt(p, f, args...)				\
+#define arch_efi_call_virt(p, f, args...) ({				\
-	efi_call((void *)p->f, args)					\
+	u64 ret, ibt = ibt_save();					\
+	ret = efi_call((void *)p->f, args);				\
+	ibt_restore(ibt);						\
+	ret;								\
+})
 #define arch_efi_call_virt_teardown()					\
 ({									\

arch/x86/include/asm/ibt.h+105−0

@@ -0,0 +1,105 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _ASM_X86_IBT_H
	3	#define _ASM_X86_IBT_H
	4
	5	#include <linux/types.h>
	6
	7	/*
	8	* The rules for enabling IBT are:
	9	*
	10	* - CC_HAS_IBT: the toolchain supports it
	11	* - X86_KERNEL_IBT: it is selected in Kconfig
	12	* - !__DISABLE_EXPORTS: this is regular kernel code
	13	*
	14	* Esp. that latter one is a bit non-obvious, but some code like compressed,
	15	* purgatory, realmode etc.. is built with custom CFLAGS that do not include
	16	* -fcf-protection=branch and things will go bang.
	17	*
	18	* When all the above are satisfied, HAS_KERNEL_IBT will be 1, otherwise 0.
	19	*/
	20	#if defined(CONFIG_X86_KERNEL_IBT) && !defined(__DISABLE_EXPORTS)
	21
	22	#define HAS_KERNEL_IBT 1
	23
	24	#ifndef __ASSEMBLY__
	25
	26	#ifdef CONFIG_X86_64
	27	#define ASM_ENDBR "endbr64\n\t"
	28	#else
	29	#define ASM_ENDBR "endbr32\n\t"
	30	#endif
	31
	32	#define __noendbr __attribute__((nocf_check))
	33
	34	static inline __attribute_const__ u32 gen_endbr(void)
	35	{
	36	u32 endbr;
	37
	38	/*
	39	* Generate ENDBR64 in a way that is sure to not result in
	40	* an ENDBR64 instruction as immediate.
	41	*/
	42	asm ( "mov $~0xfa1e0ff3, %[endbr]\n\t"
	43	"not %[endbr]\n\t"
	44	: [endbr] "=&r" (endbr) );
	45
	46	return endbr;
	47	}
	48
	49	static inline __attribute_const__ u32 gen_endbr_poison(void)
	50	{
	51	/*
	52	* 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
	53	* will be unique to (former) ENDBR sites.
	54	*/
	55	return 0x001f0f66; /* osp nopl (%rax) */
	56	}
	57
	58	static inline bool is_endbr(u32 val)
	59	{
	60	if (val == gen_endbr_poison())
	61	return true;
	62
	63	val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
	64	return val == gen_endbr();
	65	}
	66
	67	extern __noendbr u64 ibt_save(void);
	68	extern __noendbr void ibt_restore(u64 save);
	69
	70	#else /* __ASSEMBLY__ */
	71
	72	#ifdef CONFIG_X86_64
	73	#define ENDBR endbr64
	74	#else
	75	#define ENDBR endbr32
	76	#endif
	77
	78	#endif /* __ASSEMBLY__ */
	79
	80	#else /* !IBT */
	81
	82	#define HAS_KERNEL_IBT 0
	83
	84	#ifndef __ASSEMBLY__
	85
	86	#define ASM_ENDBR
	87
	88	#define __noendbr
	89
	90	static inline bool is_endbr(u32 val) { return false; }
	91
	92	static inline u64 ibt_save(void) { return 0; }
	93	static inline void ibt_restore(u64 save) { }
	94
	95	#else /* __ASSEMBLY__ */
	96
	97	#define ENDBR
	98
	99	#endif /* __ASSEMBLY__ */
	100
	101	#endif /* CONFIG_X86_KERNEL_IBT */
	102
	103	#define ENDBR_INSN_SIZE (4*HAS_KERNEL_IBT)
	104
	105	#endif /* _ASM_X86_IBT_H */

arch/x86/include/asm/idtentry.h+16−9

 /* Interrupts/Exceptions */
 #include <asm/trapnr.h>
+#define IDT_ALIGN	(8 * (1 + HAS_KERNEL_IBT))
 #ifndef __ASSEMBLY__
 #include <linux/entry-common.h>
 #include <linux/hardirq.h>
 /*
  * ASM code to emit the common vector entry stubs where each stub is
- * packed into 8 bytes.
+ * packed into IDT_ALIGN bytes.
  *
  * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
  * GCC treats the local vector variable as unsigned int and would expand
  * point is to mask off the bits above bit 7 because the push is sign
  * extending.
  */
-	.align 8
+	.align IDT_ALIGN
 SYM_CODE_START(irq_entries_start)
     vector=FIRST_EXTERNAL_VECTOR
     .rept NR_EXTERNAL_VECTORS
 	UNWIND_HINT_IRET_REGS
 0 :
+	ENDBR
 	.byte	0x6a, vector
 	jmp	asm_common_interrupt
-	nop
+	/* Ensure that the above is IDT_ALIGN bytes max */
-	/* Ensure that the above is 8 bytes max */
+	.fill 0b + IDT_ALIGN - ., 1, 0xcc
-	. = 0b + 8
 	vector = vector+1
     .endr
 SYM_CODE_END(irq_entries_start)
 #ifdef CONFIG_X86_LOCAL_APIC
-	.align 8
+	.align IDT_ALIGN
 SYM_CODE_START(spurious_entries_start)
     vector=FIRST_SYSTEM_VECTOR
     .rept NR_SYSTEM_VECTORS
 	UNWIND_HINT_IRET_REGS
 0 :
+	ENDBR
 	.byte	0x6a, vector
 	jmp	asm_spurious_interrupt
-	nop
+	/* Ensure that the above is IDT_ALIGN bytes max */
-	/* Ensure that the above is 8 bytes max */
+	.fill 0b + IDT_ALIGN - ., 1, 0xcc
-	. = 0b + 8
 	vector = vector+1
     .endr
 SYM_CODE_END(spurious_entries_start)
 DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF,	xenpv_exc_double_fault);
 #endif
+/* #CP */
+#ifdef CONFIG_X86_KERNEL_IBT
+DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP,	exc_control_protection);
+#endif
 /* #VC */
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 DECLARE_IDTENTRY_VC(X86_TRAP_VC,	exc_vmm_communication);

arch/x86/include/asm/irq_stack.h+3−1

 #define _ASM_X86_IRQ_STACK_H
 #include <linux/ptrace.h>
+#include <linux/objtool.h>
 #include <asm/processor.h>
 }
 #define ASM_CALL_ARG0							\
-	"call %P[__func]				\n"
+	"call %P[__func]				\n"		\
+	ASM_REACHABLE
 #define ASM_CALL_ARG1							\
 	"movq	%[arg1], %%rdi				\n"		\

arch/x86/include/asm/irqflags.h+0−5

 #ifdef CONFIG_X86_64
 #ifdef CONFIG_XEN_PV
 #define SWAPGS	ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV
-#define INTERRUPT_RETURN						\
-	ANNOTATE_RETPOLINE_SAFE;					\
-	ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);",		\
-		X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")
 #else
 #define SWAPGS	swapgs
-#define INTERRUPT_RETURN	jmp native_iret
 #endif
 #endif
 #endif /* !__ASSEMBLY__ */

arch/x86/include/asm/linkage.h+31−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#define _ASM_X86_LINKAGE_H	3	#define _ASM_X86_LINKAGE_H
4		4
5	#include <linux/stringify.h>	5	#include <linux/stringify.h>
		6	#include <asm/ibt.h>
6		7
7	#undef notrace	8	#undef notrace
8	#define notrace __attribute__((no_instrument_function))	9	#define notrace __attribute__((no_instrument_function))
@@ -34,5 +35,35 @@ · 25 unmodified lines
34		35
35	#endif /* __ASSEMBLY__ */	36	#endif /* __ASSEMBLY__ */
36		37
		38	/* SYM_FUNC_START -- use for global functions */
		39	#define SYM_FUNC_START(name) \
		40	SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
		41	ENDBR
		42
		43	/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */
		44	#define SYM_FUNC_START_NOALIGN(name) \
		45	SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \
		46	ENDBR
		47
		48	/* SYM_FUNC_START_LOCAL -- use for local functions */
		49	#define SYM_FUNC_START_LOCAL(name) \
		50	SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \
		51	ENDBR
		52
		53	/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */
		54	#define SYM_FUNC_START_LOCAL_NOALIGN(name) \
		55	SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \
		56	ENDBR
		57
		58	/* SYM_FUNC_START_WEAK -- use for weak functions */
		59	#define SYM_FUNC_START_WEAK(name) \
		60	SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \
		61	ENDBR
		62
		63	/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */
		64	#define SYM_FUNC_START_WEAK_NOALIGN(name) \
		65	SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \
		66	ENDBR
		67
37	#endif /* _ASM_X86_LINKAGE_H */	68	#endif /* _ASM_X86_LINKAGE_H */
38		69

arch/x86/include/asm/msr-index.h+19−1

@@ -362,11 +362,29 @@ · 361 unmodified lines
362	#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c	362	#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c
363	#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d	363	#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d
364		364
365
366	#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690	365	#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690
367	#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0	366	#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0
368	#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1	367	#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1
369		368
		369	/* Control-flow Enforcement Technology MSRs */
		370	#define MSR_IA32_U_CET 0x000006a0 /* user mode cet */
		371	#define MSR_IA32_S_CET 0x000006a2 /* kernel mode cet */
		372	#define CET_SHSTK_EN BIT_ULL(0)
		373	#define CET_WRSS_EN BIT_ULL(1)
		374	#define CET_ENDBR_EN BIT_ULL(2)
		375	#define CET_LEG_IW_EN BIT_ULL(3)
		376	#define CET_NO_TRACK_EN BIT_ULL(4)
		377	#define CET_SUPPRESS_DISABLE BIT_ULL(5)
		378	#define CET_RESERVED (BIT_ULL(6) \| BIT_ULL(7) \| BIT_ULL(8) \| BIT_ULL(9))
		379	#define CET_SUPPRESS BIT_ULL(10)
		380	#define CET_WAIT_ENDBR BIT_ULL(11)
		381
		382	#define MSR_IA32_PL0_SSP 0x000006a4 /* ring-0 shadow stack pointer */
		383	#define MSR_IA32_PL1_SSP 0x000006a5 /* ring-1 shadow stack pointer */
		384	#define MSR_IA32_PL2_SSP 0x000006a6 /* ring-2 shadow stack pointer */
		385	#define MSR_IA32_PL3_SSP 0x000006a7 /* ring-3 shadow stack pointer */
		386	#define MSR_IA32_INT_SSP_TAB 0x000006a8 /* exception shadow stack table */
		387
370	/* Hardware P state interface */	388	/* Hardware P state interface */
371	#define MSR_PPERF 0x0000064e	389	#define MSR_PPERF 0x0000064e
372	#define MSR_PERF_LIMIT_REASONS 0x0000064f	390	#define MSR_PERF_LIMIT_REASONS 0x0000064f

arch/x86/include/asm/paravirt.h+1−0

@@ -666,6 +666,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); · 665 unmodified lines
666	".globl " PV_THUNK_NAME(func) ";" \	666	".globl " PV_THUNK_NAME(func) ";" \
667	".type " PV_THUNK_NAME(func) ", @function;" \	667	".type " PV_THUNK_NAME(func) ", @function;" \
668	PV_THUNK_NAME(func) ":" \	668	PV_THUNK_NAME(func) ":" \
		669	ASM_ENDBR \
669	FRAME_BEGIN \	670	FRAME_BEGIN \
670	PV_SAVE_ALL_CALLER_REGS \	671	PV_SAVE_ALL_CALLER_REGS \
671	"call " #func ";" \	672	"call " #func ";" \

arch/x86/include/asm/paravirt_types.h+0−1

@@ -272,7 +272,6 @@ struct paravirt_patch_template { · 271 unmodified lines
272		272
273	extern struct pv_info pv_info;	273	extern struct pv_info pv_info;
274	extern struct paravirt_patch_template pv_ops;	274	extern struct paravirt_patch_template pv_ops;
275	extern void (*paravirt_iret)(void);
276		275
277	#define PARAVIRT_PATCH(x) \	276	#define PARAVIRT_PATCH(x) \
278	(offsetof(struct paravirt_patch_template, x) / sizeof(void *))	277	(offsetof(struct paravirt_patch_template, x) / sizeof(void *))

arch/x86/include/asm/processor.h+1−1

@@ -837,7 +837,7 @@ bool xen_set_default_idle(void); · 836 unmodified lines
837	#define xen_set_default_idle 0	837	#define xen_set_default_idle 0
838	#endif	838	#endif
839		839
840	void stop_this_cpu(void *dummy);	840	void __noreturn stop_this_cpu(void *dummy);
841	void microcode_check(void);	841	void microcode_check(void);
842		842
843	enum l1tf_mitigations {	843	enum l1tf_mitigations {

arch/x86/include/asm/qspinlock_paravirt.h+3−0

 #ifndef __ASM_QSPINLOCK_PARAVIRT_H
 #define __ASM_QSPINLOCK_PARAVIRT_H
+#include <asm/ibt.h>
 /*
  * For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
  * registers. For i386, however, only 1 32-bit register needs to be saved
 	".type " PV_UNLOCK ", @function;"
 	".align 4,0x90;"
 	PV_UNLOCK ": "
+	ASM_ENDBR
 	FRAME_BEGIN
 	"push  %rdx;"
 	"mov   $0x1,%eax;"

arch/x86/include/asm/segment.h+3−2

 #include <linux/const.h>
 #include <asm/alternative.h>
+#include <asm/ibt.h>
 /*
  * Constructor for a conventional segment GDT (or LDT) entry.
  * vector has no error code (two bytes), a 'push $vector_number' (two
  * bytes), and a jump to the common entry code (up to five bytes).
  */
-#define EARLY_IDT_HANDLER_SIZE 9
+#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)
 /*
  * xen_early_idt_handler_array is for Xen pv guests: for each entry in
  * pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to
  * max 8 bytes.
  */
-#define XEN_EARLY_IDT_HANDLER_SIZE 8
+#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)
 #ifndef __ASSEMBLY__

arch/x86/include/asm/setup.h+2−1

 #include <linux/linkage.h>
 #include <asm/page_types.h>
+#include <asm/ibt.h>
 #ifdef __i386__
  * executable.)
  */
 #define RESERVE_BRK(name,sz)						\
-	static void __section(".discard.text") __used notrace		\
+	static void __section(".discard.text") __noendbr __used notrace	\
 	__brk_reservation_fn_##name##__(void) {				\
 		asm volatile (						\
 			".pushsection .brk_reservation,\"aw\",@nobits;" \

arch/x86/include/asm/syscall_wrapper.h+3−3

arch/x86/include/asm/text-patching.h+23−7

@@ -97,24 +97,40 @@ union text_poke_insn { · 96 unmodified lines
97	};	97	};
98		98
99	static __always_inline	99	static __always_inline
100	void text_gen_insn(u8 opcode, const void addr, const void *dest)	100	void __text_gen_insn(void buf, u8 opcode, const void addr, const void *dest, int size)
101	{	101	{
102	static union text_poke_insn insn; /* per instance */	102	union text_poke_insn *insn = buf;
103	int size = text_opcode_size(opcode);	103
		104	BUG_ON(size < text_opcode_size(opcode));
		105
		106	/*
		107	* Hide the addresses to avoid the compiler folding in constants when
		108	* referencing code, these can mess up annotations like
		109	* ANNOTATE_NOENDBR.
		110	*/
		111	OPTIMIZER_HIDE_VAR(insn);
		112	OPTIMIZER_HIDE_VAR(addr);
		113	OPTIMIZER_HIDE_VAR(dest);
104		114
105	insn.opcode = opcode;	115	insn->opcode = opcode;
106		116
107	if (size > 1) {	117	if (size > 1) {
108	insn.disp = (long)dest - (long)(addr + size);	118	insn->disp = (long)dest - (long)(addr + size);
109	if (size == 2) {	119	if (size == 2) {
110	/*	120	/*
111	* Ensure that for JMP9 the displacement	121	* Ensure that for JMP8 the displacement
112	* actually fits the signed byte.	122	* actually fits the signed byte.
113	*/	123	*/
114	BUG_ON((insn.disp >> 31) != (insn.disp >> 7));	124	BUG_ON((insn->disp >> 31) != (insn->disp >> 7));
115	}	125	}
116	}	126	}
		127	}
117		128
		129	static __always_inline
		130	void text_gen_insn(u8 opcode, const void addr, const void *dest)
		131	{
		132	static union text_poke_insn insn; /* per instance */
		133	__text_gen_insn(&insn, opcode, addr, dest, text_opcode_size(opcode));
118	return &insn.text;	134	return &insn.text;
119	}	135	}
120		136

arch/x86/include/asm/traps.h+2−0

@@ -18,6 +18,8 @@ void __init trap_init(void); · 17 unmodified lines
18	asmlinkage __visible noinstr struct pt_regs vc_switch_off_ist(struct pt_regs eregs);	18	asmlinkage __visible noinstr struct pt_regs vc_switch_off_ist(struct pt_regs eregs);
19	#endif	19	#endif
20		20
		21	extern bool ibt_selftest(void);
		22
21	#ifdef CONFIG_X86_F00F_BUG	23	#ifdef CONFIG_X86_F00F_BUG
22	/* For handling the FOOF bug */	24	/* For handling the FOOF bug */
23	void handle_invalid_op(struct pt_regs *regs);	25	void handle_invalid_op(struct pt_regs *regs);

arch/x86/include/asm/vdso.h+1−1

@@ -37,7 +37,7 @@ struct vdso_image { · 36 unmodified lines
37	extern const struct vdso_image vdso_image_64;	37	extern const struct vdso_image vdso_image_64;
38	#endif	38	#endif
39		39
40	#ifdef CONFIG_X86_X32	40	#ifdef CONFIG_X86_X32_ABI
41	extern const struct vdso_image vdso_image_x32;	41	extern const struct vdso_image vdso_image_x32;
42	#endif	42	#endif
43		43

arch/x86/include/uapi/asm/processor-flags.h+2−0

@@ -130,6 +130,8 @@ · 129 unmodified lines
130	#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)	130	#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)
131	#define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */	131	#define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */
132	#define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT)	132	#define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT)
		133	#define X86_CR4_CET_BIT 23 /* enable Control-flow Enforcement Technology */
		134	#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)
133		135
134	/*	136	/*
135	* x86-64 Task Priority Register, CR8	137	* x86-64 Task Priority Register, CR8

arch/x86/kernel/alternative.c+52−14

 }
 extern s32 __retpoline_sites[], __retpoline_sites_end[];
+extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern s32 __smp_locks[], __smp_locks_end[];
 void text_poke_early(void *addr, const void *opcode, size_t len);
 #endif /* CONFIG_RETPOLINE && CONFIG_STACK_VALIDATION */
+#ifdef CONFIG_X86_KERNEL_IBT
+/*
+ * Generated by: objtool --ibt
+ */
+void __init_or_module noinline apply_ibt_endbr(s32 *start, s32 *end)
+{
+	s32 *s;
+	for (s = start; s < end; s++) {
+		u32 endbr, poison = gen_endbr_poison();
+		void *addr = (void *)s + *s;
+		if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
+			continue;
+		if (WARN_ON_ONCE(!is_endbr(endbr)))
+			continue;
+		DPRINTK("ENDBR at: %pS (%px)", addr, addr);
+		/*
+		 * When we have IBT, the lack of ENDBR will trigger #CP
+		 */
+		DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
+		DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
+		text_poke_early(addr, &poison, 4);
+	}
+}
+#else
+void __init_or_module noinline apply_ibt_endbr(s32 *start, s32 *end) { }
+#endif /* CONFIG_X86_KERNEL_IBT */
 #ifdef CONFIG_SMP
 static void alternatives_smp_lock(const s32 *start, const s32 *end,
 				  u8 *text, u8 *text_end)
 "	.pushsection	.init.text, \"ax\", @progbits\n"
 "	.type		int3_magic, @function\n"
 "int3_magic:\n"
+	ANNOTATE_NOENDBR
 "	movl	$1, (%" _ASM_ARG1 ")\n"
 	ASM_RET
 "	.size		int3_magic, .-int3_magic\n"
 "	.popsection\n"
 );
-extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
+extern void int3_selftest_ip(void); /* defined in asm below */
 static int __init
 int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
 {
+	unsigned long selftest = (unsigned long)&int3_selftest_ip;
 	struct die_args *args = data;
 	struct pt_regs *regs = args->regs;
+	OPTIMIZER_HIDE_VAR(selftest);
 	if (!regs || user_mode(regs))
 		return NOTIFY_DONE;
 	if (val != DIE_INT3)
 		return NOTIFY_DONE;
-	if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
+	if (regs->ip - INT3_INSN_SIZE != selftest)
 		return NOTIFY_DONE;
 	int3_emulate_call(regs, (unsigned long)&int3_magic);
 	return NOTIFY_STOP;
 }
-static void __init int3_selftest(void)
+/* Must be noinline to ensure uniqueness of int3_selftest_ip. */
+static noinline void __init int3_selftest(void)
 {
 	static __initdata struct notifier_block int3_exception_nb = {
 		.notifier_call	= int3_exception_notify,
 	/*
 	 * Basically: int3_magic(&val); but really complicated :-)
 	 *
-	 * Stick the address of the INT3 instruction into int3_selftest_ip,
+	 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb
-	 * then trigger the INT3, padded with NOPs to match a CALL instruction
+	 * notifier above will emulate CALL for us.
-	 * length.
 	 */
-	asm volatile ("1: int3; nop; nop; nop; nop\n\t"
+	asm volatile ("int3_selftest_ip:\n\t"
-		      ".pushsection .init.data,\"aw\"\n\t"
+		      ANNOTATE_NOENDBR
-		      ".align " __ASM_SEL(4, 8) "\n\t"
+		      "    int3; nop; nop; nop; nop\n\t"
-		      ".type int3_selftest_ip, @object\n\t"
-		      ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
-		      "int3_selftest_ip:\n\t"
-		      __ASM_SEL(.long, .quad) " 1b\n\t"
-		      ".popsection\n\t"
 		      : ASM_CALL_CONSTRAINT
 		      : __ASM_SEL_RAW(a, D) (&val)
 		      : "memory");
 	 */
 	apply_alternatives(__alt_instructions, __alt_instructions_end);
+	apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
 #ifdef CONFIG_SMP
 	/* Patch to UP if other cpus not imminent. */
 	if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {

arch/x86/kernel/apm_32.c+7−0

 #include <asm/paravirt.h>
 #include <asm/reboot.h>
 #include <asm/nospec-branch.h>
+#include <asm/ibt.h>
 #if defined(CONFIG_APM_DISPLAY_BLANK) && defined(CONFIG_VT)
 extern int (*console_blank_hook)(int);
 	struct desc_struct	save_desc_40;
 	struct desc_struct	*gdt;
 	struct apm_bios_call	*call = _call;
+	u64			ibt;
 	cpu = get_cpu();
 	BUG_ON(cpu != 0);
 	apm_irq_save(flags);
 	firmware_restrict_branch_speculation_start();
+	ibt = ibt_save();
 	APM_DO_SAVE_SEGS;
 	apm_bios_call_asm(call->func, call->ebx, call->ecx,
 			  &call->eax, &call->ebx, &call->ecx, &call->edx,
 			  &call->esi);
 	APM_DO_RESTORE_SEGS;
+	ibt_restore(ibt);
 	firmware_restrict_branch_speculation_end();
 	apm_irq_restore(flags);
 	gdt[0x40 / 8] = save_desc_40;
 	struct desc_struct	save_desc_40;
 	struct desc_struct	*gdt;
 	struct apm_bios_call	*call = _call;
+	u64			ibt;
 	cpu = get_cpu();
 	BUG_ON(cpu != 0);
 	apm_irq_save(flags);
 	firmware_restrict_branch_speculation_start();
+	ibt = ibt_save();
 	APM_DO_SAVE_SEGS;
 	error = apm_bios_call_simple_asm(call->func, call->ebx, call->ecx,
 					 &call->eax);
 	APM_DO_RESTORE_SEGS;
+	ibt_restore(ibt);
 	firmware_restrict_branch_speculation_end();
 	apm_irq_restore(flags);
 	gdt[0x40 / 8] = save_desc_40;

arch/x86/kernel/cpu/common.c+58−1

@@ -59,6 +59,7 @@ · 58 unmodified lines
59	#include <asm/cpu_device_id.h>	59	#include <asm/cpu_device_id.h>
60	#include <asm/uv/uv.h>	60	#include <asm/uv/uv.h>
61	#include <asm/sigframe.h>	61	#include <asm/sigframe.h>
		62	#include <asm/traps.h>
62		63
63	#include "cpu.h"	64	#include "cpu.h"
64		65
@@ -438,7 +439,8 @@ out: · 373 unmodified lines
438		439
439	/* These bits should not change their value after CPU init is finished. */	440	/* These bits should not change their value after CPU init is finished. */
440	static const unsigned long cr4_pinned_mask =	441	static const unsigned long cr4_pinned_mask =
441	X86_CR4_SMEP \| X86_CR4_SMAP \| X86_CR4_UMIP \| X86_CR4_FSGSBASE;	442	X86_CR4_SMEP \| X86_CR4_SMAP \| X86_CR4_UMIP \|
		443	X86_CR4_FSGSBASE \| X86_CR4_CET;
442	static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);	444	static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
443	static unsigned long cr4_pinned_bits __ro_after_init;	445	static unsigned long cr4_pinned_bits __ro_after_init;
444		446
@@ -592,6 +594,58 @@ static __init int setup_disable_pku(char *arg) · 147 unmodified lines
592	__setup("nopku", setup_disable_pku);	594	__setup("nopku", setup_disable_pku);
593	#endif /* CONFIG_X86_64 */	595	#endif /* CONFIG_X86_64 */
594		596
		597	#ifdef CONFIG_X86_KERNEL_IBT
		598
		599	__noendbr u64 ibt_save(void)
		600	{
		601	u64 msr = 0;
		602
		603	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
		604	rdmsrl(MSR_IA32_S_CET, msr);
		605	wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
		606	}
		607
		608	return msr;
		609	}
		610
		611	__noendbr void ibt_restore(u64 save)
		612	{
		613	u64 msr;
		614
		615	if (cpu_feature_enabled(X86_FEATURE_IBT)) {
		616	rdmsrl(MSR_IA32_S_CET, msr);
		617	msr &= ~CET_ENDBR_EN;
		618	msr \|= (save & CET_ENDBR_EN);
		619	wrmsrl(MSR_IA32_S_CET, msr);
		620	}
		621	}
		622
		623	#endif
		624
		625	static __always_inline void setup_cet(struct cpuinfo_x86 *c)
		626	{
		627	u64 msr = CET_ENDBR_EN;
		628
		629	if (!HAS_KERNEL_IBT \|\|
		630	!cpu_feature_enabled(X86_FEATURE_IBT))
		631	return;
		632
		633	wrmsrl(MSR_IA32_S_CET, msr);
		634	cr4_set_bits(X86_CR4_CET);
		635
		636	if (!ibt_selftest()) {
		637	pr_err("IBT selftest: Failed!\n");
		638	setup_clear_cpu_cap(X86_FEATURE_IBT);
		639	return;
		640	}
		641	}
		642
		643	__noendbr void cet_disable(void)
		644	{
		645	if (cpu_feature_enabled(X86_FEATURE_IBT))
		646	wrmsrl(MSR_IA32_S_CET, 0);
		647	}
		648
595	/*	649	/*
596	* Some CPU features depend on higher CPUID levels, which may not always	650	* Some CPU features depend on higher CPUID levels, which may not always
597	* be available due to CPUID level capping or broken virtualization	651	* be available due to CPUID level capping or broken virtualization
@@ -1709,6 +1763,7 @@ static void identify_cpu(struct cpuinfo_x86 *c) · 1111 unmodified lines
1709		1763
1710	x86_init_rdrand(c);	1764	x86_init_rdrand(c);
1711	setup_pku(c);	1765	setup_pku(c);
		1766	setup_cet(c);
1712		1767
1713	/*	1768	/*
1714	* Clear/Set all flags overridden by options, need do it	1769	* Clear/Set all flags overridden by options, need do it
@@ -1777,6 +1832,8 @@ void enable_sep_cpu(void) · 62 unmodified lines
1777	void __init identify_boot_cpu(void)	1832	void __init identify_boot_cpu(void)
1778	{	1833	{
1779	identify_cpu(&boot_cpu_data);	1834	identify_cpu(&boot_cpu_data);
		1835	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
		1836	pr_info("CET detected: Indirect Branch Tracking enabled\n");
1780	#ifdef CONFIG_X86_32	1837	#ifdef CONFIG_X86_32
1781	sysenter_setup();	1838	sysenter_setup();
1782	enable_sep_cpu();	1839	enable_sep_cpu();

arch/x86/kernel/ftrace.c+2−7

@@ -316,12 +316,12 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 315 unmodified lines
316	unsigned long offset;	316	unsigned long offset;
317	unsigned long npages;	317	unsigned long npages;
318	unsigned long size;	318	unsigned long size;
319	unsigned long retq;
320	unsigned long *ptr;	319	unsigned long *ptr;
321	void *trampoline;	320	void *trampoline;
322	void *ip;	321	void *ip;
323	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */	322	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
324	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };	323	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
		324	unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
325	union ftrace_op_code_union op_ptr;	325	union ftrace_op_code_union op_ptr;
326	int ret;	326	int ret;
327		327
@@ -359,12 +359,7 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 31 unmodified lines
359	goto fail;	359	goto fail;
360		360
361	ip = trampoline + size;	361	ip = trampoline + size;
362		362	memcpy(ip, retq, RET_SIZE);
363	/* The trampoline ends with ret(q) */
364	retq = (unsigned long)ftrace_stub;
365	ret = copy_from_kernel_nofault(ip, (void *)retq, RET_SIZE);
366	if (WARN_ON(ret < 0))
367	goto fail;
368		363
369	/* No need to test direct calls on created trampolines */	364	/* No need to test direct calls on created trampolines */
370	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {	365	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {

arch/x86/kernel/ftrace_64.S+24−4

@@ -145,6 +145,7 @@ SYM_FUNC_START(ftrace_caller) · 144 unmodified lines
145	movq %rcx, RSP(%rsp)	145	movq %rcx, RSP(%rsp)
146		146
147	SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)	147	SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
		148	ANNOTATE_NOENDBR
148	/* Load the ftrace_ops into the 3rd parameter */	149	/* Load the ftrace_ops into the 3rd parameter */
149	movq function_trace_op(%rip), %rdx	150	movq function_trace_op(%rip), %rdx
150		151
@@ -155,6 +156,7 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) · 4 unmodified lines
155	movq $0, CS(%rsp)	156	movq $0, CS(%rsp)
156		157
157	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)	158	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
		159	ANNOTATE_NOENDBR
158	call ftrace_stub	160	call ftrace_stub
159		161
160	/* Handlers can change the RIP */	162	/* Handlers can change the RIP */
@@ -169,6 +171,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) · 8 unmodified lines
169	* layout here.	171	* layout here.
170	*/	172	*/
171	SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)	173	SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
		174	ANNOTATE_NOENDBR
172		175
173	jmp ftrace_epilogue	176	jmp ftrace_epilogue
174	SYM_FUNC_END(ftrace_caller);	177	SYM_FUNC_END(ftrace_caller);
@@ -176,10 +179,10 @@ SYM_FUNC_END(ftrace_caller); · 1 unmodified line
176	SYM_FUNC_START(ftrace_epilogue)	179	SYM_FUNC_START(ftrace_epilogue)
177	/*	180	/*
178	* This is weak to keep gas from relaxing the jumps.	181	* This is weak to keep gas from relaxing the jumps.
179	* It is also used to copy the RET for trampolines.
180	*/	182	*/
181	SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)	183	SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
182	UNWIND_HINT_FUNC	184	UNWIND_HINT_FUNC
		185	ENDBR
183	RET	186	RET
184	SYM_FUNC_END(ftrace_epilogue)	187	SYM_FUNC_END(ftrace_epilogue)
185		188
@@ -192,6 +195,7 @@ SYM_FUNC_START(ftrace_regs_caller) · 6 unmodified lines
192	/* save_mcount_regs fills in first two parameters */	195	/* save_mcount_regs fills in first two parameters */
193		196
194	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)	197	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
		198	ANNOTATE_NOENDBR
195	/* Load the ftrace_ops into the 3rd parameter */	199	/* Load the ftrace_ops into the 3rd parameter */
196	movq function_trace_op(%rip), %rdx	200	movq function_trace_op(%rip), %rdx
197		201
@@ -221,6 +225,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) · 23 unmodified lines
221	leaq (%rsp), %rcx	225	leaq (%rsp), %rcx
222		226
223	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)	227	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
		228	ANNOTATE_NOENDBR
224	call ftrace_stub	229	call ftrace_stub
225		230
226	/* Copy flags back to SS, to restore them */	231	/* Copy flags back to SS, to restore them */
@@ -248,6 +253,7 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 21 unmodified lines
248	*/	253	*/
249	testq %rax, %rax	254	testq %rax, %rax
250	SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)	255	SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)
		256	ANNOTATE_NOENDBR
251	jnz 1f	257	jnz 1f
252		258
253	restore_mcount_regs	259	restore_mcount_regs
@@ -261,6 +267,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) · 7 unmodified lines
261	* to the return.	267	* to the return.
262	*/	268	*/
263	SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)	269	SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
		270	ANNOTATE_NOENDBR
264	jmp ftrace_epilogue	271	jmp ftrace_epilogue
265		272
266	/* Swap the flags with orig_rax */	273	/* Swap the flags with orig_rax */
@@ -284,6 +291,7 @@ SYM_FUNC_START(__fentry__) · 17 unmodified lines
284	jnz trace	291	jnz trace
285		292
286	SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)	293	SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
		294	ENDBR
287	RET	295	RET
288		296
289	trace:	297	trace:
@@ -307,7 +315,7 @@ EXPORT_SYMBOL(__fentry__) · 17 unmodified lines
307		315
308	#ifdef CONFIG_FUNCTION_GRAPH_TRACER	316	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
309	SYM_FUNC_START(return_to_handler)	317	SYM_FUNC_START(return_to_handler)
310	subq $24, %rsp	318	subq $16, %rsp
311		319
312	/* Save the return values */	320	/* Save the return values */
313	movq %rax, (%rsp)	321	movq %rax, (%rsp)
@@ -319,7 +327,19 @@ SYM_FUNC_START(return_to_handler) · 5 unmodified lines
319	movq %rax, %rdi	327	movq %rax, %rdi
320	movq 8(%rsp), %rdx	328	movq 8(%rsp), %rdx
321	movq (%rsp), %rax	329	movq (%rsp), %rax
322	addq $24, %rsp	330
323	JMP_NOSPEC rdi	331	addq $16, %rsp
		332	/*
		333	* Jump back to the old return address. This cannot be JMP_NOSPEC rdi
		334	* since IBT would demand that contain ENDBR, which simply isn't so for
		335	* return addresses. Use a retpoline here to keep the RSB balanced.
		336	*/
		337	ANNOTATE_INTRA_FUNCTION_CALL
		338	call .Ldo_rop
		339	int3
		340	.Ldo_rop:
		341	mov %rdi, (%rsp)
		342	UNWIND_HINT_FUNC
		343	RET
324	SYM_FUNC_END(return_to_handler)	344	SYM_FUNC_END(return_to_handler)
325	#endif	345	#endif

arch/x86/kernel/head_64.S+12−2

 SYM_CODE_START(secondary_startup_64)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR
 	/*
 	 * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
 	 * and someone has loaded a mapped page table.
 	 */
 SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR
 	/*
 	 * Retrieve the modifier (SME encryption mask if SME is active) to be
 	jmp	*%rax
 1:
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR // above
 	/*
 	 * We must switch to a new descriptor in kernel space for the GDT
 	pushq	%rax		# target address in negative space
 	lretq
 .Lafter_lret:
+	ANNOTATE_NOENDBR
 SYM_CODE_END(secondary_startup_64)
 #include "verify_cpu.S"
  */
 SYM_CODE_START_NOALIGN(vc_boot_ghcb)
 	UNWIND_HINT_IRET_REGS offset=8
+	ENDBR
 	/* Build pt_regs */
 	PUSH_AND_CLEAR_REGS
 	/* Remove Error Code */
 	addq    $8, %rsp
-	/* Pure iret required here - don't use INTERRUPT_RETURN */
 	iretq
 SYM_CODE_END(vc_boot_ghcb)
 #endif
 	.rept NUM_EXCEPTION_VECTORS
 	.if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
 		UNWIND_HINT_IRET_REGS
+		ENDBR
 		pushq $0	# Dummy error code, to make stack frame uniform
 	.else
 		UNWIND_HINT_IRET_REGS offset=8
+		ENDBR
 	.endif
 	pushq $i		# 72(%rsp) Vector number
 	jmp early_idt_handler_common
 	i = i + 1
 	.fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
 	.endr
-	UNWIND_HINT_IRET_REGS offset=16
 SYM_CODE_END(early_idt_handler_array)
+	ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]
 SYM_CODE_START_LOCAL(early_idt_handler_common)
+	UNWIND_HINT_IRET_REGS offset=16
 	/*
 	 * The stack is the hardware frame, an error code or zero, and the
 	 * vector number.
  * early_idt_handler_array can't be used because it returns via the
  * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
  *
+ * XXX it does, fix this.
+ *
  * This handler will end up in the .init.text section and not be
  * available to boot secondary CPUs.
  */
 SYM_CODE_START_NOALIGN(vc_no_ghcb)
 	UNWIND_HINT_IRET_REGS offset=8
+	ENDBR
 	/* Build pt_regs */
 	PUSH_AND_CLEAR_REGS

arch/x86/kernel/idt.c+7−2

 #include <asm/proto.h>
 #include <asm/desc.h>
 #include <asm/hw_irq.h>
+#include <asm/idtentry.h>
 #define DPL0		0x0
 #define DPL3		0x3
 	ISTG(X86_TRAP_MC,		asm_exc_machine_check, IST_INDEX_MCE),
 #endif
+#ifdef CONFIG_X86_KERNEL_IBT
+	INTG(X86_TRAP_CP,		asm_exc_control_protection),
+#endif
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 	ISTG(X86_TRAP_VC,		asm_exc_vmm_communication, IST_INDEX_VC),
 #endif
 	idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
 	for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
-		entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
+		entry = irq_entries_start + IDT_ALIGN * (i - FIRST_EXTERNAL_VECTOR);
 		set_intr_gate(i, entry);
 	}
 		 * system_vectors bitmap. Otherwise they show up in
 		 * /proc/interrupts.
 		 */
-		entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
+		entry = spurious_entries_start + IDT_ALIGN * (i - FIRST_SYSTEM_VECTOR);
 		set_intr_gate(i, entry);
 	}
 #endif

arch/x86/kernel/kprobes/core.c+20−9

 #include <asm/insn.h>
 #include <asm/debugreg.h>
 #include <asm/set_memory.h>
+#include <asm/ibt.h>
 #include "common.h"
 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
 	struct kprobe *kp;
-	unsigned long faddr;
+	bool faddr;
 	kp = get_kprobe((void *)addr);
-	faddr = ftrace_location(addr);
+	faddr = ftrace_location(addr) == addr;
-	/*
-	 * Addresses inside the ftrace location are refused by
-	 * arch_check_ftrace_location(). Something went terribly wrong
-	 * if such an address is checked here.
-	 */
-	if (WARN_ON(faddr && faddr != addr))
-		return 0UL;
 	/*
 	 * Use the current code if it is not modified by Kprobe
 	 * and it cannot be modified by ftrace.
 	return (addr == paddr);
 }
+/* If x86 supports IBT (ENDBR) it must be skipped. */
+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
+					 bool *on_func_entry)
+{
+	if (is_endbr(*(u32 *)addr)) {
+		*on_func_entry = !offset || offset == 4;
+		if (*on_func_entry)
+			offset = 4;
+	} else {
+		*on_func_entry = !offset;
+	}
+	return (kprobe_opcode_t *)(addr + offset);
+}
 /*
  * Copy an instruction with recovering modified instruction by kprobes
  * and adjust the displacement if the instruction uses the %rip-relative
 	".type __kretprobe_trampoline, @function\n"
 	"__kretprobe_trampoline:\n"
 #ifdef CONFIG_X86_64
+	ANNOTATE_NOENDBR
 	/* Push a fake return address to tell the unwinder it's a kretprobe. */
 	"	pushq $__kretprobe_trampoline\n"
 	UNWIND_HINT_FUNC

arch/x86/kernel/kvm.c+2−1

 ".global __raw_callee_save___kvm_vcpu_is_preempted;"
 ".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
 "__raw_callee_save___kvm_vcpu_is_preempted:"
+ASM_ENDBR
 "movq	__per_cpu_offset(,%rdi,8), %rax;"
 "cmpb	$0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
 "setne	%al;"
-"ret;"
+ASM_RET
 ".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
 ".popsection");

arch/x86/kernel/machine_kexec_64.c+3−1

 #include <asm/kexec-bzimage64.h>
 #include <asm/setup.h>
 #include <asm/set_memory.h>
+#include <asm/cpu.h>
 #ifdef CONFIG_ACPI
 /*
 	/* Interrupts aren't acceptable while we reboot */
 	local_irq_disable();
 	hw_breakpoint_disable();
+	cet_disable();
 	if (image->preserve_context) {
 #ifdef CONFIG_X86_IO_APIC
 	}
 	control_page = page_address(image->control_code_page) + PAGE_SIZE;
-	memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
+	__memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
 	page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
 	page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;

arch/x86/kernel/module.c+7−1

 {
 	const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL,
 		*para = NULL, *orc = NULL, *orc_ip = NULL,
-		*retpolines = NULL;
+		*retpolines = NULL, *ibt_endbr = NULL;
 	char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
 			orc_ip = s;
 		if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
 			retpolines = s;
+		if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
+			ibt_endbr = s;
 	}
 	/*
 		void *aseg = (void *)alt->sh_addr;
 		apply_alternatives(aseg, aseg + alt->sh_size);
 	}
+	if (ibt_endbr) {
+		void *iseg = (void *)ibt_endbr->sh_addr;
+		apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);
+	}
 	if (locks && text) {
 		void *lseg = (void *)locks->sh_addr;
 		void *tseg = (void *)text->sh_addr;

arch/x86/kernel/paravirt.c+5−24

 asm (".pushsection .entry.text, \"ax\"\n"
      ".global _paravirt_nop\n"
      "_paravirt_nop:\n\t"
+     ASM_ENDBR
      ASM_RET
      ".size _paravirt_nop, . - _paravirt_nop\n\t"
      ".type _paravirt_nop, @function\n\t"
 asm (".pushsection .entry.text, \"ax\"\n"
      ".global paravirt_ret0\n"
      "paravirt_ret0:\n\t"
+     ASM_ENDBR
      "xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
      ASM_RET
      ".size paravirt_ret0, . - paravirt_ret0\n\t"
 	BUG();
 }
-struct branch {
-	unsigned char opcode;
-	u32 delta;
-} __attribute__((packed));
 static unsigned paravirt_patch_call(void *insn_buff, const void *target,
 				    unsigned long addr, unsigned len)
 {
-	const int call_len = 5;
+	__text_gen_insn(insn_buff, CALL_INSN_OPCODE,
-	struct branch *b = insn_buff;
+			(void *)addr, target, CALL_INSN_SIZE);
-	unsigned long delta = (unsigned long)target - (addr+call_len);
+	return CALL_INSN_SIZE;
-	if (len < call_len) {
-		pr_warn("paravirt: Failed to patch indirect CALL at %ps\n", (void *)addr);
-		/* Kernel might not be viable if patching fails, bail out: */
-		BUG_ON(1);
-	}
-	b->opcode = 0xe8; /* call */
-	b->delta = delta;
-	BUILD_BUG_ON(sizeof(*b) != call_len);
-	return call_len;
 }
 #ifdef CONFIG_PARAVIRT_XXL
 }
 /* These are in entry.S */
-extern void native_iret(void);
 static struct resource reserve_ioports = {
 	.start = 0,
 	.end = IO_SPACE_LIMIT,
 #ifdef CONFIG_PARAVIRT_XXL
 NOKPROBE_SYMBOL(native_load_idt);
-void (*paravirt_iret)(void) = native_iret;
 #endif
 EXPORT_SYMBOL(pv_ops);

arch/x86/kernel/process.c+1−1

@@ -747,7 +747,7 @@ bool xen_set_default_idle(void) · 746 unmodified lines
747	}	747	}
748	#endif	748	#endif
749		749
750	void stop_this_cpu(void *dummy)	750	void __noreturn stop_this_cpu(void *dummy)
751	{	751	{
752	local_irq_disable();	752	local_irq_disable();
753	/*	753	/*

arch/x86/kernel/process_64.c+1−1

@@ -681,7 +681,7 @@ void set_personality_64bit(void) · 680 unmodified lines
681		681
682	static void __set_personality_x32(void)	682	static void __set_personality_x32(void)
683	{	683	{
684	#ifdef CONFIG_X86_X32	684	#ifdef CONFIG_X86_X32_ABI
685	if (current->mm)	685	if (current->mm)
686	current->mm->context.flags = 0;	686	current->mm->context.flags = 0;
687		687

arch/x86/kernel/relocate_kernel_64.S+10−0

 	.code64
 SYM_CODE_START_NOALIGN(relocate_kernel)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR
 	/*
 	 * %rdi indirection_page
 	 * %rsi page_list
 	/* store the start address on the stack */
 	pushq   %rdx
+	/*
+	 * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
+	 * below.
+	 */
+	movq	%cr4, %rax
+	andq	$~(X86_CR4_CET), %rax
+	movq	%rax, %cr4
 	/*
 	 * Set cr0 to a known state:
 	 *  - Paging enabled
 SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR // RET target, above
 	movq	RSP(%r8), %rsp
 	movq	CR4(%r8), %rax
 	movq	%rax, %cr4

arch/x86/kernel/traps.c+75−0

@@ -210,6 +210,81 @@ DEFINE_IDTENTRY(exc_overflow) · 209 unmodified lines
210	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);	210	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);
211	}	211	}
212		212
		213	#ifdef CONFIG_X86_KERNEL_IBT
		214
		215	static __ro_after_init bool ibt_fatal = true;
		216
		217	extern void ibt_selftest_ip(void); /* code label defined in asm below */
		218
		219	enum cp_error_code {
		220	CP_EC = (1 << 15) - 1,
		221
		222	CP_RET = 1,
		223	CP_IRET = 2,
		224	CP_ENDBR = 3,
		225	CP_RSTRORSSP = 4,
		226	CP_SETSSBSY = 5,
		227
		228	CP_ENCL = 1 << 15,
		229	};
		230
		231	DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
		232	{
		233	if (!cpu_feature_enabled(X86_FEATURE_IBT)) {
		234	pr_err("Unexpected #CP\n");
		235	BUG();
		236	}
		237
		238	if (WARN_ON_ONCE(user_mode(regs) \|\| (error_code & CP_EC) != CP_ENDBR))
		239	return;
		240
		241	if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {
		242	regs->ax = 0;
		243	return;
		244	}
		245
		246	pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
		247	if (!ibt_fatal) {
		248	printk(KERN_DEFAULT CUT_HERE);
		249	__warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
		250	return;
		251	}
		252	BUG();
		253	}
		254
		255	/* Must be noinline to ensure uniqueness of ibt_selftest_ip. */
		256	noinline bool ibt_selftest(void)
		257	{
		258	unsigned long ret;
		259
		260	asm (" lea ibt_selftest_ip(%%rip), %%rax\n\t"
		261	ANNOTATE_RETPOLINE_SAFE
		262	" jmp *%%rax\n\t"
		263	"ibt_selftest_ip:\n\t"
		264	UNWIND_HINT_FUNC
		265	ANNOTATE_NOENDBR
		266	" nop\n\t"
		267
		268	: "=a" (ret) : : "memory");
		269
		270	return !ret;
		271	}
		272
		273	static int __init ibt_setup(char *str)
		274	{
		275	if (!strcmp(str, "off"))
		276	setup_clear_cpu_cap(X86_FEATURE_IBT);
		277
		278	if (!strcmp(str, "warn"))
		279	ibt_fatal = false;
		280
		281	return 1;
		282	}
		283
		284	__setup("ibt=", ibt_setup);
		285
		286	#endif /* CONFIG_X86_KERNEL_IBT */
		287
213	#ifdef CONFIG_X86_F00F_BUG	288	#ifdef CONFIG_X86_F00F_BUG
214	void handle_invalid_op(struct pt_regs *regs)	289	void handle_invalid_op(struct pt_regs *regs)
215	#else	290	#else

arch/x86/kernel/vmlinux.lds.S+9−0

@@ -285,6 +285,15 @@ SECTIONS · 284 unmodified lines
285	}	285	}
286	#endif	286	#endif
287		287
		288	#ifdef CONFIG_X86_KERNEL_IBT
		289	. = ALIGN(8);
		290	.ibt_endbr_seal : AT(ADDR(.ibt_endbr_seal) - LOAD_OFFSET) {
		291	__ibt_endbr_seal = .;
		292	*(.ibt_endbr_seal)
		293	__ibt_endbr_seal_end = .;
		294	}
		295	#endif
		296
288	/*	297	/*
289	* struct alt_inst entries. From the header (alternative.h):	298	* struct alt_inst entries. From the header (alternative.h):
290	* "Alternative instructions for different CPU types or capabilities"	299	* "Alternative instructions for different CPU types or capabilities"

arch/x86/kvm/emulate.c+10−6

 #include <linux/stringify.h>
 #include <asm/debugreg.h>
 #include <asm/nospec-branch.h>
+#include <asm/ibt.h>
 #include "x86.h"
 #include "tss.h"
 #define X16(x...) X8(x), X8(x)
 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
-#define FASTOP_SIZE 8
+#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))
 struct opcode {
 	u64 flags;
 #define __FOP_FUNC(name) \
 	".align " __stringify(FASTOP_SIZE) " \n\t" \
 	".type " name ", @function \n\t" \
-	name ":\n\t"
+	name ":\n\t" \
+	ASM_ENDBR
 #define FOP_FUNC(name) \
 	__FOP_FUNC(#name)
 /*
  * Depending on .config the SETcc functions look like:
  *
+ * ENDBR       [4 bytes; CONFIG_X86_KERNEL_IBT]
  * SETcc %al   [3 bytes]
  * RET         [1 byte]
  * INT3        [1 byte; CONFIG_SLS]
  *
- * Which gives possible sizes 4 or 5.  When rounded up to the
+ * Which gives possible sizes 4, 5, 8 or 9.  When rounded up to the
- * next power-of-two alignment they become 4 or 8.
+ * next power-of-two alignment they become 4, 8 or 16 resp.
  */
-#define SETCC_LENGTH	(4 + IS_ENABLED(CONFIG_SLS))
+#define SETCC_LENGTH	(ENDBR_INSN_SIZE + 4 + IS_ENABLED(CONFIG_SLS))
-#define SETCC_ALIGN	(4 << IS_ENABLED(CONFIG_SLS))
+#define SETCC_ALIGN	(4 << IS_ENABLED(CONFIG_SLS) << HAS_KERNEL_IBT)
 static_assert(SETCC_LENGTH <= SETCC_ALIGN);
 #define FOP_SETCC(op) \
 	".align " __stringify(SETCC_ALIGN) " \n\t" \
 	".type " #op ", @function \n\t" \
 	#op ": \n\t" \
+	ASM_ENDBR \
 	#op " %al \n\t" \
 	__FOP_RET(#op)

arch/x86/lib/error-inject.c+2−0

 #include <linux/linkage.h>
 #include <linux/error-injection.h>
 #include <linux/kprobes.h>
+#include <linux/objtool.h>
 asmlinkage void just_return_func(void);
 	".type just_return_func, @function\n"
 	".globl just_return_func\n"
 	"just_return_func:\n"
+		ANNOTATE_NOENDBR
 		ASM_RET
 	".size just_return_func, .-just_return_func\n"
 );

arch/x86/lib/retpoline.S+1−0

@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) · 54 unmodified lines
55		55
56	.align RETPOLINE_THUNK_SIZE	56	.align RETPOLINE_THUNK_SIZE
57	SYM_CODE_START(__x86_indirect_thunk_array)	57	SYM_CODE_START(__x86_indirect_thunk_array)
		58	ANNOTATE_NOENDBR // apply_retpolines
58		59
59	#define GEN(reg) THUNK reg	60	#define GEN(reg) THUNK reg
60	#include <asm/GEN-for-each-reg.h>	61	#include <asm/GEN-for-each-reg.h>

arch/x86/net/bpf_jit_comp.c+24−2

 #define EMIT4_off32(b1, b2, b3, b4, off) \
 	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
+#ifdef CONFIG_X86_KERNEL_IBT
+#define EMIT_ENDBR()	EMIT(gen_endbr(), 4)
+#else
+#define EMIT_ENDBR()
+#endif
 static bool is_imm8(int value)
 {
 	return value <= 127 && value >= -128;
 /* Number of bytes emit_patch() needs to generate instructions */
 #define X86_PATCH_SIZE		5
 /* Number of bytes that will be skipped on tailcall */
-#define X86_TAIL_CALL_OFFSET	11
+#define X86_TAIL_CALL_OFFSET	(11 + ENDBR_INSN_SIZE)
 static void push_callee_regs(u8 **pprog, bool *callee_regs_used)
 {
 	/* BPF trampoline can be made to work without these nops,
 	 * but let's waste 5 bytes for now and optimize later
 	 */
+	EMIT_ENDBR();
 	memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
 	prog += X86_PATCH_SIZE;
 	if (!ebpf_from_cbpf) {
 	}
 	EMIT1(0x55);             /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
+	/* X86_TAIL_CALL_OFFSET is here */
+	EMIT_ENDBR();
 	/* sub rsp, rounded_stack_depth */
 	if (stack_depth)
 		EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
 		/* BPF poking in modules is not supported */
 		return -EINVAL;
+	/*
+	 * See emit_prologue(), for IBT builds the trampoline hook is preceded
+	 * with an ENDBR instruction.
+	 */
+	if (is_endbr(*(u32 *)ip))
+		ip += ENDBR_INSN_SIZE;
 	return __bpf_arch_text_poke(ip, t, old_addr, new_addr);
 }
 	ip_off = stack_size;
-	if (flags & BPF_TRAMP_F_SKIP_FRAME)
+	if (flags & BPF_TRAMP_F_SKIP_FRAME) {
 		/* skip patched call instruction and point orig_call to actual
 		 * body of the kernel function.
 		 */
+		if (is_endbr(*(u32 *)orig_call))
+			orig_call += ENDBR_INSN_SIZE;
 		orig_call += X86_PATCH_SIZE;
+	}
 	prog = image;
+	EMIT_ENDBR();
 	EMIT1(0x55);		 /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
 	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */

arch/x86/platform/efi/Makefile+0−1

@@ -1,5 +1,4 @@
1	# SPDX-License-Identifier: GPL-2.0	1	# SPDX-License-Identifier: GPL-2.0
2	OBJECT_FILES_NON_STANDARD_efi_thunk_$(BITS).o := y
3	KASAN_SANITIZE := n	2	KASAN_SANITIZE := n
4	GCOV_PROFILE := n	3	GCOV_PROFILE := n
5		4

arch/x86/platform/efi/efi_thunk_64.S+4−2

  */
 #include <linux/linkage.h>
+#include <linux/objtool.h>
 #include <asm/page_types.h>
 #include <asm/segment.h>
 	.text
 	.code64
-SYM_CODE_START(__efi64_thunk)
+SYM_FUNC_START(__efi64_thunk)
+STACK_FRAME_NON_STANDARD __efi64_thunk
 	push	%rbp
 	push	%rbx
 2:	pushl	$__KERNEL_CS
 	pushl	%ebp
 	lret
-SYM_CODE_END(__efi64_thunk)
+SYM_FUNC_END(__efi64_thunk)
 	.bss
 	.balign 8

arch/x86/xen/enlighten_pv.c+9−1

 	TRAP_ENTRY(exc_coprocessor_error,		false ),
 	TRAP_ENTRY(exc_alignment_check,			false ),
 	TRAP_ENTRY(exc_simd_coprocessor_error,		false ),
+#ifdef CONFIG_X86_KERNEL_IBT
+	TRAP_ENTRY(exc_control_protection,		false ),
+#endif
 };
 static bool __ref get_trap_addr(void **addr, unsigned int ist)
 	x86_platform.legacy.rtc = 0;
 }
+extern void early_xen_iret_patch(void);
 /* First C function to be called on Xen boot */
 asmlinkage __visible void __init xen_start_kernel(void)
 {
 	if (!xen_start_info)
 		return;
+	__text_gen_insn(&early_xen_iret_patch,
+			JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret,
+			JMP32_INSN_SIZE);
 	xen_domain_type = XEN_PV_DOMAIN;
 	xen_start_flags = xen_start_info->flags;
 	/* Install Xen paravirt ops */
 	pv_info = xen_info;
 	pv_ops.cpu = xen_cpu_ops.cpu;
-	paravirt_iret = xen_iret;
 	xen_init_irq_ops();
 	/*

arch/x86/xen/xen-asm.S+10−0

 .macro xen_pv_trap name
 SYM_CODE_START(xen_\name)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	pop %rcx
 	pop %r11
 	jmp  \name
 xen_pv_trap asm_exc_spurious_interrupt_bug
 xen_pv_trap asm_exc_coprocessor_error
 xen_pv_trap asm_exc_alignment_check
+#ifdef CONFIG_X86_KERNEL_IBT
+xen_pv_trap asm_exc_control_protection
+#endif
 #ifdef CONFIG_X86_MCE
 xen_pv_trap asm_xenpv_exc_machine_check
 #endif /* CONFIG_X86_MCE */
 	i = 0
 	.rept NUM_EXCEPTION_VECTORS
 	UNWIND_HINT_EMPTY
+	ENDBR
 	pop %rcx
 	pop %r11
 	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
  */
 SYM_CODE_START(xen_iret)
 	UNWIND_HINT_EMPTY
+	ANNOTATE_NOENDBR
 	pushq $0
 	jmp hypercall_iret
 SYM_CODE_END(xen_iret)
 /* Normal 64-bit system call target */
 SYM_CODE_START(xen_syscall_target)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	popq %rcx
 	popq %r11
 /* 32-bit compat syscall target */
 SYM_CODE_START(xen_syscall32_target)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	popq %rcx
 	popq %r11
 /* 32-bit compat sysenter target */
 SYM_CODE_START(xen_sysenter_target)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	/*
 	 * NB: Xen is polite and clears TF from EFLAGS for us.  This means
 	 * that we don't need to guard against single step exceptions here.
 SYM_CODE_START(xen_syscall32_target)
 SYM_CODE_START(xen_sysenter_target)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	lea 16(%rsp), %rsp	/* strip %rcx, %r11 */
 	mov $-ENOSYS, %rax
 	pushq $0

arch/x86/xen/xen-head.S+7−2

 SYM_CODE_START(hypercall_page)
 	.rept (PAGE_SIZE / 32)
 		UNWIND_HINT_FUNC
-		.skip 31, 0x90
+		ANNOTATE_NOENDBR
-		RET
+		ret
+		/*
+		 * Xen will write the hypercall page, and sort out ENDBR.
+		 */
+		.skip 31, 0xcc
 	.endr
 #define HYPERCALL(n) \
 .pushsection .text
 SYM_CODE_START(asm_cpu_bringup_and_idle)
 	UNWIND_HINT_EMPTY
+	ENDBR
 	call cpu_bringup_and_idle
 SYM_CODE_END(asm_cpu_bringup_and_idle)

fs/fuse/ioctl.c+1−1

@@ -170,7 +170,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 169 unmodified lines
170	#else	170	#else
171	if (flags & FUSE_IOCTL_COMPAT) {	171	if (flags & FUSE_IOCTL_COMPAT) {
172	inarg.flags \|= FUSE_IOCTL_32BIT;	172	inarg.flags \|= FUSE_IOCTL_32BIT;
173	#ifdef CONFIG_X86_X32	173	#ifdef CONFIG_X86_X32_ABI
174	if (in_x32_syscall())	174	if (in_x32_syscall())
175	inarg.flags \|= FUSE_IOCTL_COMPAT_X32;	175	inarg.flags \|= FUSE_IOCTL_COMPAT_X32;
176	#endif	176	#endif

fs/xfs/xfs_ioctl32.c+1−1

@@ -217,7 +217,7 @@ xfs_compat_ioc_fsbulkstat( · 216 unmodified lines
217	inumbers_fmt_pf inumbers_func = xfs_fsinumbers_fmt_compat;	217	inumbers_fmt_pf inumbers_func = xfs_fsinumbers_fmt_compat;
218	bulkstat_one_fmt_pf bs_one_func = xfs_fsbulkstat_one_fmt_compat;	218	bulkstat_one_fmt_pf bs_one_func = xfs_fsbulkstat_one_fmt_compat;
219		219
220	#ifdef CONFIG_X86_X32	220	#ifdef CONFIG_X86_X32_ABI
221	if (in_x32_syscall()) {	221	if (in_x32_syscall()) {
222	/*	222	/*
223	* ... but on x32 the input xfs_fsop_bulkreq has pointers	223	* ... but on x32 the input xfs_fsop_bulkreq has pointers

include/asm-generic/vmlinux.lds.h+4−0

 	KEEP(*(__jump_table))						\
 	__stop___jump_table = .;
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
 #define STATIC_CALL_DATA						\
 	. = ALIGN(8);							\
 	__start_static_call_sites = .;					\
 	__start_static_call_tramp_key = .;				\
 	KEEP(*(.static_call_tramp_key))					\
 	__stop_static_call_tramp_key = .;
+#else
+#define STATIC_CALL_DATA
+#endif
 /*
  * Allow architectures to handle ro_after_init data on their

include/linux/cfi.h+10−1

@@ -34,8 +34,17 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr · 33 unmodified lines
34		34
35	#else /* !CONFIG_CFI_CLANG */	35	#else /* !CONFIG_CFI_CLANG */
36		36
37	#define __CFI_ADDRESSABLE(fn, __attr)	37	#ifdef CONFIG_X86_KERNEL_IBT
		38
		39	#define __CFI_ADDRESSABLE(fn, __attr) \
		40	const void __cfi_jt_ ## fn __visible __attr = (void )&fn
		41
		42	#endif /* CONFIG_X86_KERNEL_IBT */
38		43
39	#endif /* CONFIG_CFI_CLANG */	44	#endif /* CONFIG_CFI_CLANG */
40		45
		46	#ifndef __CFI_ADDRESSABLE
		47	#define __CFI_ADDRESSABLE(fn, __attr)
		48	#endif
		49
41	#endif /* _LINUX_CFI_H */	50	#endif /* _LINUX_CFI_H */

include/linux/compiler.h+0−7

 })
 #define annotate_unreachable() __annotate_unreachable(__COUNTER__)
-#define ASM_REACHABLE							\
-	"998:\n\t"							\
-	".pushsection .discard.reachable\n\t"				\
-	".long 998b - .\n\t"						\
-	".popsection\n\t"
 /* Annotate a C jump table to allow objtool to follow the code flow */
 #define __annotate_jump_table __section(".rodata..c_jump_table")
 #else
 #define annotate_unreachable()
-# define ASM_REACHABLE
 #define __annotate_jump_table
 #endif

include/linux/cred.h+1−1

@@ -176,7 +176,7 @@ extern int set_cred_ucounts(struct cred *); · 175 unmodified lines
176	* check for validity of credentials	176	* check for validity of credentials
177	*/	177	*/
178	#ifdef CONFIG_DEBUG_CREDENTIALS	178	#ifdef CONFIG_DEBUG_CREDENTIALS
179	extern void __invalid_creds(const struct cred , const char , unsigned);	179	extern void __noreturn __invalid_creds(const struct cred , const char , unsigned);
180	extern void __validate_process_creds(struct task_struct *,	180	extern void __validate_process_creds(struct task_struct *,
181	const char *, unsigned);	181	const char *, unsigned);
182		182

include/linux/kprobes.h+2−1

 extern void kprobes_inc_nmissed_count(struct kprobe *p);
 extern bool arch_within_kprobe_blacklist(unsigned long addr);
 extern int arch_populate_kprobe_blacklist(void);
-extern bool arch_kprobe_on_func_entry(unsigned long offset);
 extern int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset);
 extern bool within_kprobe_blacklist(unsigned long addr);
 }
 kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset);
+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, bool *on_func_entry);
 int register_kprobe(struct kprobe *p);
 void unregister_kprobe(struct kprobe *p);
 int register_kprobes(struct kprobe **kps, int num);

include/linux/objtool.h+32−0

@@ -77,6 +77,18 @@ struct unwind_hint { · 76 unmodified lines
77	#define STACK_FRAME_NON_STANDARD_FP(func)	77	#define STACK_FRAME_NON_STANDARD_FP(func)
78	#endif	78	#endif
79		79
		80	#define ANNOTATE_NOENDBR \
		81	"986: \n\t" \
		82	".pushsection .discard.noendbr\n\t" \
		83	_ASM_PTR " 986b\n\t" \
		84	".popsection\n\t"
		85
		86	#define ASM_REACHABLE \
		87	"998:\n\t" \
		88	".pushsection .discard.reachable\n\t" \
		89	".long 998b - .\n\t" \
		90	".popsection\n\t"
		91
80	#else /* __ASSEMBLY__ */	92	#else /* __ASSEMBLY__ */
81		93
82	/*	94	/*
@@ -129,6 +141,20 @@ struct unwind_hint { · 46 unmodified lines
129	.popsection	141	.popsection
130	.endm	142	.endm
131		143
		144	.macro ANNOTATE_NOENDBR
		145	.Lhere_\@:
		146	.pushsection .discard.noendbr
		147	.quad .Lhere_\@
		148	.popsection
		149	.endm
		150
		151	.macro REACHABLE
		152	.Lhere_\@:
		153	.pushsection .discard.reachable
		154	.long .Lhere_\@ - .
		155	.popsection
		156	.endm
		157
132	#endif /* __ASSEMBLY__ */	158	#endif /* __ASSEMBLY__ */
133		159
134	#else /* !CONFIG_STACK_VALIDATION */	160	#else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +165,18 @@ struct unwind_hint { · 4 unmodified lines
139	"\n\t"	165	"\n\t"
140	#define STACK_FRAME_NON_STANDARD(func)	166	#define STACK_FRAME_NON_STANDARD(func)
141	#define STACK_FRAME_NON_STANDARD_FP(func)	167	#define STACK_FRAME_NON_STANDARD_FP(func)
		168	#define ANNOTATE_NOENDBR
		169	#define ASM_REACHABLE
142	#else	170	#else
143	#define ANNOTATE_INTRA_FUNCTION_CALL	171	#define ANNOTATE_INTRA_FUNCTION_CALL
144	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0	172	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0
145	.endm	173	.endm
146	.macro STACK_FRAME_NON_STANDARD func:req	174	.macro STACK_FRAME_NON_STANDARD func:req
147	.endm	175	.endm
		176	.macro ANNOTATE_NOENDBR
		177	.endm
		178	.macro REACHABLE
		179	.endm
148	#endif	180	#endif
149		181
150	#endif /* CONFIG_STACK_VALIDATION */	182	#endif /* CONFIG_STACK_VALIDATION */

include/linux/sched/task.h+1−1

@@ -79,7 +79,7 @@ static inline void exit_thread(struct task_struct *tsk) · 78 unmodified lines
79	{	79	{
80	}	80	}
81	#endif	81	#endif
82	extern void do_group_exit(int);	82	extern __noreturn void do_group_exit(int);
83		83
84	extern void exit_files(struct task_struct *);	84	extern void exit_files(struct task_struct *);
85	extern void exit_itimers(struct signal_struct *);	85	extern void exit_itimers(struct signal_struct *);

kernel/bpf/trampoline.c+4−16

 	tr->mod = NULL;
 }
-static int is_ftrace_location(void *ip)
-{
-	long addr;
-	addr = ftrace_location((long)ip);
-	if (!addr)
-		return 0;
-	if (WARN_ON_ONCE(addr != (long)ip))
-		return -EFAULT;
-	return 1;
-}
 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
 {
 	void *ip = tr->func.addr;
 static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
 {
 	void *ip = tr->func.addr;
+	unsigned long faddr;
 	int ret;
-	ret = is_ftrace_location(ip);
+	faddr = ftrace_location((unsigned long)ip);
-	if (ret < 0)
+	if (faddr)
-		return ret;
+		tr->func.ftrace_managed = true;
-	tr->func.ftrace_managed = ret;
 	if (bpf_trampoline_module_get(tr))
 		return -ENOENT;

kernel/cred.c+1−1

@@ -870,7 +870,7 @@ static void dump_invalid_creds(const struct cred cred, const char label, · 869 unmodified lines
870	/*	870	/*
871	* report use of invalid credentials	871	* report use of invalid credentials
872	*/	872	*/
873	void __invalid_creds(const struct cred cred, const char file, unsigned line)	873	void __noreturn __invalid_creds(const struct cred cred, const char file, unsigned line)
874	{	874	{
875	printk(KERN_ERR "CRED: Invalid credentials\n");	875	printk(KERN_ERR "CRED: Invalid credentials\n");
876	printk(KERN_ERR "CRED: At %s:%u\n", file, line);	876	printk(KERN_ERR "CRED: At %s:%u\n", file, line);

kernel/exit.c+1−1

@@ -895,7 +895,7 @@ SYSCALL_DEFINE1(exit, int, error_code) · 894 unmodified lines
895	* Take down every thread in the group. This is called by fatal signals	895	* Take down every thread in the group. This is called by fatal signals
896	* as well as by sys_exit_group (below).	896	* as well as by sys_exit_group (below).
897	*/	897	*/
898	void	898	void __noreturn
899	do_group_exit(int exit_code)	899	do_group_exit(int exit_code)
900	{	900	{
901	struct signal_struct *sig = current->signal;	901	struct signal_struct *sig = current->signal;

kernel/kprobes.c+54−20

 	return false;
 }
+/*
+ * arch_adjust_kprobe_addr - adjust the address
+ * @addr: symbol base address
+ * @offset: offset within the symbol
+ * @on_func_entry: was this @addr+@offset on the function entry
+ *
+ * Typically returns @addr + @offset, except for special cases where the
+ * function might be prefixed by a CFI landing pad, in that case any offset
+ * inside the landing pad is mapped to the first 'real' instruction of the
+ * symbol.
+ *
+ * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
+ * instruction at +0.
+ */
+kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
+						unsigned long offset,
+						bool *on_func_entry)
+{
+	*on_func_entry = !offset;
+	return (kprobe_opcode_t *)(addr + offset);
+}
 /*
  * If 'symbol_name' is specified, look it up and add the 'offset'
  * to it. This way, we can specify a relative address to a symbol.
  * This returns encoded errors if it fails to look up symbol or invalid
  * combination of parameters.
  */
-static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
+static kprobe_opcode_t *
-			const char *symbol_name, unsigned int offset)
+_kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name,
+	     unsigned long offset, bool *on_func_entry)
 {
 	if ((symbol_name && addr) || (!symbol_name && !addr))
 		goto invalid;
 	if (symbol_name) {
+		/*
+		 * Input: @sym + @offset
+		 * Output: @addr + @offset
+		 *
+		 * NOTE: kprobe_lookup_name() does *NOT* fold the offset
+		 *       argument into it's output!
+		 */
 		addr = kprobe_lookup_name(symbol_name, offset);
 		if (!addr)
 			return ERR_PTR(-ENOENT);
 	}
-	addr = (kprobe_opcode_t *)(((char *)addr) + offset);
+	/*
+	 * So here we have @addr + @offset, displace it into a new
+	 * @addr' + @offset' where @addr' is the symbol start address.
+	 */
+	addr = (void *)addr + offset;
+	if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
+		return ERR_PTR(-ENOENT);
+	addr = (void *)addr - offset;
+	/*
+	 * Then ask the architecture to re-combine them, taking care of
+	 * magical function entry details while telling us if this was indeed
+	 * at the start of the function.
+	 */
+	addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
 	if (addr)
 		return addr;
 static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
 {
-	return _kprobe_addr(p->addr, p->symbol_name, p->offset);
+	bool on_func_entry;
+	return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
 }
 /*
 static int check_ftrace_location(struct kprobe *p)
 {
-	unsigned long ftrace_addr;
+	unsigned long addr = (unsigned long)p->addr;
-	ftrace_addr = ftrace_location((unsigned long)p->addr);
+	if (ftrace_location(addr) == addr) {
-	if (ftrace_addr) {
 #ifdef CONFIG_KPROBES_ON_FTRACE
-		/* Given address is not on the instruction boundary */
-		if ((unsigned long)p->addr != ftrace_addr)
-			return -EILSEQ;
 		p->flags |= KPROBE_FLAG_FTRACE;
 #else	/* !CONFIG_KPROBES_ON_FTRACE */
 		return -EINVAL;
 }
 NOKPROBE_SYMBOL(pre_handler_kretprobe);
-bool __weak arch_kprobe_on_func_entry(unsigned long offset)
-{
-	return !offset;
-}
 /**
  * kprobe_on_func_entry() -- check whether given address is function entry
  * @addr: Target address
  */
 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
 {
-	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
+	bool on_func_entry;
+	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
 	if (IS_ERR(kp_addr))
 		return PTR_ERR(kp_addr);
-	if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset))
+	if (!on_func_entry)
-		return -ENOENT;
-	if (!arch_kprobe_on_func_entry(offset))
 		return -EINVAL;
 	return 0;

kernel/livepatch/patch.c+2−17

 	ftrace_test_recursion_unlock(bit);
 }
-/*
- * Convert a function address into the appropriate ftrace location.
- *
- * Usually this is just the address of the function, but on some architectures
- * it's more complicated so allow them to provide a custom behaviour.
- */
-#ifndef klp_get_ftrace_location
-static unsigned long klp_get_ftrace_location(unsigned long faddr)
-{
-	return faddr;
-}
-#endif
 static void klp_unpatch_func(struct klp_func *func)
 {
 	struct klp_ops *ops;
 	if (list_is_singular(&ops->func_stack)) {
 		unsigned long ftrace_loc;
-		ftrace_loc =
+		ftrace_loc = ftrace_location((unsigned long)func->old_func);
-			klp_get_ftrace_location((unsigned long)func->old_func);
 		if (WARN_ON(!ftrace_loc))
 			return;
 	if (!ops) {
 		unsigned long ftrace_loc;
-		ftrace_loc =
+		ftrace_loc = ftrace_location((unsigned long)func->old_func);
-			klp_get_ftrace_location((unsigned long)func->old_func);
 		if (!ftrace_loc) {
 			pr_err("failed to find location for function '%s'\n",
 				func->old_name);

kernel/trace/ftrace.c+40−8

 }
 /**
- * ftrace_location - return true if the ip giving is a traced location
+ * ftrace_location - return the ftrace location
  * @ip: the instruction pointer to check
  *
- * Returns rec->ip if @ip given is a pointer to a ftrace location.
+ * If @ip matches the ftrace location, return @ip.
- * That is, the instruction that is either a NOP or call to
+ * If @ip matches sym+0, return sym's ftrace location.
- * the function tracer. It checks the ftrace internal tables to
+ * Otherwise, return 0.
- * determine if the address belongs or not.
  */
 unsigned long ftrace_location(unsigned long ip)
 {
-	return ftrace_location_range(ip, ip);
+	struct dyn_ftrace *rec;
+	unsigned long offset;
+	unsigned long size;
+	rec = lookup_rec(ip, ip);
+	if (!rec) {
+		if (!kallsyms_lookup_size_offset(ip, &size, &offset))
+			goto out;
+		/* map sym+0 to __fentry__ */
+		if (!offset)
+			rec = lookup_rec(ip, ip + size - 1);
+	}
+	if (rec)
+		return rec->ip;
+out:
+	return 0;
 }
 /**
 {
 	struct ftrace_func_entry *entry;
-	if (!ftrace_location(ip))
+	ip = ftrace_location(ip);
+	if (!ip)
 		return -EINVAL;
 	if (remove) {
 	struct ftrace_func_entry *entry;
 	struct ftrace_hash *free_hash = NULL;
 	struct dyn_ftrace *rec;
-	int ret = -EBUSY;
+	int ret = -ENODEV;
 	mutex_lock(&direct_mutex);
+	ip = ftrace_location(ip);
+	if (!ip)
+		goto out_unlock;
 	/* See if there's a direct function at @ip already */
+	ret = -EBUSY;
 	if (ftrace_find_rec_direct(ip))
 		goto out_unlock;
 	mutex_lock(&direct_mutex);
+	ip = ftrace_location(ip);
+	if (!ip)
+		goto out_unlock;
 	entry = find_direct_entry(&ip, NULL);
 	if (!entry)
 		goto out_unlock;
 	mutex_lock(&direct_mutex);
 	mutex_lock(&ftrace_lock);
+	ip = ftrace_location(ip);
+	if (!ip)
+		goto out_unlock;
 	entry = find_direct_entry(&ip, &rec);
 	if (!entry)
 		goto out_unlock;

samples/ftrace/ftrace-direct-modify.c+5−0

 #ifdef CONFIG_X86_64
+#include <asm/ibt.h>
 asm (
 "	.pushsection    .text, \"ax\", @progbits\n"
 "	.type		my_tramp1, @function\n"
 "	.globl		my_tramp1\n"
 "   my_tramp1:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	call my_direct_func1\n"
 "	leave\n"
 "	.size		my_tramp1, .-my_tramp1\n"
 	ASM_RET
 "	.type		my_tramp2, @function\n"
 "	.globl		my_tramp2\n"
 "   my_tramp2:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	call my_direct_func2\n"

samples/ftrace/ftrace-direct-multi-modify.c+7−3

 #ifdef CONFIG_X86_64
+#include <asm/ibt.h>
 asm (
 "	.pushsection    .text, \"ax\", @progbits\n"
 "	.type		my_tramp1, @function\n"
 "	.globl		my_tramp1\n"
 "   my_tramp1:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	pushq %rdi\n"
 "	call my_direct_func1\n"
 "	popq %rdi\n"
 "	leave\n"
-"	ret\n"
+	ASM_RET
 "	.size		my_tramp1, .-my_tramp1\n"
 "	.type		my_tramp2, @function\n"
-"\n"
 "	.globl		my_tramp2\n"
 "   my_tramp2:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	pushq %rdi\n"
 "	call my_direct_func2\n"
 "	popq %rdi\n"
 "	leave\n"
-"	ret\n"
+	ASM_RET
 "	.size		my_tramp2, .-my_tramp2\n"
 "	.popsection\n"
 );

samples/ftrace/ftrace-direct-multi.c+4−1

 #ifdef CONFIG_X86_64
+#include <asm/ibt.h>
 asm (
 "	.pushsection    .text, \"ax\", @progbits\n"
 "	.type		my_tramp, @function\n"
 "	.globl		my_tramp\n"
 "   my_tramp:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	pushq %rdi\n"
 "	call my_direct_func\n"
 "	popq %rdi\n"
 "	leave\n"
-"	ret\n"
+	ASM_RET
 "	.size		my_tramp, .-my_tramp\n"
 "	.popsection\n"
 );

samples/ftrace/ftrace-direct-too.c+3−0

 #ifdef CONFIG_X86_64
+#include <asm/ibt.h>
 asm (
 "	.pushsection    .text, \"ax\", @progbits\n"
 "	.type		my_tramp, @function\n"
 "	.globl		my_tramp\n"
 "   my_tramp:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	pushq %rdi\n"

samples/ftrace/ftrace-direct.c+3−0

 #ifdef CONFIG_X86_64
+#include <asm/ibt.h>
 asm (
 "	.pushsection    .text, \"ax\", @progbits\n"
 "	.type		my_tramp, @function\n"
 "	.globl		my_tramp\n"
 "   my_tramp:"
+	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
 "	pushq %rdi\n"

scripts/Makefile.build+14−13

 targets-for-modules := $(patsubst %.o, %.mod, $(filter %.o, $(obj-m)))
-ifdef CONFIG_LTO_CLANG
+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
-targets-for-modules += $(patsubst %.o, %.lto.o, $(filter %.o, $(obj-m)))
+targets-for-modules += $(patsubst %.o, %.prelink.o, $(filter %.o, $(obj-m)))
 endif
 ifdef need-modorder
 objtool_args =								\
 	$(if $(CONFIG_UNWINDER_ORC),orc generate,check)			\
 	$(if $(part-of-module), --module)				\
+	$(if $(CONFIG_X86_KERNEL_IBT), --lto --ibt)			\
 	$(if $(CONFIG_FRAME_POINTER),, --no-fp)				\
 	$(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
 	$(if $(CONFIG_RETPOLINE), --retpoline)				\
 endif # CONFIG_STACK_VALIDATION
-ifdef CONFIG_LTO_CLANG
+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
 # Skip objtool for LLVM bitcode
 $(obj)/%.o: objtool-enabled :=
 	$(call if_changed_rule,cc_o_c)
 	$(call cmd,force_checksrc)
-ifdef CONFIG_LTO_CLANG
+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
 # Module .o files may contain LLVM bitcode, compile them into native code
 # before ELF processing
-quiet_cmd_cc_lto_link_modules = LTO [M] $@
+quiet_cmd_cc_prelink_modules = LD [M]  $@
-cmd_cc_lto_link_modules =						\
+      cmd_cc_prelink_modules =						\
 	$(LD) $(ld_flags) -r -o $@					\
-		$(shell [ -s $(@:.lto.o=.o.symversions) ] &&		\
+               $(shell [ -s $(@:.prelink.o=.o.symversions) ] &&		\
-			echo -T $(@:.lto.o=.o.symversions))		\
+                       echo -T $(@:.prelink.o=.o.symversions))		\
 		--whole-archive $(filter-out FORCE,$^)			\
 		$(cmd_objtool)
 # objtool was skipped for LLVM bitcode, run it now that we have compiled
 # modules into native code
-$(obj)/%.lto.o: objtool-enabled = y
+$(obj)/%.prelink.o: objtool-enabled = y
-$(obj)/%.lto.o: part-of-module := y
+$(obj)/%.prelink.o: part-of-module := y
-$(obj)/%.lto.o: $(obj)/%.o FORCE
+$(obj)/%.prelink.o: $(obj)/%.o FORCE
-	$(call if_changed,cc_lto_link_modules)
+	$(call if_changed,cc_prelink_modules)
 endif
 cmd_mod = { \
 # Do not replace $(filter %.o,^) with $(real-prereqs). When a single object
 # module is turned into a multi object module, $^ will contain header file
 # dependencies recorded in the .*.cmd file.
-ifdef CONFIG_LTO_CLANG
+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
 quiet_cmd_link_multi-m = AR [M]  $@
 cmd_link_multi-m =						\
 	$(cmd_update_lto_symversions);				\

scripts/Makefile.lib+2−2

@@ -225,11 +225,11 @@ dtc_cpp_flags = -Wp,-MMD,$(depfile).pre.tmp -nostdinc \ · 224 unmodified lines
225	$(addprefix -I,$(DTC_INCLUDE)) \	225	$(addprefix -I,$(DTC_INCLUDE)) \
226	-undef -D__DTS__	226	-undef -D__DTS__
227		227
228	ifeq ($(CONFIG_LTO_CLANG),y)	228	ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
229	# With CONFIG_LTO_CLANG, .o files in modules might be LLVM bitcode, so we	229	# With CONFIG_LTO_CLANG, .o files in modules might be LLVM bitcode, so we
230	# need to run LTO to compile them into native code (.lto.o) before further	230	# need to run LTO to compile them into native code (.lto.o) before further
231	# processing.	231	# processing.
232	mod-prelink-ext := .lto	232	mod-prelink-ext := .prelink
233	endif	233	endif
234		234
235	# Useful for describing the dependency of composite objects	235	# Useful for describing the dependency of composite objects

scripts/link-vmlinux.sh+9−3

 	local objtoolcmd;
 	local objtoolopt;
-	if is_enabled CONFIG_LTO_CLANG && is_enabled CONFIG_STACK_VALIDATION; then
+	if is_enabled CONFIG_STACK_VALIDATION && \
+	   ( is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT ); then
 		# Don't perform vmlinux validation unless explicitly requested,
 		# but run objtool on vmlinux.o now that we have an object file.
 		if is_enabled CONFIG_UNWINDER_ORC; then
 			objtoolcmd="orc generate"
 		fi
-		objtoolopt="${objtoolopt} --duplicate"
+		objtoolopt="${objtoolopt} --lto"
+		if is_enabled CONFIG_X86_KERNEL_IBT; then
+			objtoolopt="${objtoolopt} --ibt"
+		fi
 		if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then
 			objtoolopt="${objtoolopt} --mcount"
 	# skip output file argument
 	shift
-	if is_enabled CONFIG_LTO_CLANG; then
+	if is_enabled CONFIG_LTO_CLANG || is_enabled CONFIG_X86_KERNEL_IBT; then
 		# Use vmlinux.o instead of performing the slow LTO link again.
 		objs=vmlinux.o
 		libs=

scripts/mod/modpost.c+6−6

@@ -1996,9 +1996,9 @@ static char remove_dot(char s) · 1995 unmodified lines
1996	if (m && (s[n + m] == '.' \|\| s[n + m] == 0))	1996	if (m && (s[n + m] == '.' \|\| s[n + m] == 0))
1997	s[n] = 0;	1997	s[n] = 0;
1998		1998
1999	/* strip trailing .lto */	1999	/* strip trailing .prelink */
2000	if (strends(s, ".lto"))	2000	if (strends(s, ".prelink"))
2001	s[strlen(s) - 4] = '\0';	2001	s[strlen(s) - 8] = '\0';
2002	}	2002	}
2003	return s;	2003	return s;
2004	}	2004	}
@@ -2022,9 +2022,9 @@ static void read_symbols(const char *modname) · 17 unmodified lines
2022	/* strip trailing .o */	2022	/* strip trailing .o */
2023	tmp = NOFAIL(strdup(modname));	2023	tmp = NOFAIL(strdup(modname));
2024	tmp[strlen(tmp) - 2] = '\0';	2024	tmp[strlen(tmp) - 2] = '\0';
2025	/* strip trailing .lto */	2025	/* strip trailing .prelink */
2026	if (strends(tmp, ".lto"))	2026	if (strends(tmp, ".prelink"))
2027	tmp[strlen(tmp) - 4] = '\0';	2027	tmp[strlen(tmp) - 8] = '\0';
2028	mod = new_module(tmp);	2028	mod = new_module(tmp);
2029	free(tmp);	2029	free(tmp);
2030	}	2030	}

sound/core/control_compat.c+8−8

@@ -150,7 +150,7 @@ struct snd_ctl_elem_value32 { · 149 unmodified lines
150	unsigned char reserved[128];	150	unsigned char reserved[128];
151	};	151	};
152		152
153	#ifdef CONFIG_X86_X32	153	#ifdef CONFIG_X86_X32_ABI
154	/* x32 has a different alignment for 64bit values from ia32 */	154	/* x32 has a different alignment for 64bit values from ia32 */
155	struct snd_ctl_elem_value_x32 {	155	struct snd_ctl_elem_value_x32 {
156	struct snd_ctl_elem_id id;	156	struct snd_ctl_elem_id id;
@@ -162,7 +162,7 @@ struct snd_ctl_elem_value_x32 { · 5 unmodified lines
162	} value;	162	} value;
163	unsigned char reserved[128];	163	unsigned char reserved[128];
164	};	164	};
165	#endif /* CONFIG_X86_X32 */	165	#endif /* CONFIG_X86_X32_ABI */
166		166
167	/* get the value type and count of the control */	167	/* get the value type and count of the control */
168	static int get_ctl_type(struct snd_card card, struct snd_ctl_elem_id id,	168	static int get_ctl_type(struct snd_card card, struct snd_ctl_elem_id id,
@@ -347,7 +347,7 @@ static int snd_ctl_elem_write_user_compat(struct snd_ctl_file *file, · 178 unmodified lines
347	return ctl_elem_write_user(file, data32, &data32->value);	347	return ctl_elem_write_user(file, data32, &data32->value);
348	}	348	}
349		349
350	#ifdef CONFIG_X86_X32	350	#ifdef CONFIG_X86_X32_ABI
351	static int snd_ctl_elem_read_user_x32(struct snd_card *card,	351	static int snd_ctl_elem_read_user_x32(struct snd_card *card,
352	struct snd_ctl_elem_value_x32 __user *data32)	352	struct snd_ctl_elem_value_x32 __user *data32)
353	{	353	{
@@ -359,7 +359,7 @@ static int snd_ctl_elem_write_user_x32(struct snd_ctl_file *file, · 5 unmodified lines
359	{	359	{
360	return ctl_elem_write_user(file, data32, &data32->value);	360	return ctl_elem_write_user(file, data32, &data32->value);
361	}	361	}
362	#endif /* CONFIG_X86_X32 */	362	#endif /* CONFIG_X86_X32_ABI */
363		363
364	/* add or replace a user control */	364	/* add or replace a user control */
365	static int snd_ctl_elem_add_compat(struct snd_ctl_file *file,	365	static int snd_ctl_elem_add_compat(struct snd_ctl_file *file,
@@ -418,10 +418,10 @@ enum { · 52 unmodified lines
418	SNDRV_CTL_IOCTL_ELEM_WRITE32 = _IOWR('U', 0x13, struct snd_ctl_elem_value32),	418	SNDRV_CTL_IOCTL_ELEM_WRITE32 = _IOWR('U', 0x13, struct snd_ctl_elem_value32),
419	SNDRV_CTL_IOCTL_ELEM_ADD32 = _IOWR('U', 0x17, struct snd_ctl_elem_info32),	419	SNDRV_CTL_IOCTL_ELEM_ADD32 = _IOWR('U', 0x17, struct snd_ctl_elem_info32),
420	SNDRV_CTL_IOCTL_ELEM_REPLACE32 = _IOWR('U', 0x18, struct snd_ctl_elem_info32),	420	SNDRV_CTL_IOCTL_ELEM_REPLACE32 = _IOWR('U', 0x18, struct snd_ctl_elem_info32),
421	#ifdef CONFIG_X86_X32	421	#ifdef CONFIG_X86_X32_ABI
422	SNDRV_CTL_IOCTL_ELEM_READ_X32 = _IOWR('U', 0x12, struct snd_ctl_elem_value_x32),	422	SNDRV_CTL_IOCTL_ELEM_READ_X32 = _IOWR('U', 0x12, struct snd_ctl_elem_value_x32),
423	SNDRV_CTL_IOCTL_ELEM_WRITE_X32 = _IOWR('U', 0x13, struct snd_ctl_elem_value_x32),	423	SNDRV_CTL_IOCTL_ELEM_WRITE_X32 = _IOWR('U', 0x13, struct snd_ctl_elem_value_x32),
424	#endif /* CONFIG_X86_X32 */	424	#endif /* CONFIG_X86_X32_ABI */
425	};	425	};
426		426
427	static inline long snd_ctl_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)	427	static inline long snd_ctl_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)
@@ -460,12 +460,12 @@ static inline long snd_ctl_ioctl_compat(struct file *file, unsigned int cmd, uns · 32 unmodified lines
460	return snd_ctl_elem_add_compat(ctl, argp, 0);	460	return snd_ctl_elem_add_compat(ctl, argp, 0);
461	case SNDRV_CTL_IOCTL_ELEM_REPLACE32:	461	case SNDRV_CTL_IOCTL_ELEM_REPLACE32:
462	return snd_ctl_elem_add_compat(ctl, argp, 1);	462	return snd_ctl_elem_add_compat(ctl, argp, 1);
463	#ifdef CONFIG_X86_X32	463	#ifdef CONFIG_X86_X32_ABI
464	case SNDRV_CTL_IOCTL_ELEM_READ_X32:	464	case SNDRV_CTL_IOCTL_ELEM_READ_X32:
465	return snd_ctl_elem_read_user_x32(ctl->card, argp);	465	return snd_ctl_elem_read_user_x32(ctl->card, argp);
466	case SNDRV_CTL_IOCTL_ELEM_WRITE_X32:	466	case SNDRV_CTL_IOCTL_ELEM_WRITE_X32:
467	return snd_ctl_elem_write_user_x32(ctl, argp);	467	return snd_ctl_elem_write_user_x32(ctl, argp);
468	#endif /* CONFIG_X86_X32 */	468	#endif /* CONFIG_X86_X32_ABI */
469	}	469	}
470		470
471	down_read(&snd_ioctl_rwsem);	471	down_read(&snd_ioctl_rwsem);

sound/core/pcm_compat.c+10−10

@@ -147,13 +147,13 @@ static int snd_pcm_ioctl_channel_info_compat(struct snd_pcm_substream *substream · 146 unmodified lines
147	return err;	147	return err;
148	}	148	}
149		149
150	#ifdef CONFIG_X86_X32	150	#ifdef CONFIG_X86_X32_ABI
151	/* X32 ABI has the same struct as x86-64 for snd_pcm_channel_info */	151	/* X32 ABI has the same struct as x86-64 for snd_pcm_channel_info */
152	static int snd_pcm_channel_info_user(struct snd_pcm_substream *substream,	152	static int snd_pcm_channel_info_user(struct snd_pcm_substream *substream,
153	struct snd_pcm_channel_info __user *src);	153	struct snd_pcm_channel_info __user *src);
154	#define snd_pcm_ioctl_channel_info_x32(s, p) \	154	#define snd_pcm_ioctl_channel_info_x32(s, p) \
155	snd_pcm_channel_info_user(s, p)	155	snd_pcm_channel_info_user(s, p)
156	#endif /* CONFIG_X86_X32 */	156	#endif /* CONFIG_X86_X32_ABI */
157		157
158	struct compat_snd_pcm_status64 {	158	struct compat_snd_pcm_status64 {
159	snd_pcm_state_t state;	159	snd_pcm_state_t state;
@@ -375,7 +375,7 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, · 215 unmodified lines
375	return err;	375	return err;
376	}	376	}
377		377
378	#ifdef CONFIG_X86_X32	378	#ifdef CONFIG_X86_X32_ABI
379	/* X32 ABI has 64bit timespec and 64bit alignment */	379	/* X32 ABI has 64bit timespec and 64bit alignment */
380	struct snd_pcm_mmap_status_x32 {	380	struct snd_pcm_mmap_status_x32 {
381	snd_pcm_state_t state;	381	snd_pcm_state_t state;
@@ -468,7 +468,7 @@ static int snd_pcm_ioctl_sync_ptr_x32(struct snd_pcm_substream *substream, · 86 unmodified lines
468		468
469	return 0;	469	return 0;
470	}	470	}
471	#endif /* CONFIG_X86_X32 */	471	#endif /* CONFIG_X86_X32_ABI */
472		472
473	#ifdef __BIG_ENDIAN	473	#ifdef __BIG_ENDIAN
474	typedef char __pad_before_u32[4];	474	typedef char __pad_before_u32[4];
@@ -560,10 +560,10 @@ enum { · 85 unmodified lines
560	SNDRV_PCM_IOCTL_READN_FRAMES32 = _IOR('A', 0x53, struct snd_xfern32),	560	SNDRV_PCM_IOCTL_READN_FRAMES32 = _IOR('A', 0x53, struct snd_xfern32),
561	SNDRV_PCM_IOCTL_STATUS_COMPAT64 = _IOR('A', 0x20, struct compat_snd_pcm_status64),	561	SNDRV_PCM_IOCTL_STATUS_COMPAT64 = _IOR('A', 0x20, struct compat_snd_pcm_status64),
562	SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64 = _IOWR('A', 0x24, struct compat_snd_pcm_status64),	562	SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64 = _IOWR('A', 0x24, struct compat_snd_pcm_status64),
563	#ifdef CONFIG_X86_X32	563	#ifdef CONFIG_X86_X32_ABI
564	SNDRV_PCM_IOCTL_CHANNEL_INFO_X32 = _IOR('A', 0x32, struct snd_pcm_channel_info),	564	SNDRV_PCM_IOCTL_CHANNEL_INFO_X32 = _IOR('A', 0x32, struct snd_pcm_channel_info),
565	SNDRV_PCM_IOCTL_SYNC_PTR_X32 = _IOWR('A', 0x23, struct snd_pcm_sync_ptr_x32),	565	SNDRV_PCM_IOCTL_SYNC_PTR_X32 = _IOWR('A', 0x23, struct snd_pcm_sync_ptr_x32),
566	#endif /* CONFIG_X86_X32 */	566	#endif /* CONFIG_X86_X32_ABI */
567	};	567	};
568		568
569	static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)	569	static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)
@@ -607,10 +607,10 @@ static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned l · 37 unmodified lines
607	case __SNDRV_PCM_IOCTL_SYNC_PTR32:	607	case __SNDRV_PCM_IOCTL_SYNC_PTR32:
608	return snd_pcm_common_ioctl(file, substream, cmd, argp);	608	return snd_pcm_common_ioctl(file, substream, cmd, argp);
609	case __SNDRV_PCM_IOCTL_SYNC_PTR64:	609	case __SNDRV_PCM_IOCTL_SYNC_PTR64:
610	#ifdef CONFIG_X86_X32	610	#ifdef CONFIG_X86_X32_ABI
611	if (in_x32_syscall())	611	if (in_x32_syscall())
612	return snd_pcm_ioctl_sync_ptr_x32(substream, argp);	612	return snd_pcm_ioctl_sync_ptr_x32(substream, argp);
613	#endif /* CONFIG_X86_X32 */	613	#endif /* CONFIG_X86_X32_ABI */
614	return snd_pcm_ioctl_sync_ptr_buggy(substream, argp);	614	return snd_pcm_ioctl_sync_ptr_buggy(substream, argp);
615	case SNDRV_PCM_IOCTL_HW_REFINE32:	615	case SNDRV_PCM_IOCTL_HW_REFINE32:
616	return snd_pcm_ioctl_hw_params_compat(substream, 1, argp);	616	return snd_pcm_ioctl_hw_params_compat(substream, 1, argp);
@@ -642,10 +642,10 @@ static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned l · 25 unmodified lines
642	return snd_pcm_status_user_compat64(substream, argp, false);	642	return snd_pcm_status_user_compat64(substream, argp, false);
643	case SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64:	643	case SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64:
644	return snd_pcm_status_user_compat64(substream, argp, true);	644	return snd_pcm_status_user_compat64(substream, argp, true);
645	#ifdef CONFIG_X86_X32	645	#ifdef CONFIG_X86_X32_ABI
646	case SNDRV_PCM_IOCTL_CHANNEL_INFO_X32:	646	case SNDRV_PCM_IOCTL_CHANNEL_INFO_X32:
647	return snd_pcm_ioctl_channel_info_x32(substream, argp);	647	return snd_pcm_ioctl_channel_info_x32(substream, argp);
648	#endif /* CONFIG_X86_X32 */	648	#endif /* CONFIG_X86_X32_ABI */
649	}	649	}
650		650
651	return -ENOIOCTLCMD;	651	return -ENOIOCTLCMD;

tools/include/linux/objtool.h+32−0

@@ -77,6 +77,18 @@ struct unwind_hint { · 76 unmodified lines
77	#define STACK_FRAME_NON_STANDARD_FP(func)	77	#define STACK_FRAME_NON_STANDARD_FP(func)
78	#endif	78	#endif
79		79
		80	#define ANNOTATE_NOENDBR \
		81	"986: \n\t" \
		82	".pushsection .discard.noendbr\n\t" \
		83	_ASM_PTR " 986b\n\t" \
		84	".popsection\n\t"
		85
		86	#define ASM_REACHABLE \
		87	"998:\n\t" \
		88	".pushsection .discard.reachable\n\t" \
		89	".long 998b - .\n\t" \
		90	".popsection\n\t"
		91
80	#else /* __ASSEMBLY__ */	92	#else /* __ASSEMBLY__ */
81		93
82	/*	94	/*
@@ -129,6 +141,20 @@ struct unwind_hint { · 46 unmodified lines
129	.popsection	141	.popsection
130	.endm	142	.endm
131		143
		144	.macro ANNOTATE_NOENDBR
		145	.Lhere_\@:
		146	.pushsection .discard.noendbr
		147	.quad .Lhere_\@
		148	.popsection
		149	.endm
		150
		151	.macro REACHABLE
		152	.Lhere_\@:
		153	.pushsection .discard.reachable
		154	.long .Lhere_\@ - .
		155	.popsection
		156	.endm
		157
132	#endif /* __ASSEMBLY__ */	158	#endif /* __ASSEMBLY__ */
133		159
134	#else /* !CONFIG_STACK_VALIDATION */	160	#else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +165,18 @@ struct unwind_hint { · 4 unmodified lines
139	"\n\t"	165	"\n\t"
140	#define STACK_FRAME_NON_STANDARD(func)	166	#define STACK_FRAME_NON_STANDARD(func)
141	#define STACK_FRAME_NON_STANDARD_FP(func)	167	#define STACK_FRAME_NON_STANDARD_FP(func)
		168	#define ANNOTATE_NOENDBR
		169	#define ASM_REACHABLE
142	#else	170	#else
143	#define ANNOTATE_INTRA_FUNCTION_CALL	171	#define ANNOTATE_INTRA_FUNCTION_CALL
144	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0	172	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0
145	.endm	173	.endm
146	.macro STACK_FRAME_NON_STANDARD func:req	174	.macro STACK_FRAME_NON_STANDARD func:req
147	.endm	175	.endm
		176	.macro ANNOTATE_NOENDBR
		177	.endm
		178	.macro REACHABLE
		179	.endm
148	#endif	180	#endif
149		181
150	#endif /* CONFIG_STACK_VALIDATION */	182	#endif /* CONFIG_STACK_VALIDATION */

tools/objtool/arch/x86/decode.c+29−5

 #define rm_is_mem(reg)	(mod_is_mem() && !is_RIP() && rm_is(reg))
 #define rm_is_reg(reg)	(mod_is_reg() && modrm_rm == (reg))
+static bool has_notrack_prefix(struct insn *insn)
+{
+	int i;
+	for (i = 0; i < insn->prefixes.nbytes; i++) {
+		if (insn->prefixes.bytes[i] == 0x3e)
+			return true;
+	}
+	return false;
+}
 int arch_decode_instruction(struct objtool_file *file, const struct section *sec,
 			    unsigned long offset, unsigned int maxlen,
 			    unsigned int *len, enum insn_type *type,
 	const struct elf *elf = file->elf;
 	struct insn insn;
 	int x86_64, ret;
-	unsigned char op1, op2, op3,
+	unsigned char op1, op2, op3, prefix,
 		      rex = 0, rex_b = 0, rex_r = 0, rex_w = 0, rex_x = 0,
 		      modrm = 0, modrm_mod = 0, modrm_rm = 0, modrm_reg = 0,
 		      sib = 0, /* sib_scale = 0, */ sib_index = 0, sib_base = 0;
 	if (insn.vex_prefix.nbytes)
 		return 0;
+	prefix = insn.prefixes.bytes[0];
 	op1 = insn.opcode.bytes[0];
 	op2 = insn.opcode.bytes[1];
 	op3 = insn.opcode.bytes[2];
 			/* nopl/nopw */
 			*type = INSN_NOP;
+		} else if (op2 == 0x1e) {
+			if (prefix == 0xf3 && (modrm == 0xfa || modrm == 0xfb))
+				*type = INSN_ENDBR;
 		} else if (op2 == 0x38 && op3 == 0xf8) {
 			if (insn.prefixes.nbytes == 1 &&
 			    insn.prefixes.bytes[0] == 0xf2) {
 		break;
 	case 0xff:
-		if (modrm_reg == 2 || modrm_reg == 3)
+		if (modrm_reg == 2 || modrm_reg == 3) {
 			*type = INSN_CALL_DYNAMIC;
+			if (has_notrack_prefix(&insn))
+				WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
-		else if (modrm_reg == 4)
+		} else if (modrm_reg == 4) {
 			*type = INSN_JUMP_DYNAMIC;
+			if (has_notrack_prefix(&insn))
+				WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
-		else if (modrm_reg == 5)
+		} else if (modrm_reg == 5) {
 			/* jmpf */
 			*type = INSN_CONTEXT_SWITCH;
-		else if (modrm_reg == 6) {
+		} else if (modrm_reg == 6) {
 			/* push from mem */
 			ADD_OP(op) {

tools/objtool/builtin-check.c+5−2

 #include <objtool/objtool.h>
 bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
-     validate_dup, vmlinux, mcount, noinstr, backup, sls;
+     lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
+     ibt;
 static const char * const check_usage[] = {
 	"objtool check [<options>] file.o",
 	OPT_BOOLEAN('b', "backtrace", &backtrace, "unwind on error"),
 	OPT_BOOLEAN('a', "uaccess", &uaccess, "enable uaccess checking"),
 	OPT_BOOLEAN('s', "stats", &stats, "print statistics"),
-	OPT_BOOLEAN('d', "duplicate", &validate_dup, "duplicate validation for vmlinux.o"),
+	OPT_BOOLEAN(0, "lto", &lto, "whole-archive like runs"),
 	OPT_BOOLEAN('n', "noinstr", &noinstr, "noinstr validation for vmlinux.o"),
 	OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"),
 	OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
 	OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"),
 	OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"),
+	OPT_BOOLEAN(0, "dry-run", &dryrun, "don't write the modifications"),
+	OPT_BOOLEAN(0, "ibt", &ibt, "validate ENDBR placement"),
 	OPT_END(),
 };

tools/objtool/check.c+376−24

@@ -181,6 +181,9 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 180 unmodified lines
181	"kunit_try_catch_throw",	181	"kunit_try_catch_throw",
182	"xen_start_kernel",	182	"xen_start_kernel",
183	"cpu_bringup_and_idle",	183	"cpu_bringup_and_idle",
		184	"do_group_exit",
		185	"stop_this_cpu",
		186	"__invalid_creds",
184	};	187	};
185		188
186	if (!func)	189	if (!func)
@@ -380,6 +383,7 @@ static int decode_instructions(struct objtool_file *file) · 193 unmodified lines
380	memset(insn, 0, sizeof(*insn));	383	memset(insn, 0, sizeof(*insn));
381	INIT_LIST_HEAD(&insn->alts);	384	INIT_LIST_HEAD(&insn->alts);
382	INIT_LIST_HEAD(&insn->stack_ops);	385	INIT_LIST_HEAD(&insn->stack_ops);
		386	INIT_LIST_HEAD(&insn->call_node);
383		387
384	insn->sec = sec;	388	insn->sec = sec;
385	insn->offset = offset;	389	insn->offset = offset;
@@ -392,6 +396,14 @@ static int decode_instructions(struct objtool_file *file) · 6 unmodified lines
392	if (ret)	396	if (ret)
393	goto err;	397	goto err;
394		398
		399	/*
		400	* By default, "ud2" is a dead end unless otherwise
		401	* annotated, because GCC 7 inserts it for certain
		402	* divide-by-zero cases.
		403	*/
		404	if (insn->type == INSN_BUG)
		405	insn->dead_end = true;
		406
395	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));	407	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
396	list_add_tail(&insn->list, &file->insn_list);	408	list_add_tail(&insn->list, &file->insn_list);
397	nr_insns++;	409	nr_insns++;
@@ -407,8 +419,17 @@ static int decode_instructions(struct objtool_file *file) · 9 unmodified lines
407	return -1;	419	return -1;
408	}	420	}
409		421
410	sym_for_each_insn(file, func, insn)	422	sym_for_each_insn(file, func, insn) {
411	insn->func = func;	423	insn->func = func;
		424	if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
		425	if (insn->offset == insn->func->offset) {
		426	list_add_tail(&insn->call_node, &file->endbr_list);
		427	file->nr_endbr++;
		428	} else {
		429	file->nr_endbr_int++;
		430	}
		431	}
		432	}
412	}	433	}
413	}	434	}
414		435
@@ -520,14 +541,6 @@ static int add_dead_ends(struct objtool_file *file) · 105 unmodified lines
520	struct reloc *reloc;	541	struct reloc *reloc;
521	struct instruction *insn;	542	struct instruction *insn;
522		543
523	/*
524	* By default, "ud2" is a dead end unless otherwise annotated, because
525	* GCC 7 inserts it for certain divide-by-zero cases.
526	*/
527	for_each_insn(file, insn)
528	if (insn->type == INSN_BUG)
529	insn->dead_end = true;
530
531	/*	544	/*
532	* Check for manually annotated dead ends.	545	* Check for manually annotated dead ends.
533	*/	546	*/
@@ -731,6 +744,58 @@ static int create_retpoline_sites_sections(struct objtool_file *file) · 197 unmodified lines
731	return 0;	744	return 0;
732	}	745	}
733		746
		747	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
		748	{
		749	struct instruction *insn;
		750	struct section *sec;
		751	int idx;
		752
		753	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
		754	if (sec) {
		755	WARN("file already has .ibt_endbr_seal, skipping");
		756	return 0;
		757	}
		758
		759	idx = 0;
		760	list_for_each_entry(insn, &file->endbr_list, call_node)
		761	idx++;
		762
		763	if (stats) {
		764	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
		765	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
		766	printf("ibt: superfluous ENDBR: %d\n", idx);
		767	}
		768
		769	if (!idx)
		770	return 0;
		771
		772	sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
		773	sizeof(int), idx);
		774	if (!sec) {
		775	WARN("elf_create_section: .ibt_endbr_seal");
		776	return -1;
		777	}
		778
		779	idx = 0;
		780	list_for_each_entry(insn, &file->endbr_list, call_node) {
		781
		782	int site = (int )sec->data->d_buf + idx;
		783	*site = 0;
		784
		785	if (elf_add_reloc_to_insn(file->elf, sec,
		786	idx * sizeof(int),
		787	R_X86_64_PC32,
		788	insn->sec, insn->offset)) {
		789	WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
		790	return -1;
		791	}
		792
		793	idx++;
		794	}
		795
		796	return 0;
		797	}
		798
734	static int create_mcount_loc_sections(struct objtool_file *file)	799	static int create_mcount_loc_sections(struct objtool_file *file)
735	{	800	{
736	struct section *sec;	801	struct section *sec;
@@ -1111,6 +1176,9 @@ static void annotate_call_site(struct objtool_file *file, · 374 unmodified lines
1111	list_add_tail(&insn->call_node, &file->mcount_loc_list);	1176	list_add_tail(&insn->call_node, &file->mcount_loc_list);
1112	return;	1177	return;
1113	}	1178	}
		1179
		1180	if (!sibling && dead_end_function(file, sym))
		1181	insn->dead_end = true;
1114	}	1182	}
1115		1183
1116	static void add_call_dest(struct objtool_file file, struct instruction insn,	1184	static void add_call_dest(struct objtool_file file, struct instruction insn,
@@ -1165,6 +1233,19 @@ static void add_retpoline_call(struct objtool_file file, struct instruction in · 48 unmodified lines
1165		1233
1166	annotate_call_site(file, insn, false);	1234	annotate_call_site(file, insn, false);
1167	}	1235	}
		1236
		1237	static bool same_function(struct instruction insn1, struct instruction insn2)
		1238	{
		1239	return insn1->func->pfunc == insn2->func->pfunc;
		1240	}
		1241
		1242	static bool is_first_func_insn(struct instruction *insn)
		1243	{
		1244	return insn->offset == insn->func->offset \|\|
		1245	(insn->type == INSN_ENDBR &&
		1246	insn->offset == insn->func->offset + insn->len);
		1247	}
		1248
1168	/*	1249	/*
1169	* Find the destination instructions for all jumps.	1250	* Find the destination instructions for all jumps.
1170	*/	1251	*/
@@ -1245,8 +1326,8 @@ static int add_jump_destinations(struct objtool_file *file) · 74 unmodified lines
1245	insn->func->cfunc = insn->jump_dest->func;	1326	insn->func->cfunc = insn->jump_dest->func;
1246	insn->jump_dest->func->pfunc = insn->func;	1327	insn->jump_dest->func->pfunc = insn->func;
1247		1328
1248	} else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&	1329	} else if (!same_function(insn, insn->jump_dest) &&
1249	insn->jump_dest->offset == insn->jump_dest->func->offset) {	1330	is_first_func_insn(insn->jump_dest)) {
1250	/* internal sibling call (without reloc) */	1331	/* internal sibling call (without reloc) */
1251	add_call_dest(file, insn, insn->jump_dest->func, true);	1332	add_call_dest(file, insn, insn->jump_dest->func, true);
1252	}	1333	}
@@ -1836,6 +1917,16 @@ static int read_unwind_hints(struct objtool_file *file) · 583 unmodified lines
1836		1917
1837	insn->hint = true;	1918	insn->hint = true;
1838		1919
		1920	if (ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
		1921	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
		1922
		1923	if (sym && sym->bind == STB_GLOBAL &&
		1924	insn->type != INSN_ENDBR && !insn->noendbr) {
		1925	WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
		1926	insn->sec, insn->offset);
		1927	}
		1928	}
		1929
1839	if (hint->type == UNWIND_HINT_TYPE_FUNC) {	1930	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1840	insn->cfi = &func_cfi;	1931	insn->cfi = &func_cfi;
1841	continue;	1932	continue;
@@ -1860,6 +1951,32 @@ static int read_unwind_hints(struct objtool_file *file) · 18 unmodified lines
1860	return 0;	1951	return 0;
1861	}	1952	}
1862		1953
		1954	static int read_noendbr_hints(struct objtool_file *file)
		1955	{
		1956	struct section *sec;
		1957	struct instruction *insn;
		1958	struct reloc *reloc;
		1959
		1960	sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
		1961	if (!sec)
		1962	return 0;
		1963
		1964	list_for_each_entry(reloc, &sec->reloc_list, list) {
		1965	insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
		1966	if (!insn) {
		1967	WARN("bad .discard.noendbr entry");
		1968	return -1;
		1969	}
		1970
		1971	if (insn->type == INSN_ENDBR)
		1972	WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset);
		1973
		1974	insn->noendbr = 1;
		1975	}
		1976
		1977	return 0;
		1978	}
		1979
1863	static int read_retpoline_hints(struct objtool_file *file)	1980	static int read_retpoline_hints(struct objtool_file *file)
1864	{	1981	{
1865	struct section *sec;	1982	struct section *sec;
@@ -2086,10 +2203,6 @@ static int decode_sections(struct objtool_file *file) · 220 unmodified lines
2086	if (ret)	2203	if (ret)
2087	return ret;	2204	return ret;
2088		2205
2089	ret = add_dead_ends(file);
2090	if (ret)
2091	return ret;
2092
2093	add_ignores(file);	2206	add_ignores(file);
2094	add_uaccess_safe(file);	2207	add_uaccess_safe(file);
2095		2208
@@ -2097,6 +2210,13 @@ static int decode_sections(struct objtool_file *file) · 1 unmodified line
2097	if (ret)	2210	if (ret)
2098	return ret;	2211	return ret;
2099		2212
		2213	/*
		2214	* Must be before read_unwind_hints() since that needs insn->noendbr.
		2215	*/
		2216	ret = read_noendbr_hints(file);
		2217	if (ret)
		2218	return ret;
		2219
2100	/*	2220	/*
2101	* Must be before add_{jump_call}_destination.	2221	* Must be before add_{jump_call}_destination.
2102	*/	2222	*/
@@ -2128,6 +2248,14 @@ static int decode_sections(struct objtool_file *file) · 25 unmodified lines
2128	if (ret)	2248	if (ret)
2129	return ret;	2249	return ret;
2130		2250
		2251	/*
		2252	* Must be after add_call_destinations() such that it can override
		2253	* dead_end_function() marks.
		2254	*/
		2255	ret = add_dead_ends(file);
		2256	if (ret)
		2257	return ret;
		2258
2131	ret = add_jump_table_alts(file);	2259	ret = add_jump_table_alts(file);
2132	if (ret)	2260	if (ret)
2133	return ret;	2261	return ret;
@@ -3026,6 +3154,115 @@ static struct instruction next_insn_to_validate(struct objtool_file file, · 892 unmodified lines
3026	return next_insn_same_sec(file, insn);	3154	return next_insn_same_sec(file, insn);
3027	}	3155	}
3028		3156
		3157	static struct instruction *
		3158	validate_ibt_reloc(struct objtool_file file, struct reloc reloc)
		3159	{
		3160	struct instruction *dest;
		3161	struct section *sec;
		3162	unsigned long off;
		3163
		3164	sec = reloc->sym->sec;
		3165	off = reloc->sym->offset;
		3166
		3167	if ((reloc->sec->base->sh.sh_flags & SHF_EXECINSTR) &&
		3168	(reloc->type == R_X86_64_PC32 \|\| reloc->type == R_X86_64_PLT32))
		3169	off += arch_dest_reloc_offset(reloc->addend);
		3170	else
		3171	off += reloc->addend;
		3172
		3173	dest = find_insn(file, sec, off);
		3174	if (!dest)
		3175	return NULL;
		3176
		3177	if (dest->type == INSN_ENDBR) {
		3178	if (!list_empty(&dest->call_node))
		3179	list_del_init(&dest->call_node);
		3180
		3181	return NULL;
		3182	}
		3183
		3184	if (reloc->sym->static_call_tramp)
		3185	return NULL;
		3186
		3187	return dest;
		3188	}
		3189
		3190	static void warn_noendbr(const char msg, struct section sec, unsigned long offset,
		3191	struct instruction *dest)
		3192	{
		3193	WARN_FUNC("%srelocation to !ENDBR: %s+0x%lx", sec, offset, msg,
		3194	dest->func ? dest->func->name : dest->sec->name,
		3195	dest->func ? dest->offset - dest->func->offset : dest->offset);
		3196	}
		3197
		3198	static void validate_ibt_dest(struct objtool_file file, struct instruction insn,
		3199	struct instruction *dest)
		3200	{
		3201	if (dest->func && dest->func == insn->func) {
		3202	/*
		3203	* Anything from->to self is either _THIS_IP_ or IRET-to-self.
		3204	*
		3205	* There is no sane way to annotate _THIS_IP_ since the compiler treats the
		3206	* relocation as a constant and is happy to fold in offsets, skewing any
		3207	* annotation we do, leading to vast amounts of false-positives.
		3208	*
		3209	* There's also compiler generated _THIS_IP_ through KCOV and
		3210	* such which we have no hope of annotating.
		3211	*
		3212	* As such, blanket accept self-references without issue.
		3213	*/
		3214	return;
		3215	}
		3216
		3217	if (dest->noendbr)
		3218	return;
		3219
		3220	warn_noendbr("", insn->sec, insn->offset, dest);
		3221	}
		3222
		3223	static void validate_ibt_insn(struct objtool_file file, struct instruction insn)
		3224	{
		3225	struct instruction *dest;
		3226	struct reloc *reloc;
		3227
		3228	switch (insn->type) {
		3229	case INSN_CALL:
		3230	case INSN_CALL_DYNAMIC:
		3231	case INSN_JUMP_CONDITIONAL:
		3232	case INSN_JUMP_UNCONDITIONAL:
		3233	case INSN_JUMP_DYNAMIC:
		3234	case INSN_JUMP_DYNAMIC_CONDITIONAL:
		3235	case INSN_RETURN:
		3236	/*
		3237	* We're looking for code references setting up indirect code
		3238	* flow. As such, ignore direct code flow and the actual
		3239	* dynamic branches.
		3240	*/
		3241	return;
		3242
		3243	case INSN_NOP:
		3244	/*
		3245	* handle_group_alt() will create INSN_NOP instruction that
		3246	* don't belong to any section, ignore all NOP since they won't
		3247	* carry a (useful) relocation anyway.
		3248	*/
		3249	return;
		3250
		3251	default:
		3252	break;
		3253	}
		3254
		3255	for (reloc = insn_reloc(file, insn);
		3256	reloc;
		3257	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
		3258	reloc->offset + 1,
		3259	(insn->offset + insn->len) - (reloc->offset + 1))) {
		3260	dest = validate_ibt_reloc(file, reloc);
		3261	if (dest)
		3262	validate_ibt_dest(file, insn, dest);
		3263	}
		3264	}
		3265
3029	/*	3266	/*
3030	* Follow the branch starting at the given instruction, and recursively follow	3267	* Follow the branch starting at the given instruction, and recursively follow
3031	* any other branches (jumps). Meanwhile, track the frame pointer state at	3268	* any other branches (jumps). Meanwhile, track the frame pointer state at
@@ -3115,9 +3352,8 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 83 unmodified lines
3115	switch (insn->type) {	3352	switch (insn->type) {
3116		3353
3117	case INSN_RETURN:	3354	case INSN_RETURN:
3118	if (next_insn && next_insn->type == INSN_TRAP) {	3355	if (sls && !insn->retpoline_safe &&
3119	next_insn->ignore = true;	3356	next_insn && next_insn->type != INSN_TRAP) {
3120	} else if (sls && !insn->retpoline_safe) {
3121	WARN_FUNC("missing int3 after ret",	3357	WARN_FUNC("missing int3 after ret",
3122	insn->sec, insn->offset);	3358	insn->sec, insn->offset);
3123	}	3359	}
@@ -3136,7 +3372,7 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 12 unmodified lines
3136	return 1;	3372	return 1;
3137	}	3373	}
3138		3374
3139	if (dead_end_function(file, insn->call_dest))	3375	if (insn->dead_end)
3140	return 0;	3376	return 0;
3141		3377
3142	break;	3378	break;
@@ -3164,9 +3400,8 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 21 unmodified lines
3164	break;	3400	break;
3165		3401
3166	case INSN_JUMP_DYNAMIC:	3402	case INSN_JUMP_DYNAMIC:
3167	if (next_insn && next_insn->type == INSN_TRAP) {	3403	if (sls && !insn->retpoline_safe &&
3168	next_insn->ignore = true;	3404	next_insn && next_insn->type != INSN_TRAP) {
3169	} else if (sls && !insn->retpoline_safe) {
3170	WARN_FUNC("missing int3 after indirect jump",	3405	WARN_FUNC("missing int3 after indirect jump",
3171	insn->sec, insn->offset);	3406	insn->sec, insn->offset);
3172	}	3407	}
@@ -3237,6 +3472,9 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 64 unmodified lines
3237	break;	3472	break;
3238	}	3473	}
3239		3474
		3475	if (ibt)
		3476	validate_ibt_insn(file, insn);
		3477
3240	if (insn->dead_end)	3478	if (insn->dead_end)
3241	return 0;	3479	return 0;
3242		3480
@@ -3337,7 +3575,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 94 unmodified lines
3337	int i;	3575	int i;
3338	struct instruction *prev_insn;	3576	struct instruction *prev_insn;
3339		3577
3340	if (insn->ignore \|\| insn->type == INSN_NOP)	3578	if (insn->ignore \|\| insn->type == INSN_NOP \|\| insn->type == INSN_TRAP)
3341	return true;	3579	return true;
3342		3580
3343	/*	3581	/*
@@ -3348,6 +3586,49 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 4 unmodified lines
3348	!strcmp(insn->sec->name, ".altinstr_aux"))	3586	!strcmp(insn->sec->name, ".altinstr_aux"))
3349	return true;	3587	return true;
3350		3588
		3589	/*
		3590	* Whole archive runs might encounder dead code from weak symbols.
		3591	* This is where the linker will have dropped the weak symbol in
		3592	* favour of a regular symbol, but leaves the code in place.
		3593	*
		3594	* In this case we'll find a piece of code (whole function) that is not
		3595	* covered by a !section symbol. Ignore them.
		3596	*/
		3597	if (!insn->func && lto) {
		3598	int size = find_symbol_hole_containing(insn->sec, insn->offset);
		3599	unsigned long end = insn->offset + size;
		3600
		3601	if (!size) /* not a hole */
		3602	return false;
		3603
		3604	if (size < 0) /* hole until the end */
		3605	return true;
		3606
		3607	sec_for_each_insn_continue(file, insn) {
		3608	/*
		3609	* If we reach a visited instruction at or before the
		3610	* end of the hole, ignore the unreachable.
		3611	*/
		3612	if (insn->visited)
		3613	return true;
		3614
		3615	if (insn->offset >= end)
		3616	break;
		3617
		3618	/*
		3619	* If this hole jumps to a .cold function, mark it ignore too.
		3620	*/
		3621	if (insn->jump_dest && insn->jump_dest->func &&
		3622	strstr(insn->jump_dest->func->name, ".cold")) {
		3623	struct instruction *dest = insn->jump_dest;
		3624	func_for_each_insn(file, dest->func, dest)
		3625	dest->ignore = true;
		3626	}
		3627	}
		3628
		3629	return false;
		3630	}
		3631
3351	if (!insn->func)	3632	if (!insn->func)
3352	return false;	3633	return false;
3353		3634
@@ -3479,6 +3760,53 @@ static int validate_functions(struct objtool_file *file) · 125 unmodified lines
3479	return warnings;	3760	return warnings;
3480	}	3761	}
3481		3762
		3763	static int validate_ibt(struct objtool_file *file)
		3764	{
		3765	struct section *sec;
		3766	struct reloc *reloc;
		3767
		3768	for_each_sec(file, sec) {
		3769	bool is_data;
		3770
		3771	/* already done in validate_branch() */
		3772	if (sec->sh.sh_flags & SHF_EXECINSTR)
		3773	continue;
		3774
		3775	if (!sec->reloc)
		3776	continue;
		3777
		3778	if (!strncmp(sec->name, ".orc", 4))
		3779	continue;
		3780
		3781	if (!strncmp(sec->name, ".discard", 8))
		3782	continue;
		3783
		3784	if (!strncmp(sec->name, ".debug", 6))
		3785	continue;
		3786
		3787	if (!strcmp(sec->name, "_error_injection_whitelist"))
		3788	continue;
		3789
		3790	if (!strcmp(sec->name, "_kprobe_blacklist"))
		3791	continue;
		3792
		3793	is_data = strstr(sec->name, ".data") \|\| strstr(sec->name, ".rodata");
		3794
		3795	list_for_each_entry(reloc, &sec->reloc->reloc_list, list) {
		3796	struct instruction *dest;
		3797
		3798	dest = validate_ibt_reloc(file, reloc);
		3799	if (is_data && dest && !dest->noendbr) {
		3800	warn_noendbr("data ", reloc->sym->sec,
		3801	reloc->sym->offset + reloc->addend,
		3802	dest);
		3803	}
		3804	}
		3805	}
		3806
		3807	return 0;
		3808	}
		3809
3482	static int validate_reachable_instructions(struct objtool_file *file)	3810	static int validate_reachable_instructions(struct objtool_file *file)
3483	{	3811	{
3484	struct instruction *insn;	3812	struct instruction *insn;
@@ -3501,6 +3829,16 @@ int check(struct objtool_file *file) · 16 unmodified lines
3501	{	3829	{
3502	int ret, warnings = 0;	3830	int ret, warnings = 0;
3503		3831
		3832	if (lto && !(vmlinux \|\| module)) {
		3833	fprintf(stderr, "--lto requires: --vmlinux or --module\n");
		3834	return 1;
		3835	}
		3836
		3837	if (ibt && !lto) {
		3838	fprintf(stderr, "--ibt requires: --lto\n");
		3839	return 1;
		3840	}
		3841
3504	arch_initial_func_cfi_state(&initial_func_cfi);	3842	arch_initial_func_cfi_state(&initial_func_cfi);
3505	init_cfi_state(&init_cfi);	3843	init_cfi_state(&init_cfi);
3506	init_cfi_state(&func_cfi);	3844	init_cfi_state(&func_cfi);
@@ -3521,7 +3859,7 @@ int check(struct objtool_file *file) · 14 unmodified lines
3521	if (list_empty(&file->insn_list))	3859	if (list_empty(&file->insn_list))
3522	goto out;	3860	goto out;
3523		3861
3524	if (vmlinux && !validate_dup) {	3862	if (vmlinux && !lto) {
3525	ret = validate_vmlinux_functions(file);	3863	ret = validate_vmlinux_functions(file);
3526	if (ret < 0)	3864	if (ret < 0)
3527	goto out;	3865	goto out;
@@ -3547,6 +3885,13 @@ int check(struct objtool_file *file) · 19 unmodified lines
3547	goto out;	3885	goto out;
3548	warnings += ret;	3886	warnings += ret;
3549		3887
		3888	if (ibt) {
		3889	ret = validate_ibt(file);
		3890	if (ret < 0)
		3891	goto out;
		3892	warnings += ret;
		3893	}
		3894
3550	if (!warnings) {	3895	if (!warnings) {
3551	ret = validate_reachable_instructions(file);	3896	ret = validate_reachable_instructions(file);
3552	if (ret < 0)	3897	if (ret < 0)
@@ -3573,6 +3918,13 @@ int check(struct objtool_file *file) · 20 unmodified lines
3573	warnings += ret;	3918	warnings += ret;
3574	}	3919	}
3575		3920
		3921	if (ibt) {
		3922	ret = create_ibt_endbr_seal_sections(file);
		3923	if (ret < 0)
		3924	goto out;
		3925	warnings += ret;
		3926	}
		3927
3576	if (stats) {	3928	if (stats) {
3577	printf("nr_insns_visited: %ld\n", nr_insns_visited);	3929	printf("nr_insns_visited: %ld\n", nr_insns_visited);
3578	printf("nr_cfi: %ld\n", nr_cfi);	3930	printf("nr_cfi: %ld\n", nr_cfi);

tools/objtool/elf.c+63−0

@@ -83,6 +83,31 @@ static int symbol_by_offset(const void key, const struct rb_node node) · 82 unmodified lines
83	return 0;	83	return 0;
84	}	84	}
85		85
		86	struct symbol_hole {
		87	unsigned long key;
		88	const struct symbol *sym;
		89	};
		90
		91	/*
		92	* Find !section symbol where @offset is after it.
		93	*/
		94	static int symbol_hole_by_offset(const void key, const struct rb_node node)
		95	{
		96	const struct symbol *s = rb_entry(node, struct symbol, node);
		97	struct symbol_hole sh = (void )key;
		98
		99	if (sh->key < s->offset)
		100	return -1;
		101
		102	if (sh->key >= s->offset + s->len) {
		103	if (s->type != STT_SECTION)
		104	sh->sym = s;
		105	return 1;
		106	}
		107
		108	return 0;
		109	}
		110
86	struct section find_section_by_name(const struct elf elf, const char *name)	111	struct section find_section_by_name(const struct elf elf, const char *name)
87	{	112	{
88	struct section *sec;	113	struct section *sec;
@@ -162,6 +187,41 @@ struct symbol find_symbol_containing(const struct section sec, unsigned long o · 73 unmodified lines
162	return NULL;	187	return NULL;
163	}	188	}
164		189
		190	/*
		191	* Returns size of hole starting at @offset.
		192	*/
		193	int find_symbol_hole_containing(const struct section *sec, unsigned long offset)
		194	{
		195	struct symbol_hole hole = {
		196	.key = offset,
		197	.sym = NULL,
		198	};
		199	struct rb_node *n;
		200	struct symbol *s;
		201
		202	/*
		203	* Find the rightmost symbol for which @offset is after it.
		204	*/
		205	n = rb_find(&hole, &sec->symbol_tree, symbol_hole_by_offset);
		206
		207	/* found a symbol that contains @offset */
		208	if (n)
		209	return 0; /* not a hole */
		210
		211	/* didn't find a symbol for which @offset is after it */
		212	if (!hole.sym)
		213	return 0; /* not a hole */
		214
		215	/* @offset >= sym->offset + sym->len, find symbol after it */
		216	n = rb_next(&hole.sym->node);
		217	if (!n)
		218	return -1; /* until end of address space */
		219
		220	/* hole until start of next symbol */
		221	s = rb_entry(n, struct symbol, node);
		222	return s->offset - offset;
		223	}
		224
165	struct symbol find_func_containing(struct section sec, unsigned long offset)	225	struct symbol find_func_containing(struct section sec, unsigned long offset)
166	{	226	{
167	struct rb_node *node;	227	struct rb_node *node;
@@ -1019,6 +1079,9 @@ int elf_write(struct elf *elf) · 851 unmodified lines
1019	struct section *sec;	1079	struct section *sec;
1020	Elf_Scn *s;	1080	Elf_Scn *s;
1021		1081
		1082	if (dryrun)
		1083	return 0;
		1084
1022	/* Update changed relocation sections and section headers: */	1085	/* Update changed relocation sections and section headers: */
1023	list_for_each_entry(sec, &elf->sections, list) {	1086	list_for_each_entry(sec, &elf->sections, list) {
1024	if (sec->changed) {	1087	if (sec->changed) {

tools/objtool/include/objtool/arch.h+1−0

@@ -27,6 +27,7 @@ enum insn_type { · 26 unmodified lines
27	INSN_STD,	27	INSN_STD,
28	INSN_CLD,	28	INSN_CLD,
29	INSN_TRAP,	29	INSN_TRAP,
		30	INSN_ENDBR,
30	INSN_OTHER,	31	INSN_OTHER,
31	};	32	};
32		33

tools/objtool/include/objtool/builtin.h+2−1

@@ -9,7 +9,8 @@ · 8 unmodified lines
9		9
10	extern const struct option check_options[];	10	extern const struct option check_options[];
11	extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,	11	extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
12	validate_dup, vmlinux, mcount, noinstr, backup, sls;	12	lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
		13	ibt;
13		14
14	extern int cmd_parse_options(int argc, const char *argv, const char const usage[]);	15	extern int cmd_parse_options(int argc, const char *argv, const char const usage[]);
15		16

tools/objtool/include/objtool/check.h+10−3

@@ -45,11 +45,18 @@ struct instruction { · 44 unmodified lines
45	unsigned int len;	45	unsigned int len;
46	enum insn_type type;	46	enum insn_type type;
47	unsigned long immediate;	47	unsigned long immediate;
48	bool dead_end, ignore, ignore_alts;	48
49	bool hint;	49	u8 dead_end : 1,
50	bool retpoline_safe;	50	ignore : 1,
		51	ignore_alts : 1,
		52	hint : 1,
		53	retpoline_safe : 1,
		54	noendbr : 1;
		55	/* 2 bit hole */
51	s8 instr;	56	s8 instr;
52	u8 visited;	57	u8 visited;
		58	/* u8 hole */
		59
53	struct alt_group *alt_group;	60	struct alt_group *alt_group;
54	struct symbol *call_dest;	61	struct symbol *call_dest;
55	struct instruction *jump_dest;	62	struct instruction *jump_dest;

tools/objtool/include/objtool/elf.h+1−0

@@ -152,6 +152,7 @@ struct symbol find_func_by_offset(struct section sec, unsigned long offset); · 151 unmodified lines
152	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset);	152	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset);
153	struct symbol find_symbol_by_name(const struct elf elf, const char *name);	153	struct symbol find_symbol_by_name(const struct elf elf, const char *name);
154	struct symbol find_symbol_containing(const struct section sec, unsigned long offset);	154	struct symbol find_symbol_containing(const struct section sec, unsigned long offset);
		155	int find_symbol_hole_containing(const struct section *sec, unsigned long offset);
155	struct reloc find_reloc_by_dest(const struct elf elf, struct section *sec, unsigned long offset);	156	struct reloc find_reloc_by_dest(const struct elf elf, struct section *sec, unsigned long offset);
156	struct reloc find_reloc_by_dest_range(const struct elf elf, struct section *sec,	157	struct reloc find_reloc_by_dest_range(const struct elf elf, struct section *sec,
157	unsigned long offset, unsigned int len);	158	unsigned long offset, unsigned int len);

tools/objtool/include/objtool/objtool.h+4−0

 	struct list_head retpoline_call_list;
 	struct list_head static_call_list;
 	struct list_head mcount_loc_list;
+	struct list_head endbr_list;
 	bool ignore_unreachables, c_file, hints, rodata;
+	unsigned int nr_endbr;
+	unsigned int nr_endbr_int;
 	unsigned long jl_short, jl_long;
 	unsigned long jl_nop_short, jl_nop_long;

tools/objtool/include/objtool/warn.h+2−0

@@ -22,6 +22,8 @@ static inline char offstr(struct section sec, unsigned long offset) · 21 unmodified lines
22	unsigned long name_off;	22	unsigned long name_off;
23		23
24	func = find_func_containing(sec, offset);	24	func = find_func_containing(sec, offset);
		25	if (!func)
		26	func = find_symbol_containing(sec, offset);
25	if (func) {	27	if (func) {
26	name = func->name;	28	name = func->name;
27	name_off = offset - func->offset;	29	name_off = offset - func->offset;

tools/objtool/objtool.c+1−0

@@ -128,6 +128,7 @@ struct objtool_file objtool_open_read(const char _objname) · 127 unmodified lines
128	INIT_LIST_HEAD(&file.retpoline_call_list);	128	INIT_LIST_HEAD(&file.retpoline_call_list);
129	INIT_LIST_HEAD(&file.static_call_list);	129	INIT_LIST_HEAD(&file.static_call_list);
130	INIT_LIST_HEAD(&file.mcount_loc_list);	130	INIT_LIST_HEAD(&file.mcount_loc_list);
		131	INIT_LIST_HEAD(&file.endbr_list);
131	file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment");	132	file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment");
132	file.ignore_unreachables = no_unreachable;	133	file.ignore_unreachables = no_unreachable;
133	file.hints = false;	134	file.hints = false;

Merge tag 'x86_core_for_5.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -159,7 +159,7 @@ extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); · 158 unmodified lines
159	#endif /* CONFIG_IA32_EMULATION */	159	#endif /* CONFIG_IA32_EMULATION */
160		160
161		161
162	#ifdef CONFIG_X86_X32	162	#ifdef CONFIG_X86_X32_ABI
163	/*	163	/*
164	* For the x32 ABI, we need to create a stub for compat_sys_*() which is aware	164	* For the x32 ABI, we need to create a stub for compat_sys_*() which is aware
165	* of the x86-64-style parameter ordering of x32 syscalls. The syscalls common	165	* of the x86-64-style parameter ordering of x32 syscalls. The syscalls common
@@ -177,12 +177,12 @@ extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); · 11 unmodified lines
177		177
178	#define __X32_COMPAT_SYS_NI(name) \	178	#define __X32_COMPAT_SYS_NI(name) \
179	__SYS_NI(x64, compat_sys_##name)	179	__SYS_NI(x64, compat_sys_##name)
180	#else /* CONFIG_X86_X32 */	180	#else /* CONFIG_X86_X32_ABI */
181	#define __X32_COMPAT_SYS_STUB0(name)	181	#define __X32_COMPAT_SYS_STUB0(name)
182	#define __X32_COMPAT_SYS_STUBx(x, name, ...)	182	#define __X32_COMPAT_SYS_STUBx(x, name, ...)
183	#define __X32_COMPAT_COND_SYSCALL(name)	183	#define __X32_COMPAT_COND_SYSCALL(name)
184	#define __X32_COMPAT_SYS_NI(name)	184	#define __X32_COMPAT_SYS_NI(name)
185	#endif /* CONFIG_X86_X32 */	185	#endif /* CONFIG_X86_X32_ABI */
186		186
187		187
188	#ifdef CONFIG_COMPAT	188	#ifdef CONFIG_COMPAT