Merge tag 'x86_core_for_5.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 CET-IBT (Control-Flow-Integrity) support from Peter Zijlstra: "Add support for Intel CET-IBT, available since Tigerlake (11th gen), which is a coarse grained, hardware based, forward edge Control-Flow-Integrity mechanism where any indirect CALL/JMP must target an ENDBR instruction or suffer #CP. Additionally, since Alderlake (12th gen)/Sapphire-Rapids, speculation is limited to 2 instructions (and typically fewer) on branch targets not starting with ENDBR. CET-IBT also limits speculation of the next sequential instruction after the indirect CALL/JMP [1]. CET-IBT is fundamentally incompatible with retpolines, but provides, as described above, speculation limits itself" [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/branch-history-injection.html * tag 'x86_core_for_5.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (53 commits) kvm/emulate: Fix SETcc emulation for ENDBR x86/Kconfig: Only allow CONFIG_X86_KERNEL_IBT with ld.lld >= 14.0.0 x86/Kconfig: Only enable CONFIG_CC_HAS_IBT for clang >= 14.0.0 kbuild: Fixup the IBT kbuild changes x86/Kconfig: Do not allow CONFIG_X86_X32_ABI=y with llvm-objcopy x86: Remove toolchain check for X32 ABI capability x86/alternative: Use .ibt_endbr_seal to seal indirect calls objtool: Find unused ENDBR instructions objtool: Validate IBT assumptions objtool: Add IBT/ENDBR decoding objtool: Read the NOENDBR annotation x86: Annotate idtentry_df() x86,objtool: Move the ASM_REACHABLE annotation to objtool.h x86: Annotate call_on_stack() objtool: Rework ASM_REACHABLE x86: Mark __invalid_creds() __noreturn exit: Mark do_group_exit() __noreturn x86: Mark stop_this_cpu() __noreturn objtool: Ignore extra-symbol code objtool: Rename --duplicate to --lto ...

Linus Torvalds committed 4 years ago

Commit 70010522 parents f022814 3986f6595 files · +1454 −326Browse files

Unified · Split

95 files

arch/powerpc/include/asm/livepatch.h +0 −11
arch/powerpc/kernel/kprobes.c +21 −13
arch/um/kernel/um_arch.c +4 −0
arch/x86/Kconfig +37 −6
arch/x86/Makefile +14 −18
arch/x86/crypto/crc32c-pcl-intel-asm_64.S +3 −0
arch/x86/entry/entry_64.S +27 −4
arch/x86/entry/entry_64_compat.S +5 −0
arch/x86/entry/syscalls/Makefile +1 −1
arch/x86/include/asm/alternative.h +1 −0
arch/x86/include/asm/bug.h +1 −0
arch/x86/include/asm/cpu.h +4 −0
arch/x86/include/asm/cpufeatures.h +1 −0
arch/x86/include/asm/efi.h +7 −2
arch/x86/include/asm/ibt.h +105 −0
arch/x86/include/asm/idtentry.h +16 −9
arch/x86/include/asm/irq_stack.h +3 −1
arch/x86/include/asm/irqflags.h +0 −5
arch/x86/include/asm/linkage.h +31 −0
arch/x86/include/asm/msr-index.h +19 −1
arch/x86/include/asm/paravirt.h +1 −0
arch/x86/include/asm/paravirt_types.h +0 −1
arch/x86/include/asm/processor.h +1 −1
arch/x86/include/asm/qspinlock_paravirt.h +3 −0
arch/x86/include/asm/segment.h +3 −2
arch/x86/include/asm/setup.h +2 −1
arch/x86/include/asm/syscall_wrapper.h +3 −3
arch/x86/include/asm/text-patching.h +23 −7
arch/x86/include/asm/traps.h +2 −0
arch/x86/include/asm/vdso.h +1 −1
arch/x86/include/uapi/asm/processor-flags.h +2 −0
arch/x86/kernel/alternative.c +52 −14
arch/x86/kernel/apm_32.c +7 −0
arch/x86/kernel/cpu/common.c +58 −1
arch/x86/kernel/ftrace.c +2 −7
arch/x86/kernel/ftrace_64.S +24 −4
arch/x86/kernel/head_64.S +12 −2
arch/x86/kernel/idt.c +7 −2
arch/x86/kernel/kprobes/core.c +20 −9
arch/x86/kernel/kvm.c +2 −1
arch/x86/kernel/machine_kexec_64.c +3 −1
arch/x86/kernel/module.c +7 −1
arch/x86/kernel/paravirt.c +5 −24
arch/x86/kernel/process.c +1 −1
arch/x86/kernel/process_64.c +1 −1
arch/x86/kernel/relocate_kernel_64.S +10 −0
arch/x86/kernel/traps.c +75 −0
arch/x86/kernel/vmlinux.lds.S +9 −0
arch/x86/kvm/emulate.c +10 −6
arch/x86/lib/error-inject.c +2 −0
arch/x86/lib/retpoline.S +1 −0
arch/x86/net/bpf_jit_comp.c +24 −2
arch/x86/platform/efi/Makefile +0 −1
arch/x86/platform/efi/efi_thunk_64.S +4 −2
arch/x86/xen/enlighten_pv.c +9 −1
arch/x86/xen/xen-asm.S +10 −0
arch/x86/xen/xen-head.S +7 −2
fs/fuse/ioctl.c +1 −1
fs/xfs/xfs_ioctl32.c +1 −1
include/asm-generic/vmlinux.lds.h +4 −0
include/linux/cfi.h +10 −1
include/linux/compiler.h +0 −7
include/linux/cred.h +1 −1
include/linux/kprobes.h +2 −1
include/linux/objtool.h +32 −0
include/linux/sched/task.h +1 −1
kernel/bpf/trampoline.c +4 −16
kernel/cred.c +1 −1
kernel/exit.c +1 −1
kernel/kprobes.c +54 −20
kernel/livepatch/patch.c +2 −17
kernel/trace/ftrace.c +40 −8
samples/ftrace/ftrace-direct-modify.c +5 −0
samples/ftrace/ftrace-direct-multi-modify.c +7 −3
samples/ftrace/ftrace-direct-multi.c +4 −1
samples/ftrace/ftrace-direct-too.c +3 −0
samples/ftrace/ftrace-direct.c +3 −0
scripts/Makefile.build +14 −13
scripts/Makefile.lib +2 −2
scripts/link-vmlinux.sh +9 −3
scripts/mod/modpost.c +6 −6
sound/core/control_compat.c +8 −8
sound/core/pcm_compat.c +10 −10
tools/include/linux/objtool.h +32 −0
tools/objtool/arch/x86/decode.c +29 −5
tools/objtool/builtin-check.c +5 −2
tools/objtool/check.c +376 −24
tools/objtool/elf.c +63 −0
tools/objtool/include/objtool/arch.h +1 −0
tools/objtool/include/objtool/builtin.h +2 −1
tools/objtool/include/objtool/check.h +10 −3
tools/objtool/include/objtool/elf.h +1 −0
tools/objtool/include/objtool/objtool.h +4 −0
tools/objtool/include/objtool/warn.h +2 −0
tools/objtool/objtool.c +1 −0

arch/powerpc/include/asm/livepatch.h+0−11

@@ -17,17 +17,6 @@ static inline void klp_arch_set_pc(struct ftrace_regs *fregs, unsigned long ip) · 16 unmodified lines
17	17	ftrace_instruction_pointer_set(fregs, ip);
18	18	}
19	19
20		−#define klp_get_ftrace_location klp_get_ftrace_location
21		−static inline unsigned long klp_get_ftrace_location(unsigned long faddr)
22		−{
23		− /*
24		− * Live patch works on PPC32 and only with -mprofile-kernel on PPC64. In
25		− * both cases, the ftrace location is always within the first 16 bytes.
26		− */
27		− return ftrace_location_range(faddr, faddr + 16);
28		−}
29		−#endif /* CONFIG_LIVEPATCH */
30		−
31	20	#ifdef CONFIG_LIVEPATCH_64
32	21	static inline void klp_init_thread_info(struct task_struct *p)
33	22	{

arch/powerpc/kernel/kprobes.c+21−13

@@ -105,6 +105,27 @@ kprobe_opcode_t kprobe_lookup_name(const char name, unsigned int offset) · 104 unmodified lines
105	105	return addr;
106	106	}
107	107
	108	+static bool arch_kprobe_on_func_entry(unsigned long offset)
	109	+{
	110	+#ifdef PPC64_ELF_ABI_v2
	111	+#ifdef CONFIG_KPROBES_ON_FTRACE
	112	+ return offset <= 16;
	113	+#else
	114	+ return offset <= 8;
	115	+#endif
	116	+#else
	117	+ return !offset;
	118	+#endif
	119	+}
	120	+
	121	+/* XXX try and fold the magic of kprobe_lookup_name() in this */
	122	+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
	123	+ bool *on_func_entry)
	124	+{
	125	+ *on_func_entry = arch_kprobe_on_func_entry(offset);
	126	+ return (kprobe_opcode_t *)(addr + offset);
	127	+}
	128	+
108	129	void *alloc_insn_page(void)
109	130	{
110	131	void *page;
@@ -218,19 +239,6 @@ static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs · 107 unmodified lines
218	239	kcb->kprobe_saved_msr = regs->msr;
219	240	}
220	241
221		−bool arch_kprobe_on_func_entry(unsigned long offset)
222		−{
223		−#ifdef PPC64_ELF_ABI_v2
224		−#ifdef CONFIG_KPROBES_ON_FTRACE
225		− return offset <= 16;
226		−#else
227		− return offset <= 8;
228		−#endif
229		−#else
230		− return !offset;
231		−#endif
232		−}
233		−
234	242	void arch_prepare_kretprobe(struct kretprobe_instance ri, struct pt_regs regs)
235	243	{
236	244	ri->ret_addr = (kprobe_opcode_t *)regs->link;

arch/um/kernel/um_arch.c+4−0

@@ -424,6 +424,10 @@ void __init check_bugs(void) · 423 unmodified lines
424	424	os_check_bugs();
425	425	}
426	426
	427	+void apply_ibt_endbr(s32 start, s32 end)
	428	+{
	429	+}
	430	+
427	431	void apply_retpolines(s32 start, s32 end)
428	432	{
429	433	}

arch/x86/Kconfig+37−6

@@ -1842,6 +1842,36 @@ config X86_UMIP · 1841 unmodified lines
1842	1842	specific cases in protected and virtual-8086 modes. Emulated
1843	1843	results are dummy.
1844	1844
	1845	+config CC_HAS_IBT
	1846	+ # GCC >= 9 and binutils >= 2.29
	1847	+ # Retpoline check to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93654
	1848	+ # Clang/LLVM >= 14
	1849	+ # https://github.com/llvm/llvm-project/commit/e0b89df2e0f0130881bf6c39bf31d7f6aac00e0f
	1850	+ # https://github.com/llvm/llvm-project/commit/dfcf69770bc522b9e411c66454934a37c1f35332
	1851	+ def_bool ((CC_IS_GCC && $(cc-option, -fcf-protection=branch -mindirect-branch-register)) \|\| \
	1852	+ (CC_IS_CLANG && CLANG_VERSION >= 140000)) && \
	1853	+ $(as-instr,endbr64)
	1854	+
	1855	+config X86_KERNEL_IBT
	1856	+ prompt "Indirect Branch Tracking"
	1857	+ bool
	1858	+ depends on X86_64 && CC_HAS_IBT && STACK_VALIDATION
	1859	+ # https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
	1860	+ depends on !LD_IS_LLD \|\| LLD_VERSION >= 140000
	1861	+ help
	1862	+ Build the kernel with support for Indirect Branch Tracking, a
	1863	+ hardware support course-grain forward-edge Control Flow Integrity
	1864	+ protection. It enforces that all indirect calls must land on
	1865	+ an ENDBR instruction, as such, the compiler will instrument the
	1866	+ code with them to make this happen.
	1867	+
	1868	+ In addition to building the kernel with IBT, seal all functions that
	1869	+ are not indirect call targets, avoiding them ever becomming one.
	1870	+
	1871	+ This requires LTO like objtool runs and will slow down the build. It
	1872	+ does significantly reduce the number of ENDBR instructions in the
	1873	+ kernel image.
	1874	+
1845	1875	config X86_INTEL_MEMORY_PROTECTION_KEYS
1846	1876	prompt "Memory Protection Keys"
1847	1877	def_bool y
@@ -2815,19 +2845,20 @@ config IA32_AOUT · 967 unmodified lines
2815	2845	help
2816	2846	Support old a.out binaries in the 32bit emulation.
2817	2847
2818		−config X86_X32
	2848	+config X86_X32_ABI
2819	2849	bool "x32 ABI for 64-bit mode"
2820	2850	depends on X86_64
	2851	+ # llvm-objcopy does not convert x86_64 .note.gnu.property or
	2852	+ # compressed debug sections to x86_x32 properly:
	2853	+ # https://github.com/ClangBuiltLinux/linux/issues/514
	2854	+ # https://github.com/ClangBuiltLinux/linux/issues/1141
	2855	+ depends on $(success,$(OBJCOPY) --version \| head -n1 \| grep -qv llvm)
2821	2856	help
2822	2857	Include code to run binaries for the x32 native 32-bit ABI
2823	2858	for 64-bit processors. An x32 process gets access to the
2824	2859	full 64-bit register file and wide data path while leaving
2825	2860	pointers at 32 bits for smaller memory footprint.
2826	2861
2827		− You will need a recent binutils (2.22 or later) with
2828		− elf32_x86_64 support enabled to compile a kernel with this
2829		− option set.
2830		−
2831	2862	config COMPAT_32
2832	2863	def_bool y
2833	2864	depends on IA32_EMULATION \|\| X86_32
@@ -2836,7 +2867,7 @@ config COMPAT_32 · 2 unmodified lines
2836	2867
2837	2868	config COMPAT
2838	2869	def_bool y
2839		− depends on IA32_EMULATION \|\| X86_X32
	2870	+ depends on IA32_EMULATION \|\| X86_X32_ABI
2840	2871
2841	2872	if COMPAT
2842	2873	config COMPAT_FOR_U64_ALIGNMENT

arch/x86/Makefile+14−18

@@ -36,7 +36,7 @@ endif · 35 unmodified lines
36	36
37	37	# How to compile the 16-bit code. Note we always compile for -march=i386;
38	38	# that way we can complain to the user if the CPU is insufficient.
39		−REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \
	39	+REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
40	40	-Wall -Wstrict-prototypes -march=i386 -mregparm=3 \
41	41	-fno-strict-aliasing -fomit-frame-pointer -fno-pic \
42	42	-mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
@@ -62,8 +62,20 @@ export BITS · 19 unmodified lines
62	62	#
63	63	KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
64	64
65		−# Intel CET isn't enabled in the kernel
	65	+ifeq ($(CONFIG_X86_KERNEL_IBT),y)
	66	+#
	67	+# Kernel IBT has S_CET.NOTRACK_EN=0, as such the compilers must not generate
	68	+# NOTRACK prefixes. Current generation compilers unconditionally employ NOTRACK
	69	+# for jump-tables, as such, disable jump-tables for now.
	70	+#
	71	+# (jump-tables are implicitly disabled by RETPOLINE)
	72	+#
	73	+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
	74	+#
	75	+KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables)
	76	+else
66	77	KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
	78	+endif
67	79
68	80	ifeq ($(CONFIG_X86_32),y)
69	81	BITS := 32
@@ -140,22 +152,6 @@ else · 70 unmodified lines
140	152	KBUILD_CFLAGS += -mcmodel=kernel
141	153	endif
142	154
143		−ifdef CONFIG_X86_X32
144		− x32_ld_ok := $(call try-run,\
145		− /bin/echo -e '1: .quad 1b' \| \
146		− $(CC) $(KBUILD_AFLAGS) -c -x assembler -o "$$TMP" - && \
147		− $(OBJCOPY) -O elf32-x86-64 "$$TMP" "$$TMP.o" && \
148		− $(LD) -m elf32_x86_64 "$$TMP.o" -o "$$TMP",y,n)
149		− ifeq ($(x32_ld_ok),y)
150		− CONFIG_X86_X32_ABI := y
151		− KBUILD_AFLAGS += -DCONFIG_X86_X32_ABI
152		− KBUILD_CFLAGS += -DCONFIG_X86_X32_ABI
153		− else
154		− $(warning CONFIG_X86_X32 enabled but no binutils support)
155		− endif
156		−endif
157		−export CONFIG_X86_X32_ABI
158		−
159	155	#
160	156	# If the function graph tracer is used with mcount instead of fentry,
161	157	# '-maccumulate-outgoing-args' is needed to prevent a GCC bug

arch/x86/crypto/crc32c-pcl-intel-asm_64.S+3−0

@@ -195,6 +195,7 @@ crc_array: · 194 unmodified lines
195	195	.altmacro
196	196	LABEL crc_ %i
197	197	.noaltmacro
	198	+ ENDBR
198	199	crc32q -i*8(block_0), crc_init
199	200	crc32q -i*8(block_1), crc1
200	201	crc32q -i*8(block_2), crc2
@@ -204,6 +205,7 @@ LABEL crc_ %i · 3 unmodified lines
204	205	.altmacro
205	206	LABEL crc_ %i
206	207	.noaltmacro
	208	+ ENDBR
207	209	crc32q -i*8(block_0), crc_init
208	210	crc32q -i*8(block_1), crc1
209	211	# SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet
@@ -237,6 +239,7 @@ LABEL crc_ %i · 27 unmodified lines
237	239	################################################################
238	240
239	241	LABEL crc_ 0
	242	+ ENDBR
240	243	mov tmp, len
241	244	cmp $128*24, tmp
242	245	jae full_block

arch/x86/entry/entry_64.S+27−4

@@ -86,6 +86,7 @@ · 85 unmodified lines
86	86
87	87	SYM_CODE_START(entry_SYSCALL_64)
88	88	UNWIND_HINT_EMPTY
	89	+ ENDBR
89	90
90	91	swapgs
91	92	/* tss.sp2 is scratch space. */
@@ -94,6 +95,7 @@ SYM_CODE_START(entry_SYSCALL_64) · 2 unmodified lines
94	95	movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
95	96
96	97	SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
	98	+ ANNOTATE_NOENDBR
97	99
98	100	/* Construct struct pt_regs on stack */
99	101	pushq $__USER_DS /* pt_regs->ss */
@@ -276,6 +278,7 @@ SYM_FUNC_END(__switch_to_asm) · 176 unmodified lines
276	278	.pushsection .text, "ax"
277	279	SYM_CODE_START(ret_from_fork)
278	280	UNWIND_HINT_EMPTY
	281	+ ANNOTATE_NOENDBR // copy_thread
279	282	movq %rax, %rdi
280	283	call schedule_tail /* rdi: 'prev' task parameter */
281	284
@@ -350,6 +353,7 @@ SYM_CODE_END(ret_from_fork) · 68 unmodified lines
350	353	.macro idtentry vector asmsym cfunc has_error_code:req
351	354	SYM_CODE_START(\asmsym)
352	355	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
	356	+ ENDBR
353	357	ASM_CLAC
354	358
355	359	.if \has_error_code == 0
@@ -417,6 +421,7 @@ SYM_CODE_END(\asmsym) · 61 unmodified lines
417	421	.macro idtentry_mce_db vector asmsym cfunc
418	422	SYM_CODE_START(\asmsym)
419	423	UNWIND_HINT_IRET_REGS
	424	+ ENDBR
420	425	ASM_CLAC
421	426
422	427	pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -472,6 +477,7 @@ SYM_CODE_END(\asmsym) · 49 unmodified lines
472	477	.macro idtentry_vc vector asmsym cfunc
473	478	SYM_CODE_START(\asmsym)
474	479	UNWIND_HINT_IRET_REGS
	480	+ ENDBR
475	481	ASM_CLAC
476	482
477	483	/*
@@ -533,6 +539,7 @@ SYM_CODE_END(\asmsym) · 55 unmodified lines
533	539	.macro idtentry_df vector asmsym cfunc
534	540	SYM_CODE_START(\asmsym)
535	541	UNWIND_HINT_IRET_REGS offset=8
	542	+ ENDBR
536	543	ASM_CLAC
537	544
538	545	/* paranoid_entry returns GS information for paranoid_exit in EBX. */
@@ -544,6 +551,9 @@ SYM_CODE_START(\asmsym) · 5 unmodified lines
544	551	movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
545	552	call \cfunc
546	553
	554	+ /* For some configurations \cfunc ends up being a noreturn. */
	555	+ REACHABLE
	556	+
547	557	jmp paranoid_exit
548	558
549	559	_ASM_NOKPROBE(\asmsym)
@@ -564,6 +574,7 @@ __irqentry_text_start: · 14 unmodified lines
564	574	.align 16
565	575	.globl __irqentry_text_end
566	576	__irqentry_text_end:
	577	+ ANNOTATE_NOENDBR
567	578
568	579	SYM_CODE_START_LOCAL(common_interrupt_return)
569	580	SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
@@ -608,8 +619,8 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) · 38 unmodified lines
608	619
609	620	/* Restore RDI. */
610	621	popq %rdi
611		− SWAPGS
612		− INTERRUPT_RETURN
	622	+ swapgs
	623	+ jmp .Lnative_iret
613	624
614	625
615	626	SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
@@ -626,9 +637,14 @@ SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) · 10 unmodified lines
626	637	* ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
627	638	* when returning from IPI handler.
628	639	*/
629		− INTERRUPT_RETURN
	640	+#ifdef CONFIG_XEN_PV
	641	+SYM_INNER_LABEL(early_xen_iret_patch, SYM_L_GLOBAL)
	642	+ ANNOTATE_NOENDBR
	643	+ .byte 0xe9
	644	+ .long .Lnative_iret - (. + 4)
	645	+#endif
630	646
631		−SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
	647	+.Lnative_iret:
632	648	UNWIND_HINT_IRET_REGS
633	649	/*
634	650	* Are we returning to a stack segment from the LDT? Note: in
@@ -640,6 +656,7 @@ SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) · 5 unmodified lines
640	656	#endif
641	657
642	658	SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
	659	+ ANNOTATE_NOENDBR // exc_double_fault
643	660	/*
644	661	* This may fault. Non-paranoid faults on return to userspace are
645	662	* handled by fixup_bad_iret. These include #SS, #GP, and #NP.
@@ -734,6 +751,7 @@ SYM_FUNC_START(asm_load_gs_index) · 88 unmodified lines
734	751	FRAME_BEGIN
735	752	swapgs
736	753	.Lgs_change:
	754	+ ANNOTATE_NOENDBR // error_entry
737	755	movl %edi, %gs
738	756	2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
739	757	swapgs
@@ -804,6 +822,7 @@ SYM_CODE_END(exc_xen_hypervisor_callback) · 64 unmodified lines
804	822	*/
805	823	SYM_CODE_START(xen_failsafe_callback)
806	824	UNWIND_HINT_EMPTY
	825	+ ENDBR
807	826	movl %ds, %ecx
808	827	cmpw %cx, 0x10(%rsp)
809	828	jne 1f
@@ -1063,6 +1082,7 @@ SYM_CODE_END(error_return) · 253 unmodified lines
1063	1082	*/
1064	1083	SYM_CODE_START(asm_exc_nmi)
1065	1084	UNWIND_HINT_IRET_REGS
	1085	+ ENDBR
1066	1086
1067	1087	/*
1068	1088	* We allow breakpoints in NMIs. If a breakpoint occurs, then
@@ -1310,6 +1330,7 @@ first_nmi: · 241 unmodified lines
1310	1330	#endif
1311	1331
1312	1332	repeat_nmi:
	1333	+ ANNOTATE_NOENDBR // this code
1313	1334	/*
1314	1335	* If there was a nested NMI, the first NMI's iret will return
1315	1336	* here. But NMIs are still enabled and we can take another
@@ -1338,6 +1359,7 @@ repeat_nmi: · 22 unmodified lines
1338	1359	.endr
1339	1360	subq $(5*8), %rsp
1340	1361	end_repeat_nmi:
	1362	+ ANNOTATE_NOENDBR // this code
1341	1363
1342	1364	/*
1343	1365	* Everything below this point can be preempted by a nested NMI.
@@ -1421,6 +1443,7 @@ SYM_CODE_END(asm_exc_nmi) · 77 unmodified lines
1421	1443	*/
1422	1444	SYM_CODE_START(ignore_sysret)
1423	1445	UNWIND_HINT_EMPTY
	1446	+ ENDBR
1424	1447	mov $-ENOSYS, %eax
1425	1448	sysretl
1426	1449	SYM_CODE_END(ignore_sysret)

arch/x86/entry/entry_64_compat.S+5−0

@@ -48,6 +48,7 @@ · 47 unmodified lines
48	48	*/
49	49	SYM_CODE_START(entry_SYSENTER_compat)
50	50	UNWIND_HINT_EMPTY
	51	+ ENDBR
51	52	/* Interrupts are off on entry. */
52	53	SWAPGS
53	54
@@ -147,6 +148,7 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) · 93 unmodified lines
147	148	popfq
148	149	jmp .Lsysenter_flags_fixed
149	150	SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL)
	151	+ ANNOTATE_NOENDBR // is_sysenter_singlestep
150	152	SYM_CODE_END(entry_SYSENTER_compat)
151	153
152	154	/*
@@ -198,6 +200,7 @@ SYM_CODE_END(entry_SYSENTER_compat) · 45 unmodified lines
198	200	*/
199	201	SYM_CODE_START(entry_SYSCALL_compat)
200	202	UNWIND_HINT_EMPTY
	203	+ ENDBR
201	204	/* Interrupts are off on entry. */
202	205	swapgs
203	206
@@ -211,6 +214,7 @@ SYM_CODE_START(entry_SYSCALL_compat) · 7 unmodified lines
211	214	movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
212	215
213	216	SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
	217	+ ANNOTATE_NOENDBR
214	218
215	219	/* Construct struct pt_regs on stack */
216	220	pushq $__USER32_DS /* pt_regs->ss */
@@ -340,6 +344,7 @@ SYM_CODE_END(entry_SYSCALL_compat) · 123 unmodified lines
340	344	*/
341	345	SYM_CODE_START(entry_INT80_compat)
342	346	UNWIND_HINT_EMPTY
	347	+ ENDBR
343	348	/*
344	349	* Interrupts are off on entry.
345	350	*/

arch/x86/entry/syscalls/Makefile+1−1

@@ -67,7 +67,7 @@ uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h · 66 unmodified lines
67	67	syshdr-y += syscalls_32.h
68	68	syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h
69	69	syshdr-$(CONFIG_X86_64) += syscalls_64.h
70		−syshdr-$(CONFIG_X86_X32) += syscalls_x32.h
	70	+syshdr-$(CONFIG_X86_X32_ABI) += syscalls_x32.h
71	71	syshdr-$(CONFIG_XEN) += xen-hypercalls.h
72	72
73	73	uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y))

arch/x86/include/asm/alternative.h+1−0

@@ -76,6 +76,7 @@ extern int alternatives_patched; · 75 unmodified lines
76	76	extern void alternative_instructions(void);
77	77	extern void apply_alternatives(struct alt_instr start, struct alt_instr end);
78	78	extern void apply_retpolines(s32 start, s32 end);
	79	+extern void apply_ibt_endbr(s32 start, s32 end);
79	80
80	81	struct module;
81	82

arch/x86/include/asm/bug.h+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4
5	5	#include <linux/stringify.h>
6	6	#include <linux/instrumentation.h>
	7	+#include <linux/objtool.h>
7	8
8	9	/*
9	10	* Despite that some emulators terminate on UD2, we use it for WARN().

arch/x86/include/asm/cpu.h+4−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/topology.h>
8	8	#include <linux/nodemask.h>
9	9	#include <linux/percpu.h>
	10	+#include <asm/ibt.h>
10	11
11	12	#ifdef CONFIG_SMP
12	13
@@ -72,4 +73,7 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c); · 59 unmodified lines
72	73	#else
73	74	static inline void init_ia32_feat_ctl(struct cpuinfo_x86 *c) {}
74	75	#endif
	76	+
	77	+extern __noendbr void cet_disable(void);
	78	+
75	79	#endif /* _ASM_X86_CPU_H */

arch/x86/include/asm/cpufeatures.h+1−0

@@ -388,6 +388,7 @@ · 387 unmodified lines
388	388	#define X86_FEATURE_TSXLDTRK (1832+16) / TSX Suspend Load Address Tracking */
389	389	#define X86_FEATURE_PCONFIG (1832+18) / Intel PCONFIG */
390	390	#define X86_FEATURE_ARCH_LBR (1832+19) / Intel ARCH LBR */
	391	+#define X86_FEATURE_IBT (1832+20) / Indirect Branch Tracking */
391	392	#define X86_FEATURE_AMX_BF16 (1832+22) / AMX bf16 Support */
392	393	#define X86_FEATURE_AVX512_FP16 (1832+23) / AVX512 FP16 */
393	394	#define X86_FEATURE_AMX_TILE (1832+24) / AMX tile Support */

arch/x86/include/asm/efi.h+7−2

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <asm/tlb.h>
8	8	#include <asm/nospec-branch.h>
9	9	#include <asm/mmu_context.h>
	10	+#include <asm/ibt.h>
10	11	#include <linux/build_bug.h>
11	12	#include <linux/kernel.h>
12	13	#include <linux/pgtable.h>
@@ -120,8 +121,12 @@ extern asmlinkage u64 __efi_call(void *fp, ...); · 107 unmodified lines
120	121	efi_enter_mm(); \
121	122	})
122	123
123		−#define arch_efi_call_virt(p, f, args...) \
124		− efi_call((void *)p->f, args) \
	124	+#define arch_efi_call_virt(p, f, args...) ({ \
	125	+ u64 ret, ibt = ibt_save(); \
	126	+ ret = efi_call((void *)p->f, args); \
	127	+ ibt_restore(ibt); \
	128	+ ret; \
	129	+})
125	130
126	131	#define arch_efi_call_virt_teardown() \
127	132	({ \

arch/x86/include/asm/ibt.h+105−0

@@ -0,0 +1,105 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef _ASM_X86_IBT_H
	3	+#define _ASM_X86_IBT_H
	4	+
	5	+#include <linux/types.h>
	6	+
	7	+/*
	8	+ * The rules for enabling IBT are:
	9	+ *
	10	+ * - CC_HAS_IBT: the toolchain supports it
	11	+ * - X86_KERNEL_IBT: it is selected in Kconfig
	12	+ * - !__DISABLE_EXPORTS: this is regular kernel code
	13	+ *
	14	+ * Esp. that latter one is a bit non-obvious, but some code like compressed,
	15	+ * purgatory, realmode etc.. is built with custom CFLAGS that do not include
	16	+ * -fcf-protection=branch and things will go bang.
	17	+ *
	18	+ * When all the above are satisfied, HAS_KERNEL_IBT will be 1, otherwise 0.
	19	+ */
	20	+#if defined(CONFIG_X86_KERNEL_IBT) && !defined(__DISABLE_EXPORTS)
	21	+
	22	+#define HAS_KERNEL_IBT 1
	23	+
	24	+#ifndef __ASSEMBLY__
	25	+
	26	+#ifdef CONFIG_X86_64
	27	+#define ASM_ENDBR "endbr64\n\t"
	28	+#else
	29	+#define ASM_ENDBR "endbr32\n\t"
	30	+#endif
	31	+
	32	+#define __noendbr __attribute__((nocf_check))
	33	+
	34	+static inline __attribute_const__ u32 gen_endbr(void)
	35	+{
	36	+ u32 endbr;
	37	+
	38	+ /*
	39	+ * Generate ENDBR64 in a way that is sure to not result in
	40	+ * an ENDBR64 instruction as immediate.
	41	+ */
	42	+ asm ( "mov $~0xfa1e0ff3, %[endbr]\n\t"
	43	+ "not %[endbr]\n\t"
	44	+ : [endbr] "=&r" (endbr) );
	45	+
	46	+ return endbr;
	47	+}
	48	+
	49	+static inline __attribute_const__ u32 gen_endbr_poison(void)
	50	+{
	51	+ /*
	52	+ * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
	53	+ * will be unique to (former) ENDBR sites.
	54	+ */
	55	+ return 0x001f0f66; /* osp nopl (%rax) */
	56	+}
	57	+
	58	+static inline bool is_endbr(u32 val)
	59	+{
	60	+ if (val == gen_endbr_poison())
	61	+ return true;
	62	+
	63	+ val &= ~0x01000000U; /* ENDBR32 -> ENDBR64 */
	64	+ return val == gen_endbr();
	65	+}
	66	+
	67	+extern __noendbr u64 ibt_save(void);
	68	+extern __noendbr void ibt_restore(u64 save);
	69	+
	70	+#else /* __ASSEMBLY__ */
	71	+
	72	+#ifdef CONFIG_X86_64
	73	+#define ENDBR endbr64
	74	+#else
	75	+#define ENDBR endbr32
	76	+#endif
	77	+
	78	+#endif /* __ASSEMBLY__ */
	79	+
	80	+#else /* !IBT */
	81	+
	82	+#define HAS_KERNEL_IBT 0
	83	+
	84	+#ifndef __ASSEMBLY__
	85	+
	86	+#define ASM_ENDBR
	87	+
	88	+#define __noendbr
	89	+
	90	+static inline bool is_endbr(u32 val) { return false; }
	91	+
	92	+static inline u64 ibt_save(void) { return 0; }
	93	+static inline void ibt_restore(u64 save) { }
	94	+
	95	+#else /* __ASSEMBLY__ */
	96	+
	97	+#define ENDBR
	98	+
	99	+#endif /* __ASSEMBLY__ */
	100	+
	101	+#endif /* CONFIG_X86_KERNEL_IBT */
	102	+
	103	+#define ENDBR_INSN_SIZE (4*HAS_KERNEL_IBT)
	104	+
	105	+#endif /* _ASM_X86_IBT_H */

arch/x86/include/asm/idtentry.h+16−9

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	5	/* Interrupts/Exceptions */
6	6	#include <asm/trapnr.h>
7	7
	8	+#define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT))
	9	+
8	10	#ifndef __ASSEMBLY__
9	11	#include <linux/entry-common.h>
10	12	#include <linux/hardirq.h>
@@ -480,7 +482,7 @@ __visible noinstr void func(struct pt_regs *regs, \ · 469 unmodified lines
480	482
481	483	/*
482	484	* ASM code to emit the common vector entry stubs where each stub is
483		− * packed into 8 bytes.
	485	+ * packed into IDT_ALIGN bytes.
484	486	*
485	487	* Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
486	488	* GCC treats the local vector variable as unsigned int and would expand
@@ -492,33 +494,33 @@ __visible noinstr void func(struct pt_regs *regs, \ · 5 unmodified lines
492	494	* point is to mask off the bits above bit 7 because the push is sign
493	495	* extending.
494	496	*/
495		− .align 8
	497	+ .align IDT_ALIGN
496	498	SYM_CODE_START(irq_entries_start)
497	499	vector=FIRST_EXTERNAL_VECTOR
498	500	.rept NR_EXTERNAL_VECTORS
499	501	UNWIND_HINT_IRET_REGS
500	502	0 :
	503	+ ENDBR
501	504	.byte 0x6a, vector
502	505	jmp asm_common_interrupt
503		− nop
504		− /* Ensure that the above is 8 bytes max */
505		− . = 0b + 8
	506	+ /* Ensure that the above is IDT_ALIGN bytes max */
	507	+ .fill 0b + IDT_ALIGN - ., 1, 0xcc
506	508	vector = vector+1
507	509	.endr
508	510	SYM_CODE_END(irq_entries_start)
509	511
510	512	#ifdef CONFIG_X86_LOCAL_APIC
511		− .align 8
	513	+ .align IDT_ALIGN
512	514	SYM_CODE_START(spurious_entries_start)
513	515	vector=FIRST_SYSTEM_VECTOR
514	516	.rept NR_SYSTEM_VECTORS
515	517	UNWIND_HINT_IRET_REGS
516	518	0 :
	519	+ ENDBR
517	520	.byte 0x6a, vector
518	521	jmp asm_spurious_interrupt
519		− nop
520		− /* Ensure that the above is 8 bytes max */
521		− . = 0b + 8
	522	+ /* Ensure that the above is IDT_ALIGN bytes max */
	523	+ .fill 0b + IDT_ALIGN - ., 1, 0xcc
522	524	vector = vector+1
523	525	.endr
524	526	SYM_CODE_END(spurious_entries_start)
@@ -615,6 +617,11 @@ DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); · 90 unmodified lines
615	617	DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault);
616	618	#endif
617	619
	620	+/* #CP */
	621	+#ifdef CONFIG_X86_KERNEL_IBT
	622	+DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);
	623	+#endif
	624	+
618	625	/* #VC */
619	626	#ifdef CONFIG_AMD_MEM_ENCRYPT
620	627	DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication);

arch/x86/include/asm/irq_stack.h+3−1

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#define _ASM_X86_IRQ_STACK_H
4	4
5	5	#include <linux/ptrace.h>
	6	+#include <linux/objtool.h>
6	7
7	8	#include <asm/processor.h>
8	9
@@ -99,7 +100,8 @@ · 90 unmodified lines
99	100	}
100	101
101	102	#define ASM_CALL_ARG0 \
102		− "call %P[__func] \n"
	103	+ "call %P[__func] \n" \
	104	+ ASM_REACHABLE
103	105
104	106	#define ASM_CALL_ARG1 \
105	107	"movq %[arg1], %%rdi \n" \

arch/x86/include/asm/irqflags.h+0−5

@@ -141,13 +141,8 @@ static __always_inline void arch_local_irq_restore(unsigned long flags) · 140 unmodified lines
141	141	#ifdef CONFIG_X86_64
142	142	#ifdef CONFIG_XEN_PV
143	143	#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV
144		−#define INTERRUPT_RETURN \
145		− ANNOTATE_RETPOLINE_SAFE; \
146		− ALTERNATIVE_TERNARY("jmp *paravirt_iret(%rip);", \
147		− X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")
148	144	#else
149	145	#define SWAPGS swapgs
150		−#define INTERRUPT_RETURN jmp native_iret
151	146	#endif
152	147	#endif
153	148	#endif /* !__ASSEMBLY__ */

arch/x86/include/asm/linkage.h+31−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#define _ASM_X86_LINKAGE_H
4	4
5	5	#include <linux/stringify.h>
	6	+#include <asm/ibt.h>
6	7
7	8	#undef notrace
8	9	#define notrace __attribute__((no_instrument_function))
@@ -34,5 +35,35 @@ · 25 unmodified lines
34	35
35	36	#endif /* __ASSEMBLY__ */
36	37
	38	+/* SYM_FUNC_START -- use for global functions */
	39	+#define SYM_FUNC_START(name) \
	40	+ SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
	41	+ ENDBR
	42	+
	43	+/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */
	44	+#define SYM_FUNC_START_NOALIGN(name) \
	45	+ SYM_START(name, SYM_L_GLOBAL, SYM_A_NONE) \
	46	+ ENDBR
	47	+
	48	+/* SYM_FUNC_START_LOCAL -- use for local functions */
	49	+#define SYM_FUNC_START_LOCAL(name) \
	50	+ SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \
	51	+ ENDBR
	52	+
	53	+/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */
	54	+#define SYM_FUNC_START_LOCAL_NOALIGN(name) \
	55	+ SYM_START(name, SYM_L_LOCAL, SYM_A_NONE) \
	56	+ ENDBR
	57	+
	58	+/* SYM_FUNC_START_WEAK -- use for weak functions */
	59	+#define SYM_FUNC_START_WEAK(name) \
	60	+ SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \
	61	+ ENDBR
	62	+
	63	+/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */
	64	+#define SYM_FUNC_START_WEAK_NOALIGN(name) \
	65	+ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \
	66	+ ENDBR
	67	+
37	68	#endif /* _ASM_X86_LINKAGE_H */
38	69

arch/x86/include/asm/msr-index.h+19−1

@@ -362,11 +362,29 @@ · 361 unmodified lines
362	362	#define MSR_ATOM_CORE_TURBO_RATIOS 0x0000066c
363	363	#define MSR_ATOM_CORE_TURBO_VIDS 0x0000066d
364	364
365		−
366	365	#define MSR_CORE_PERF_LIMIT_REASONS 0x00000690
367	366	#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0
368	367	#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1
369	368
	369	+/* Control-flow Enforcement Technology MSRs */
	370	+#define MSR_IA32_U_CET 0x000006a0 /* user mode cet */
	371	+#define MSR_IA32_S_CET 0x000006a2 /* kernel mode cet */
	372	+#define CET_SHSTK_EN BIT_ULL(0)
	373	+#define CET_WRSS_EN BIT_ULL(1)
	374	+#define CET_ENDBR_EN BIT_ULL(2)
	375	+#define CET_LEG_IW_EN BIT_ULL(3)
	376	+#define CET_NO_TRACK_EN BIT_ULL(4)
	377	+#define CET_SUPPRESS_DISABLE BIT_ULL(5)
	378	+#define CET_RESERVED (BIT_ULL(6) \| BIT_ULL(7) \| BIT_ULL(8) \| BIT_ULL(9))
	379	+#define CET_SUPPRESS BIT_ULL(10)
	380	+#define CET_WAIT_ENDBR BIT_ULL(11)
	381	+
	382	+#define MSR_IA32_PL0_SSP 0x000006a4 /* ring-0 shadow stack pointer */
	383	+#define MSR_IA32_PL1_SSP 0x000006a5 /* ring-1 shadow stack pointer */
	384	+#define MSR_IA32_PL2_SSP 0x000006a6 /* ring-2 shadow stack pointer */
	385	+#define MSR_IA32_PL3_SSP 0x000006a7 /* ring-3 shadow stack pointer */
	386	+#define MSR_IA32_INT_SSP_TAB 0x000006a8 /* exception shadow stack table */
	387	+
370	388	/* Hardware P state interface */
371	389	#define MSR_PPERF 0x0000064e
372	390	#define MSR_PERF_LIMIT_REASONS 0x0000064f

arch/x86/include/asm/paravirt.h+1−0

@@ -666,6 +666,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); · 665 unmodified lines
666	666	".globl " PV_THUNK_NAME(func) ";" \
667	667	".type " PV_THUNK_NAME(func) ", @function;" \
668	668	PV_THUNK_NAME(func) ":" \
	669	+ ASM_ENDBR \
669	670	FRAME_BEGIN \
670	671	PV_SAVE_ALL_CALLER_REGS \
671	672	"call " #func ";" \

arch/x86/include/asm/paravirt_types.h+0−1

@@ -272,7 +272,6 @@ struct paravirt_patch_template { · 271 unmodified lines
272	272
273	273	extern struct pv_info pv_info;
274	274	extern struct paravirt_patch_template pv_ops;
275		−extern void (*paravirt_iret)(void);
276	275
277	276	#define PARAVIRT_PATCH(x) \
278	277	(offsetof(struct paravirt_patch_template, x) / sizeof(void *))

arch/x86/include/asm/processor.h+1−1

@@ -837,7 +837,7 @@ bool xen_set_default_idle(void); · 836 unmodified lines
837	837	#define xen_set_default_idle 0
838	838	#endif
839	839
840		−void stop_this_cpu(void *dummy);
	840	+void __noreturn stop_this_cpu(void *dummy);
841	841	void microcode_check(void);
842	842
843	843	enum l1tf_mitigations {

arch/x86/include/asm/qspinlock_paravirt.h+3−0

@@ -2,6 +2,8 @@ · 1 unmodified line
2	2	#ifndef __ASM_QSPINLOCK_PARAVIRT_H
3	3	#define __ASM_QSPINLOCK_PARAVIRT_H
4	4
	5	+#include <asm/ibt.h>
	6	+
5	7	/*
6	8	* For x86-64, PV_CALLEE_SAVE_REGS_THUNK() saves and restores 8 64-bit
7	9	* registers. For i386, however, only 1 32-bit register needs to be saved
@@ -39,6 +41,7 @@ asm (".pushsection .text;" · 31 unmodified lines
39	41	".type " PV_UNLOCK ", @function;"
40	42	".align 4,0x90;"
41	43	PV_UNLOCK ": "
	44	+ ASM_ENDBR
42	45	FRAME_BEGIN
43	46	"push %rdx;"
44	47	"mov $0x1,%eax;"

arch/x86/include/asm/segment.h+3−2

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4
5	5	#include <linux/const.h>
6	6	#include <asm/alternative.h>
	7	+#include <asm/ibt.h>
7	8
8	9	/*
9	10	* Constructor for a conventional segment GDT (or LDT) entry.
@@ -275,7 +276,7 @@ static inline void vdso_read_cpunode(unsigned cpu, unsigned node) · 265 unmodified lines
275	276	* vector has no error code (two bytes), a 'push $vector_number' (two
276	277	* bytes), and a jump to the common entry code (up to five bytes).
277	278	*/
278		−#define EARLY_IDT_HANDLER_SIZE 9
	279	+#define EARLY_IDT_HANDLER_SIZE (9 + ENDBR_INSN_SIZE)
279	280
280	281	/*
281	282	* xen_early_idt_handler_array is for Xen pv guests: for each entry in
@@ -283,7 +284,7 @@ static inline void vdso_read_cpunode(unsigned cpu, unsigned node) · 1 unmodified line
283	284	* pop %rcx; pop %r11; jmp early_idt_handler_array[i]; summing up to
284	285	* max 8 bytes.
285	286	*/
286		−#define XEN_EARLY_IDT_HANDLER_SIZE 8
	287	+#define XEN_EARLY_IDT_HANDLER_SIZE (8 + ENDBR_INSN_SIZE)
287	288
288	289	#ifndef __ASSEMBLY__
289	290

arch/x86/include/asm/setup.h+2−1

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8
9	9	#include <linux/linkage.h>
10	10	#include <asm/page_types.h>
	11	+#include <asm/ibt.h>
11	12
12	13	#ifdef __i386__
13	14
@@ -119,7 +120,7 @@ void *extend_brk(size_t size, size_t align); · 105 unmodified lines
119	120	* executable.)
120	121	*/
121	122	#define RESERVE_BRK(name,sz) \
122		− static void __section(".discard.text") __used notrace \
	123	+ static void __section(".discard.text") __noendbr __used notrace \
123	124	__brk_reservation_fn_##name##__(void) { \
124	125	asm volatile ( \
125	126	".pushsection .brk_reservation,\"aw\",@nobits;" \

arch/x86/include/asm/syscall_wrapper.h+3−3

@@ -159,7 +159,7 @@ extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); · 158 unmodified lines
159	159	#endif /* CONFIG_IA32_EMULATION */
160	160
161	161
162		−#ifdef CONFIG_X86_X32
	162	+#ifdef CONFIG_X86_X32_ABI
163	163	/*
164	164	* For the x32 ABI, we need to create a stub for compat_sys_*() which is aware
165	165	* of the x86-64-style parameter ordering of x32 syscalls. The syscalls common
@@ -177,12 +177,12 @@ extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); · 11 unmodified lines
177	177
178	178	#define __X32_COMPAT_SYS_NI(name) \
179	179	__SYS_NI(x64, compat_sys_##name)
180		−#else /* CONFIG_X86_X32 */
	180	+#else /* CONFIG_X86_X32_ABI */
181	181	#define __X32_COMPAT_SYS_STUB0(name)
182	182	#define __X32_COMPAT_SYS_STUBx(x, name, ...)
183	183	#define __X32_COMPAT_COND_SYSCALL(name)
184	184	#define __X32_COMPAT_SYS_NI(name)
185		−#endif /* CONFIG_X86_X32 */
	185	+#endif /* CONFIG_X86_X32_ABI */
186	186
187	187
188	188	#ifdef CONFIG_COMPAT

arch/x86/include/asm/text-patching.h+23−7

@@ -97,24 +97,40 @@ union text_poke_insn { · 96 unmodified lines
97	97	};
98	98
99	99	static __always_inline
100		−void text_gen_insn(u8 opcode, const void addr, const void *dest)
	100	+void __text_gen_insn(void buf, u8 opcode, const void addr, const void *dest, int size)
101	101	{
102		− static union text_poke_insn insn; /* per instance */
103		− int size = text_opcode_size(opcode);
	102	+ union text_poke_insn *insn = buf;
	103	+
	104	+ BUG_ON(size < text_opcode_size(opcode));
	105	+
	106	+ /*
	107	+ * Hide the addresses to avoid the compiler folding in constants when
	108	+ * referencing code, these can mess up annotations like
	109	+ * ANNOTATE_NOENDBR.
	110	+ */
	111	+ OPTIMIZER_HIDE_VAR(insn);
	112	+ OPTIMIZER_HIDE_VAR(addr);
	113	+ OPTIMIZER_HIDE_VAR(dest);
104	114
105		− insn.opcode = opcode;
	115	+ insn->opcode = opcode;
106	116
107	117	if (size > 1) {
108		− insn.disp = (long)dest - (long)(addr + size);
	118	+ insn->disp = (long)dest - (long)(addr + size);
109	119	if (size == 2) {
110	120	/*
111		− * Ensure that for JMP9 the displacement
	121	+ * Ensure that for JMP8 the displacement
112	122	* actually fits the signed byte.
113	123	*/
114		− BUG_ON((insn.disp >> 31) != (insn.disp >> 7));
	124	+ BUG_ON((insn->disp >> 31) != (insn->disp >> 7));
115	125	}
116	126	}
	127	+}
117	128
	129	+static __always_inline
	130	+void text_gen_insn(u8 opcode, const void addr, const void *dest)
	131	+{
	132	+ static union text_poke_insn insn; /* per instance */
	133	+ __text_gen_insn(&insn, opcode, addr, dest, text_opcode_size(opcode));
118	134	return &insn.text;
119	135	}
120	136

arch/x86/include/asm/traps.h+2−0

@@ -18,6 +18,8 @@ void __init trap_init(void); · 17 unmodified lines
18	18	asmlinkage __visible noinstr struct pt_regs vc_switch_off_ist(struct pt_regs eregs);
19	19	#endif
20	20
	21	+extern bool ibt_selftest(void);
	22	+
21	23	#ifdef CONFIG_X86_F00F_BUG
22	24	/* For handling the FOOF bug */
23	25	void handle_invalid_op(struct pt_regs *regs);

arch/x86/include/asm/vdso.h+1−1

@@ -37,7 +37,7 @@ struct vdso_image { · 36 unmodified lines
37	37	extern const struct vdso_image vdso_image_64;
38	38	#endif
39	39
40		−#ifdef CONFIG_X86_X32
	40	+#ifdef CONFIG_X86_X32_ABI
41	41	extern const struct vdso_image vdso_image_x32;
42	42	#endif
43	43

arch/x86/include/uapi/asm/processor-flags.h+2−0

@@ -130,6 +130,8 @@ · 129 unmodified lines
130	130	#define X86_CR4_SMAP _BITUL(X86_CR4_SMAP_BIT)
131	131	#define X86_CR4_PKE_BIT 22 /* enable Protection Keys support */
132	132	#define X86_CR4_PKE _BITUL(X86_CR4_PKE_BIT)
	133	+#define X86_CR4_CET_BIT 23 /* enable Control-flow Enforcement Technology */
	134	+#define X86_CR4_CET _BITUL(X86_CR4_CET_BIT)
133	135
134	136	/*
135	137	* x86-64 Task Priority Register, CR8

arch/x86/kernel/alternative.c+52−14

@@ -115,6 +115,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len) · 114 unmodified lines
115	115	}
116	116
117	117	extern s32 __retpoline_sites[], __retpoline_sites_end[];
	118	+extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
118	119	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
119	120	extern s32 __smp_locks[], __smp_locks_end[];
120	121	void text_poke_early(void addr, const void opcode, size_t len);
@@ -512,6 +513,42 @@ void __init_or_module noinline apply_retpolines(s32 start, s32 end) { } · 391 unmodified lines
512	513
513	514	#endif /* CONFIG_RETPOLINE && CONFIG_STACK_VALIDATION */
514	515
	516	+#ifdef CONFIG_X86_KERNEL_IBT
	517	+
	518	+/*
	519	+ * Generated by: objtool --ibt
	520	+ */
	521	+void __init_or_module noinline apply_ibt_endbr(s32 start, s32 end)
	522	+{
	523	+ s32 *s;
	524	+
	525	+ for (s = start; s < end; s++) {
	526	+ u32 endbr, poison = gen_endbr_poison();
	527	+ void addr = (void )s + *s;
	528	+
	529	+ if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
	530	+ continue;
	531	+
	532	+ if (WARN_ON_ONCE(!is_endbr(endbr)))
	533	+ continue;
	534	+
	535	+ DPRINTK("ENDBR at: %pS (%px)", addr, addr);
	536	+
	537	+ /*
	538	+ * When we have IBT, the lack of ENDBR will trigger #CP
	539	+ */
	540	+ DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
	541	+ DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
	542	+ text_poke_early(addr, &poison, 4);
	543	+ }
	544	+}
	545	+
	546	+#else
	547	+
	548	+void __init_or_module noinline apply_ibt_endbr(s32 start, s32 end) { }
	549	+
	550	+#endif /* CONFIG_X86_KERNEL_IBT */
	551	+
515	552	#ifdef CONFIG_SMP
516	553	static void alternatives_smp_lock(const s32 start, const s32 end,
517	554	u8 text, u8 text_end)
@@ -713,34 +750,39 @@ asm ( · 195 unmodified lines
713	750	" .pushsection .init.text, \"ax\", @progbits\n"
714	751	" .type int3_magic, @function\n"
715	752	"int3_magic:\n"
	753	+ ANNOTATE_NOENDBR
716	754	" movl $1, (%" _ASM_ARG1 ")\n"
717	755	ASM_RET
718	756	" .size int3_magic, .-int3_magic\n"
719	757	" .popsection\n"
720	758	);
721	759
722		−extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
	760	+extern void int3_selftest_ip(void); /* defined in asm below */
723	761
724	762	static int __init
725	763	int3_exception_notify(struct notifier_block self, unsigned long val, void data)
726	764	{
	765	+ unsigned long selftest = (unsigned long)&int3_selftest_ip;
727	766	struct die_args *args = data;
728	767	struct pt_regs *regs = args->regs;
729	768
	769	+ OPTIMIZER_HIDE_VAR(selftest);
	770	+
730	771	if (!regs \|\| user_mode(regs))
731	772	return NOTIFY_DONE;
732	773
733	774	if (val != DIE_INT3)
734	775	return NOTIFY_DONE;
735	776
736		− if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
	777	+ if (regs->ip - INT3_INSN_SIZE != selftest)
737	778	return NOTIFY_DONE;
738	779
739	780	int3_emulate_call(regs, (unsigned long)&int3_magic);
740	781	return NOTIFY_STOP;
741	782	}
742	783
743		−static void __init int3_selftest(void)
	784	+/* Must be noinline to ensure uniqueness of int3_selftest_ip. */
	785	+static noinline void __init int3_selftest(void)
744	786	{
745	787	static __initdata struct notifier_block int3_exception_nb = {
746	788	.notifier_call = int3_exception_notify,
@@ -753,18 +795,12 @@ static void __init int3_selftest(void) · 6 unmodified lines
753	795	/*
754	796	* Basically: int3_magic(&val); but really complicated :-)
755	797	*
756		− * Stick the address of the INT3 instruction into int3_selftest_ip,
757		− * then trigger the INT3, padded with NOPs to match a CALL instruction
758		− * length.
	798	+ * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb
	799	+ * notifier above will emulate CALL for us.
759	800	*/
760		− asm volatile ("1: int3; nop; nop; nop; nop\n\t"
761		− ".pushsection .init.data,\"aw\"\n\t"
762		− ".align " __ASM_SEL(4, 8) "\n\t"
763		− ".type int3_selftest_ip, @object\n\t"
764		− ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
765		− "int3_selftest_ip:\n\t"
766		− __ASM_SEL(.long, .quad) " 1b\n\t"
767		− ".popsection\n\t"
	801	+ asm volatile ("int3_selftest_ip:\n\t"
	802	+ ANNOTATE_NOENDBR
	803	+ " int3; nop; nop; nop; nop\n\t"
768	804	: ASM_CALL_CONSTRAINT
769	805	: __ASM_SEL_RAW(a, D) (&val)
770	806	: "memory");
@@ -831,6 +867,8 @@ void __init alternative_instructions(void) · 60 unmodified lines
831	867	*/
832	868	apply_alternatives(__alt_instructions, __alt_instructions_end);
833	869
	870	+ apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
	871	+
834	872	#ifdef CONFIG_SMP
835	873	/* Patch to UP if other cpus not imminent. */
836	874	if (!noreplace_smp && (num_present_cpus() == 1 \|\| setup_max_cpus <= 1)) {

arch/x86/kernel/apm_32.c+7−0

@@ -232,6 +232,7 @@ · 231 unmodified lines
232	232	#include <asm/paravirt.h>
233	233	#include <asm/reboot.h>
234	234	#include <asm/nospec-branch.h>
	235	+#include <asm/ibt.h>
235	236
236	237	#if defined(CONFIG_APM_DISPLAY_BLANK) && defined(CONFIG_VT)
237	238	extern int (*console_blank_hook)(int);
@@ -598,6 +599,7 @@ static long __apm_bios_call(void *_call) · 360 unmodified lines
598	599	struct desc_struct save_desc_40;
599	600	struct desc_struct *gdt;
600	601	struct apm_bios_call *call = _call;
	602	+ u64 ibt;
601	603
602	604	cpu = get_cpu();
603	605	BUG_ON(cpu != 0);
@@ -607,11 +609,13 @@ static long __apm_bios_call(void *_call) · 3 unmodified lines
607	609
608	610	apm_irq_save(flags);
609	611	firmware_restrict_branch_speculation_start();
	612	+ ibt = ibt_save();
610	613	APM_DO_SAVE_SEGS;
611	614	apm_bios_call_asm(call->func, call->ebx, call->ecx,
612	615	&call->eax, &call->ebx, &call->ecx, &call->edx,
613	616	&call->esi);
614	617	APM_DO_RESTORE_SEGS;
	618	+ ibt_restore(ibt);
615	619	firmware_restrict_branch_speculation_end();
616	620	apm_irq_restore(flags);
617	621	gdt[0x40 / 8] = save_desc_40;
@@ -676,6 +680,7 @@ static long __apm_bios_call_simple(void *_call) · 58 unmodified lines
676	680	struct desc_struct save_desc_40;
677	681	struct desc_struct *gdt;
678	682	struct apm_bios_call *call = _call;
	683	+ u64 ibt;
679	684
680	685	cpu = get_cpu();
681	686	BUG_ON(cpu != 0);
@@ -685,10 +690,12 @@ static long __apm_bios_call_simple(void *_call) · 3 unmodified lines
685	690
686	691	apm_irq_save(flags);
687	692	firmware_restrict_branch_speculation_start();
	693	+ ibt = ibt_save();
688	694	APM_DO_SAVE_SEGS;
689	695	error = apm_bios_call_simple_asm(call->func, call->ebx, call->ecx,
690	696	&call->eax);
691	697	APM_DO_RESTORE_SEGS;
	698	+ ibt_restore(ibt);
692	699	firmware_restrict_branch_speculation_end();
693	700	apm_irq_restore(flags);
694	701	gdt[0x40 / 8] = save_desc_40;

arch/x86/kernel/cpu/common.c+58−1

@@ -59,6 +59,7 @@ · 58 unmodified lines
59	59	#include <asm/cpu_device_id.h>
60	60	#include <asm/uv/uv.h>
61	61	#include <asm/sigframe.h>
	62	+#include <asm/traps.h>
62	63
63	64	#include "cpu.h"
64	65
@@ -438,7 +439,8 @@ out: · 373 unmodified lines
438	439
439	440	/* These bits should not change their value after CPU init is finished. */
440	441	static const unsigned long cr4_pinned_mask =
441		− X86_CR4_SMEP \| X86_CR4_SMAP \| X86_CR4_UMIP \| X86_CR4_FSGSBASE;
	442	+ X86_CR4_SMEP \| X86_CR4_SMAP \| X86_CR4_UMIP \|
	443	+ X86_CR4_FSGSBASE \| X86_CR4_CET;
442	444	static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
443	445	static unsigned long cr4_pinned_bits __ro_after_init;
444	446
@@ -592,6 +594,58 @@ static __init int setup_disable_pku(char *arg) · 147 unmodified lines
592	594	__setup("nopku", setup_disable_pku);
593	595	#endif /* CONFIG_X86_64 */
594	596
	597	+#ifdef CONFIG_X86_KERNEL_IBT
	598	+
	599	+__noendbr u64 ibt_save(void)
	600	+{
	601	+ u64 msr = 0;
	602	+
	603	+ if (cpu_feature_enabled(X86_FEATURE_IBT)) {
	604	+ rdmsrl(MSR_IA32_S_CET, msr);
	605	+ wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
	606	+ }
	607	+
	608	+ return msr;
	609	+}
	610	+
	611	+__noendbr void ibt_restore(u64 save)
	612	+{
	613	+ u64 msr;
	614	+
	615	+ if (cpu_feature_enabled(X86_FEATURE_IBT)) {
	616	+ rdmsrl(MSR_IA32_S_CET, msr);
	617	+ msr &= ~CET_ENDBR_EN;
	618	+ msr \|= (save & CET_ENDBR_EN);
	619	+ wrmsrl(MSR_IA32_S_CET, msr);
	620	+ }
	621	+}
	622	+
	623	+#endif
	624	+
	625	+static __always_inline void setup_cet(struct cpuinfo_x86 *c)
	626	+{
	627	+ u64 msr = CET_ENDBR_EN;
	628	+
	629	+ if (!HAS_KERNEL_IBT \|\|
	630	+ !cpu_feature_enabled(X86_FEATURE_IBT))
	631	+ return;
	632	+
	633	+ wrmsrl(MSR_IA32_S_CET, msr);
	634	+ cr4_set_bits(X86_CR4_CET);
	635	+
	636	+ if (!ibt_selftest()) {
	637	+ pr_err("IBT selftest: Failed!\n");
	638	+ setup_clear_cpu_cap(X86_FEATURE_IBT);
	639	+ return;
	640	+ }
	641	+}
	642	+
	643	+__noendbr void cet_disable(void)
	644	+{
	645	+ if (cpu_feature_enabled(X86_FEATURE_IBT))
	646	+ wrmsrl(MSR_IA32_S_CET, 0);
	647	+}
	648	+
595	649	/*
596	650	* Some CPU features depend on higher CPUID levels, which may not always
597	651	* be available due to CPUID level capping or broken virtualization
@@ -1709,6 +1763,7 @@ static void identify_cpu(struct cpuinfo_x86 *c) · 1111 unmodified lines
1709	1763
1710	1764	x86_init_rdrand(c);
1711	1765	setup_pku(c);
	1766	+ setup_cet(c);
1712	1767
1713	1768	/*
1714	1769	* Clear/Set all flags overridden by options, need do it
@@ -1777,6 +1832,8 @@ void enable_sep_cpu(void) · 62 unmodified lines
1777	1832	void __init identify_boot_cpu(void)
1778	1833	{
1779	1834	identify_cpu(&boot_cpu_data);
	1835	+ if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
	1836	+ pr_info("CET detected: Indirect Branch Tracking enabled\n");
1780	1837	#ifdef CONFIG_X86_32
1781	1838	sysenter_setup();
1782	1839	enable_sep_cpu();

arch/x86/kernel/ftrace.c+2−7

@@ -316,12 +316,12 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 315 unmodified lines
316	316	unsigned long offset;
317	317	unsigned long npages;
318	318	unsigned long size;
319		− unsigned long retq;
320	319	unsigned long *ptr;
321	320	void *trampoline;
322	321	void *ip;
323	322	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
324	323	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
	324	+ unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
325	325	union ftrace_op_code_union op_ptr;
326	326	int ret;
327	327
@@ -359,12 +359,7 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 31 unmodified lines
359	359	goto fail;
360	360
361	361	ip = trampoline + size;
362		−
363		− /* The trampoline ends with ret(q) */
364		− retq = (unsigned long)ftrace_stub;
365		− ret = copy_from_kernel_nofault(ip, (void *)retq, RET_SIZE);
366		− if (WARN_ON(ret < 0))
367		− goto fail;
	362	+ memcpy(ip, retq, RET_SIZE);
368	363
369	364	/* No need to test direct calls on created trampolines */
370	365	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {

arch/x86/kernel/ftrace_64.S+24−4

@@ -145,6 +145,7 @@ SYM_FUNC_START(ftrace_caller) · 144 unmodified lines
145	145	movq %rcx, RSP(%rsp)
146	146
147	147	SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
	148	+ ANNOTATE_NOENDBR
148	149	/* Load the ftrace_ops into the 3rd parameter */
149	150	movq function_trace_op(%rip), %rdx
150	151
@@ -155,6 +156,7 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) · 4 unmodified lines
155	156	movq $0, CS(%rsp)
156	157
157	158	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
	159	+ ANNOTATE_NOENDBR
158	160	call ftrace_stub
159	161
160	162	/* Handlers can change the RIP */
@@ -169,6 +171,7 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) · 8 unmodified lines
169	171	* layout here.
170	172	*/
171	173	SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
	174	+ ANNOTATE_NOENDBR
172	175
173	176	jmp ftrace_epilogue
174	177	SYM_FUNC_END(ftrace_caller);
@@ -176,10 +179,10 @@ SYM_FUNC_END(ftrace_caller); · 1 unmodified line
176	179	SYM_FUNC_START(ftrace_epilogue)
177	180	/*
178	181	* This is weak to keep gas from relaxing the jumps.
179		− * It is also used to copy the RET for trampolines.
180	182	*/
181	183	SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
182	184	UNWIND_HINT_FUNC
	185	+ ENDBR
183	186	RET
184	187	SYM_FUNC_END(ftrace_epilogue)
185	188
@@ -192,6 +195,7 @@ SYM_FUNC_START(ftrace_regs_caller) · 6 unmodified lines
192	195	/* save_mcount_regs fills in first two parameters */
193	196
194	197	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
	198	+ ANNOTATE_NOENDBR
195	199	/* Load the ftrace_ops into the 3rd parameter */
196	200	movq function_trace_op(%rip), %rdx
197	201
@@ -221,6 +225,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) · 23 unmodified lines
221	225	leaq (%rsp), %rcx
222	226
223	227	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
	228	+ ANNOTATE_NOENDBR
224	229	call ftrace_stub
225	230
226	231	/* Copy flags back to SS, to restore them */
@@ -248,6 +253,7 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 21 unmodified lines
248	253	*/
249	254	testq %rax, %rax
250	255	SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)
	256	+ ANNOTATE_NOENDBR
251	257	jnz 1f
252	258
253	259	restore_mcount_regs
@@ -261,6 +267,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL) · 7 unmodified lines
261	267	* to the return.
262	268	*/
263	269	SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
	270	+ ANNOTATE_NOENDBR
264	271	jmp ftrace_epilogue
265	272
266	273	/* Swap the flags with orig_rax */
@@ -284,6 +291,7 @@ SYM_FUNC_START(__fentry__) · 17 unmodified lines
284	291	jnz trace
285	292
286	293	SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
	294	+ ENDBR
287	295	RET
288	296
289	297	trace:
@@ -307,7 +315,7 @@ EXPORT_SYMBOL(__fentry__) · 17 unmodified lines
307	315
308	316	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
309	317	SYM_FUNC_START(return_to_handler)
310		− subq $24, %rsp
	318	+ subq $16, %rsp
311	319
312	320	/* Save the return values */
313	321	movq %rax, (%rsp)
@@ -319,7 +327,19 @@ SYM_FUNC_START(return_to_handler) · 5 unmodified lines
319	327	movq %rax, %rdi
320	328	movq 8(%rsp), %rdx
321	329	movq (%rsp), %rax
322		− addq $24, %rsp
323		− JMP_NOSPEC rdi
	330	+
	331	+ addq $16, %rsp
	332	+ /*
	333	+ * Jump back to the old return address. This cannot be JMP_NOSPEC rdi
	334	+ * since IBT would demand that contain ENDBR, which simply isn't so for
	335	+ * return addresses. Use a retpoline here to keep the RSB balanced.
	336	+ */
	337	+ ANNOTATE_INTRA_FUNCTION_CALL
	338	+ call .Ldo_rop
	339	+ int3
	340	+.Ldo_rop:
	341	+ mov %rdi, (%rsp)
	342	+ UNWIND_HINT_FUNC
	343	+ RET
324	344	SYM_FUNC_END(return_to_handler)
325	345	#endif

arch/x86/kernel/head_64.S+12−2

@@ -99,6 +99,7 @@ SYM_CODE_END(startup_64) · 98 unmodified lines
99	99
100	100	SYM_CODE_START(secondary_startup_64)
101	101	UNWIND_HINT_EMPTY
	102	+ ANNOTATE_NOENDBR
102	103	/*
103	104	* At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
104	105	* and someone has loaded a mapped page table.
@@ -127,6 +128,7 @@ SYM_CODE_START(secondary_startup_64) · 22 unmodified lines
127	128	*/
128	129	SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
129	130	UNWIND_HINT_EMPTY
	131	+ ANNOTATE_NOENDBR
130	132
131	133	/*
132	134	* Retrieve the modifier (SME encryption mask if SME is active) to be
@@ -192,6 +194,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) · 59 unmodified lines
192	194	jmp *%rax
193	195	1:
194	196	UNWIND_HINT_EMPTY
	197	+ ANNOTATE_NOENDBR // above
195	198
196	199	/*
197	200	* We must switch to a new descriptor in kernel space for the GDT
@@ -299,6 +302,7 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL) · 101 unmodified lines
299	302	pushq %rax # target address in negative space
300	303	lretq
301	304	.Lafter_lret:
	305	+ ANNOTATE_NOENDBR
302	306	SYM_CODE_END(secondary_startup_64)
303	307
304	308	#include "verify_cpu.S"
@@ -328,6 +332,7 @@ SYM_CODE_END(start_cpu0) · 23 unmodified lines
328	332	*/
329	333	SYM_CODE_START_NOALIGN(vc_boot_ghcb)
330	334	UNWIND_HINT_IRET_REGS offset=8
	335	+ ENDBR
331	336
332	337	/* Build pt_regs */
333	338	PUSH_AND_CLEAR_REGS
@@ -345,7 +350,6 @@ SYM_CODE_START_NOALIGN(vc_boot_ghcb) · 11 unmodified lines
345	350	/* Remove Error Code */
346	351	addq $8, %rsp
347	352
348		− /* Pure iret required here - don't use INTERRUPT_RETURN */
349	353	iretq
350	354	SYM_CODE_END(vc_boot_ghcb)
351	355	#endif
@@ -372,9 +376,11 @@ SYM_CODE_START(early_idt_handler_array) · 20 unmodified lines
372	376	.rept NUM_EXCEPTION_VECTORS
373	377	.if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
374	378	UNWIND_HINT_IRET_REGS
	379	+ ENDBR
375	380	pushq $0 # Dummy error code, to make stack frame uniform
376	381	.else
377	382	UNWIND_HINT_IRET_REGS offset=8
	383	+ ENDBR
378	384	.endif
379	385	pushq $i # 72(%rsp) Vector number
380	386	jmp early_idt_handler_common
@@ -382,10 +388,11 @@ SYM_CODE_START(early_idt_handler_array) · 1 unmodified line
382	388	i = i + 1
383	389	.fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
384	390	.endr
385		− UNWIND_HINT_IRET_REGS offset=16
386	391	SYM_CODE_END(early_idt_handler_array)
	392	+ ANNOTATE_NOENDBR // early_idt_handler_array[NUM_EXCEPTION_VECTORS]
387	393
388	394	SYM_CODE_START_LOCAL(early_idt_handler_common)
	395	+ UNWIND_HINT_IRET_REGS offset=16
389	396	/*
390	397	* The stack is the hardware frame, an error code or zero, and the
391	398	* vector number.
@@ -426,11 +433,14 @@ SYM_CODE_END(early_idt_handler_common) · 34 unmodified lines
426	433	* early_idt_handler_array can't be used because it returns via the
427	434	* paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
428	435	*
	436	+ * XXX it does, fix this.
	437	+ *
429	438	* This handler will end up in the .init.text section and not be
430	439	* available to boot secondary CPUs.
431	440	*/
432	441	SYM_CODE_START_NOALIGN(vc_no_ghcb)
433	442	UNWIND_HINT_IRET_REGS offset=8
	443	+ ENDBR
434	444
435	445	/* Build pt_regs */
436	446	PUSH_AND_CLEAR_REGS

arch/x86/kernel/idt.c+7−2

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <asm/proto.h>
11	11	#include <asm/desc.h>
12	12	#include <asm/hw_irq.h>
	13	+#include <asm/idtentry.h>
13	14
14	15	#define DPL0 0x0
15	16	#define DPL3 0x3
@@ -103,6 +104,10 @@ static const __initconst struct idt_data def_idts[] = { · 87 unmodified lines
103	104	ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
104	105	#endif
105	106
	107	+#ifdef CONFIG_X86_KERNEL_IBT
	108	+ INTG(X86_TRAP_CP, asm_exc_control_protection),
	109	+#endif
	110	+
106	111	#ifdef CONFIG_AMD_MEM_ENCRYPT
107	112	ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),
108	113	#endif
@@ -272,7 +277,7 @@ void __init idt_setup_apic_and_irq_gates(void) · 163 unmodified lines
272	277	idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
273	278
274	279	for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
275		− entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
	280	+ entry = irq_entries_start + IDT_ALIGN * (i - FIRST_EXTERNAL_VECTOR);
276	281	set_intr_gate(i, entry);
277	282	}
278	283
@@ -283,7 +288,7 @@ void __init idt_setup_apic_and_irq_gates(void) · 4 unmodified lines
283	288	* system_vectors bitmap. Otherwise they show up in
284	289	* /proc/interrupts.
285	290	*/
286		− entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
	291	+ entry = spurious_entries_start + IDT_ALIGN * (i - FIRST_SYSTEM_VECTOR);
287	292	set_intr_gate(i, entry);
288	293	}
289	294	#endif

arch/x86/kernel/kprobes/core.c+20−9

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	52	#include <asm/insn.h>
53	53	#include <asm/debugreg.h>
54	54	#include <asm/set_memory.h>
	55	+#include <asm/ibt.h>
55	56
56	57	#include "common.h"
57	58
@@ -193,17 +194,10 @@ static unsigned long · 135 unmodified lines
193	194	__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
194	195	{
195	196	struct kprobe *kp;
196		− unsigned long faddr;
	197	+ bool faddr;
197	198
198	199	kp = get_kprobe((void *)addr);
199		− faddr = ftrace_location(addr);
200		− /*
201		− * Addresses inside the ftrace location are refused by
202		− * arch_check_ftrace_location(). Something went terribly wrong
203		− * if such an address is checked here.
204		− */
205		− if (WARN_ON(faddr && faddr != addr))
206		− return 0UL;
	200	+ faddr = ftrace_location(addr) == addr;
207	201	/*
208	202	* Use the current code if it is not modified by Kprobe
209	203	* and it cannot be modified by ftrace.
@@ -301,6 +295,22 @@ static int can_probe(unsigned long paddr) · 91 unmodified lines
301	295	return (addr == paddr);
302	296	}
303	297
	298	+/* If x86 supports IBT (ENDBR) it must be skipped. */
	299	+kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
	300	+ bool *on_func_entry)
	301	+{
	302	+ if (is_endbr((u32 )addr)) {
	303	+ *on_func_entry = !offset \|\| offset == 4;
	304	+ if (*on_func_entry)
	305	+ offset = 4;
	306	+
	307	+ } else {
	308	+ *on_func_entry = !offset;
	309	+ }
	310	+
	311	+ return (kprobe_opcode_t *)(addr + offset);
	312	+}
	313	+
304	314	/*
305	315	* Copy an instruction with recovering modified instruction by kprobes
306	316	* and adjust the displacement if the instruction uses the %rip-relative
@@ -1023,6 +1033,7 @@ asm( · 716 unmodified lines
1023	1033	".type __kretprobe_trampoline, @function\n"
1024	1034	"__kretprobe_trampoline:\n"
1025	1035	#ifdef CONFIG_X86_64
	1036	+ ANNOTATE_NOENDBR
1026	1037	/* Push a fake return address to tell the unwinder it's a kretprobe. */
1027	1038	" pushq $__kretprobe_trampoline\n"
1028	1039	UNWIND_HINT_FUNC

arch/x86/kernel/kvm.c+2−1

@@ -1029,10 +1029,11 @@ asm( · 1028 unmodified lines
1029	1029	".global __raw_callee_save___kvm_vcpu_is_preempted;"
1030	1030	".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
1031	1031	"__raw_callee_save___kvm_vcpu_is_preempted:"
	1032	+ASM_ENDBR
1032	1033	"movq __per_cpu_offset(,%rdi,8), %rax;"
1033	1034	"cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
1034	1035	"setne %al;"
1035		−"ret;"
	1036	+ASM_RET
1036	1037	".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
1037	1038	".popsection");
1038	1039

arch/x86/kernel/machine_kexec_64.c+3−1

@@ -27,6 +27,7 @@ · 26 unmodified lines
27	27	#include <asm/kexec-bzimage64.h>
28	28	#include <asm/setup.h>
29	29	#include <asm/set_memory.h>
	30	+#include <asm/cpu.h>
30	31
31	32	#ifdef CONFIG_ACPI
32	33	/*
@@ -310,6 +311,7 @@ void machine_kexec(struct kimage *image) · 277 unmodified lines
310	311	/* Interrupts aren't acceptable while we reboot */
311	312	local_irq_disable();
312	313	hw_breakpoint_disable();
	314	+ cet_disable();
313	315
314	316	if (image->preserve_context) {
315	317	#ifdef CONFIG_X86_IO_APIC
@@ -325,7 +327,7 @@ void machine_kexec(struct kimage *image) · 9 unmodified lines
325	327	}
326	328
327	329	control_page = page_address(image->control_code_page) + PAGE_SIZE;
328		− memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
	330	+ __memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE);
329	331
330	332	page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page);
331	333	page_list[VA_CONTROL_PAGE] = (unsigned long)control_page;

arch/x86/kernel/module.c+7−1

@@ -253,7 +253,7 @@ int module_finalize(const Elf_Ehdr *hdr, · 252 unmodified lines
253	253	{
254	254	const Elf_Shdr s, text = NULL, alt = NULL, locks = NULL,
255	255	para = NULL, orc = NULL, *orc_ip = NULL,
256		− *retpolines = NULL;
	256	+ retpolines = NULL, ibt_endbr = NULL;
257	257	char secstrings = (void )hdr + sechdrs[hdr->e_shstrndx].sh_offset;
258	258
259	259	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -271,6 +271,8 @@ int module_finalize(const Elf_Ehdr *hdr, · 11 unmodified lines
271	271	orc_ip = s;
272	272	if (!strcmp(".retpoline_sites", secstrings + s->sh_name))
273	273	retpolines = s;
	274	+ if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
	275	+ ibt_endbr = s;
274	276	}
275	277
276	278	/*
@@ -290,6 +292,10 @@ int module_finalize(const Elf_Ehdr *hdr, · 13 unmodified lines
290	292	void aseg = (void )alt->sh_addr;
291	293	apply_alternatives(aseg, aseg + alt->sh_size);
292	294	}
	295	+ if (ibt_endbr) {
	296	+ void iseg = (void )ibt_endbr->sh_addr;
	297	+ apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);
	298	+ }
293	299	if (locks && text) {
294	300	void lseg = (void )locks->sh_addr;
295	301	void tseg = (void )text->sh_addr;

arch/x86/kernel/paravirt.c+5−24

@@ -41,6 +41,7 @@ extern void _paravirt_nop(void); · 40 unmodified lines
41	41	asm (".pushsection .entry.text, \"ax\"\n"
42	42	".global _paravirt_nop\n"
43	43	"_paravirt_nop:\n\t"
	44	+ ASM_ENDBR
44	45	ASM_RET
45	46	".size _paravirt_nop, . - _paravirt_nop\n\t"
46	47	".type _paravirt_nop, @function\n\t"
@@ -50,6 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n" · 3 unmodified lines
50	51	asm (".pushsection .entry.text, \"ax\"\n"
51	52	".global paravirt_ret0\n"
52	53	"paravirt_ret0:\n\t"
	54	+ ASM_ENDBR
53	55	"xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
54	56	ASM_RET
55	57	".size paravirt_ret0, . - paravirt_ret0\n\t"
@@ -69,29 +71,12 @@ noinstr void paravirt_BUG(void) · 13 unmodified lines
69	71	BUG();
70	72	}
71	73
72		−struct branch {
73		− unsigned char opcode;
74		− u32 delta;
75		−} __attribute__((packed));
76		−
77	74	static unsigned paravirt_patch_call(void insn_buff, const void target,
78	75	unsigned long addr, unsigned len)
79	76	{
80		− const int call_len = 5;
81		− struct branch *b = insn_buff;
82		− unsigned long delta = (unsigned long)target - (addr+call_len);
83		−
84		− if (len < call_len) {
85		− pr_warn("paravirt: Failed to patch indirect CALL at %ps\n", (void *)addr);
86		− /* Kernel might not be viable if patching fails, bail out: */
87		− BUG_ON(1);
88		− }
89		−
90		− b->opcode = 0xe8; /* call */
91		− b->delta = delta;
92		− BUILD_BUG_ON(sizeof(*b) != call_len);
93		−
94		− return call_len;
	77	+ __text_gen_insn(insn_buff, CALL_INSN_OPCODE,
	78	+ (void *)addr, target, CALL_INSN_SIZE);
	79	+ return CALL_INSN_SIZE;
95	80	}
96	81
97	82	#ifdef CONFIG_PARAVIRT_XXL
@@ -149,8 +134,6 @@ void paravirt_set_sched_clock(u64 (*func)(void)) · 51 unmodified lines
149	134	}
150	135
151	136	/* These are in entry.S */
152		−extern void native_iret(void);
153		−
154	137	static struct resource reserve_ioports = {
155	138	.start = 0,
156	139	.end = IO_SPACE_LIMIT,
@@ -414,8 +397,6 @@ struct paravirt_patch_template pv_ops = { · 257 unmodified lines
414	397
415	398	#ifdef CONFIG_PARAVIRT_XXL
416	399	NOKPROBE_SYMBOL(native_load_idt);
417		−
418		−void (*paravirt_iret)(void) = native_iret;
419	400	#endif
420	401
421	402	EXPORT_SYMBOL(pv_ops);

arch/x86/kernel/process.c+1−1

@@ -747,7 +747,7 @@ bool xen_set_default_idle(void) · 746 unmodified lines
747	747	}
748	748	#endif
749	749
750		−void stop_this_cpu(void *dummy)
	750	+void __noreturn stop_this_cpu(void *dummy)
751	751	{
752	752	local_irq_disable();
753	753	/*

arch/x86/kernel/process_64.c+1−1

@@ -681,7 +681,7 @@ void set_personality_64bit(void) · 680 unmodified lines
681	681
682	682	static void __set_personality_x32(void)
683	683	{
684		−#ifdef CONFIG_X86_X32
	684	+#ifdef CONFIG_X86_X32_ABI
685	685	if (current->mm)
686	686	current->mm->context.flags = 0;
687	687

arch/x86/kernel/relocate_kernel_64.S+10−0

@@ -42,6 +42,7 @@ · 41 unmodified lines
42	42	.code64
43	43	SYM_CODE_START_NOALIGN(relocate_kernel)
44	44	UNWIND_HINT_EMPTY
	45	+ ANNOTATE_NOENDBR
45	46	/*
46	47	* %rdi indirection_page
47	48	* %rsi page_list
@@ -114,6 +115,14 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) · 66 unmodified lines
114	115	/* store the start address on the stack */
115	116	pushq %rdx
116	117
	118	+ /*
	119	+ * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
	120	+ * below.
	121	+ */
	122	+ movq %cr4, %rax
	123	+ andq $~(X86_CR4_CET), %rax
	124	+ movq %rax, %cr4
	125	+
117	126	/*
118	127	* Set cr0 to a known state:
119	128	* - Paging enabled
@@ -215,6 +224,7 @@ SYM_CODE_END(identity_mapped) · 95 unmodified lines
215	224
216	225	SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
217	226	UNWIND_HINT_EMPTY
	227	+ ANNOTATE_NOENDBR // RET target, above
218	228	movq RSP(%r8), %rsp
219	229	movq CR4(%r8), %rax
220	230	movq %rax, %cr4

arch/x86/kernel/traps.c+75−0

@@ -210,6 +210,81 @@ DEFINE_IDTENTRY(exc_overflow) · 209 unmodified lines
210	210	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);
211	211	}
212	212
	213	+#ifdef CONFIG_X86_KERNEL_IBT
	214	+
	215	+static __ro_after_init bool ibt_fatal = true;
	216	+
	217	+extern void ibt_selftest_ip(void); /* code label defined in asm below */
	218	+
	219	+enum cp_error_code {
	220	+ CP_EC = (1 << 15) - 1,
	221	+
	222	+ CP_RET = 1,
	223	+ CP_IRET = 2,
	224	+ CP_ENDBR = 3,
	225	+ CP_RSTRORSSP = 4,
	226	+ CP_SETSSBSY = 5,
	227	+
	228	+ CP_ENCL = 1 << 15,
	229	+};
	230	+
	231	+DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
	232	+{
	233	+ if (!cpu_feature_enabled(X86_FEATURE_IBT)) {
	234	+ pr_err("Unexpected #CP\n");
	235	+ BUG();
	236	+ }
	237	+
	238	+ if (WARN_ON_ONCE(user_mode(regs) \|\| (error_code & CP_EC) != CP_ENDBR))
	239	+ return;
	240	+
	241	+ if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {
	242	+ regs->ax = 0;
	243	+ return;
	244	+ }
	245	+
	246	+ pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
	247	+ if (!ibt_fatal) {
	248	+ printk(KERN_DEFAULT CUT_HERE);
	249	+ __warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
	250	+ return;
	251	+ }
	252	+ BUG();
	253	+}
	254	+
	255	+/* Must be noinline to ensure uniqueness of ibt_selftest_ip. */
	256	+noinline bool ibt_selftest(void)
	257	+{
	258	+ unsigned long ret;
	259	+
	260	+ asm (" lea ibt_selftest_ip(%%rip), %%rax\n\t"
	261	+ ANNOTATE_RETPOLINE_SAFE
	262	+ " jmp *%%rax\n\t"
	263	+ "ibt_selftest_ip:\n\t"
	264	+ UNWIND_HINT_FUNC
	265	+ ANNOTATE_NOENDBR
	266	+ " nop\n\t"
	267	+
	268	+ : "=a" (ret) : : "memory");
	269	+
	270	+ return !ret;
	271	+}
	272	+
	273	+static int __init ibt_setup(char *str)
	274	+{
	275	+ if (!strcmp(str, "off"))
	276	+ setup_clear_cpu_cap(X86_FEATURE_IBT);
	277	+
	278	+ if (!strcmp(str, "warn"))
	279	+ ibt_fatal = false;
	280	+
	281	+ return 1;
	282	+}
	283	+
	284	+__setup("ibt=", ibt_setup);
	285	+
	286	+#endif /* CONFIG_X86_KERNEL_IBT */
	287	+
213	288	#ifdef CONFIG_X86_F00F_BUG
214	289	void handle_invalid_op(struct pt_regs *regs)
215	290	#else

arch/x86/kernel/vmlinux.lds.S+9−0

@@ -285,6 +285,15 @@ SECTIONS · 284 unmodified lines
285	285	}
286	286	#endif
287	287
	288	+#ifdef CONFIG_X86_KERNEL_IBT
	289	+ . = ALIGN(8);
	290	+ .ibt_endbr_seal : AT(ADDR(.ibt_endbr_seal) - LOAD_OFFSET) {
	291	+ __ibt_endbr_seal = .;
	292	+ *(.ibt_endbr_seal)
	293	+ __ibt_endbr_seal_end = .;
	294	+ }
	295	+#endif
	296	+
288	297	/*
289	298	* struct alt_inst entries. From the header (alternative.h):
290	299	* "Alternative instructions for different CPU types or capabilities"

arch/x86/kvm/emulate.c+10−6

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include <linux/stringify.h>
25	25	#include <asm/debugreg.h>
26	26	#include <asm/nospec-branch.h>
	27	+#include <asm/ibt.h>
27	28
28	29	#include "x86.h"
29	30	#include "tss.h"
@@ -189,7 +190,7 @@ · 159 unmodified lines
189	190	#define X16(x...) X8(x), X8(x)
190	191
191	192	#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192		−#define FASTOP_SIZE 8
	193	+#define FASTOP_SIZE (8 * (1 + HAS_KERNEL_IBT))
193	194
194	195	struct opcode {
195	196	u64 flags;
@@ -311,7 +312,8 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 115 unmodified lines
311	312	#define __FOP_FUNC(name) \
312	313	".align " __stringify(FASTOP_SIZE) " \n\t" \
313	314	".type " name ", @function \n\t" \
314		− name ":\n\t"
	315	+ name ":\n\t" \
	316	+ ASM_ENDBR
315	317
316	318	#define FOP_FUNC(name) \
317	319	__FOP_FUNC(#name)
@@ -433,21 +435,23 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 115 unmodified lines
433	435	/*
434	436	* Depending on .config the SETcc functions look like:
435	437	*
	438	+ * ENDBR [4 bytes; CONFIG_X86_KERNEL_IBT]
436	439	* SETcc %al [3 bytes]
437	440	* RET [1 byte]
438	441	* INT3 [1 byte; CONFIG_SLS]
439	442	*
440		− * Which gives possible sizes 4 or 5. When rounded up to the
441		− * next power-of-two alignment they become 4 or 8.
	443	+ * Which gives possible sizes 4, 5, 8 or 9. When rounded up to the
	444	+ * next power-of-two alignment they become 4, 8 or 16 resp.
442	445	*/
443		−#define SETCC_LENGTH (4 + IS_ENABLED(CONFIG_SLS))
444		−#define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS))
	446	+#define SETCC_LENGTH (ENDBR_INSN_SIZE + 4 + IS_ENABLED(CONFIG_SLS))
	447	+#define SETCC_ALIGN (4 << IS_ENABLED(CONFIG_SLS) << HAS_KERNEL_IBT)
445	448	static_assert(SETCC_LENGTH <= SETCC_ALIGN);
446	449
447	450	#define FOP_SETCC(op) \
448	451	".align " __stringify(SETCC_ALIGN) " \n\t" \
449	452	".type " #op ", @function \n\t" \
450	453	#op ": \n\t" \
	454	+ ASM_ENDBR \
451	455	#op " %al \n\t" \
452	456	__FOP_RET(#op)
453	457

arch/x86/lib/error-inject.c+2−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/linkage.h>
4	4	#include <linux/error-injection.h>
5	5	#include <linux/kprobes.h>
	6	+#include <linux/objtool.h>
6	7
7	8	asmlinkage void just_return_func(void);
8	9
@@ -11,6 +12,7 @@ asm( · 2 unmodified lines
11	12	".type just_return_func, @function\n"
12	13	".globl just_return_func\n"
13	14	"just_return_func:\n"
	15	+ ANNOTATE_NOENDBR
14	16	ASM_RET
15	17	".size just_return_func, .-just_return_func\n"
16	18	);

arch/x86/lib/retpoline.S+1−0

@@ -55,6 +55,7 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) · 54 unmodified lines
55	55
56	56	.align RETPOLINE_THUNK_SIZE
57	57	SYM_CODE_START(__x86_indirect_thunk_array)
	58	+ ANNOTATE_NOENDBR // apply_retpolines
58	59
59	60	#define GEN(reg) THUNK reg
60	61	#include <asm/GEN-for-each-reg.h>

arch/x86/net/bpf_jit_comp.c+24−2

@@ -46,6 +46,12 @@ static u8 emit_code(u8 ptr, u32 bytes, unsigned int len) · 45 unmodified lines
46	46	#define EMIT4_off32(b1, b2, b3, b4, off) \
47	47	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
48	48
	49	+#ifdef CONFIG_X86_KERNEL_IBT
	50	+#define EMIT_ENDBR() EMIT(gen_endbr(), 4)
	51	+#else
	52	+#define EMIT_ENDBR()
	53	+#endif
	54	+
49	55	static bool is_imm8(int value)
50	56	{
51	57	return value <= 127 && value >= -128;
@@ -241,7 +247,7 @@ struct jit_context { · 189 unmodified lines
241	247	/* Number of bytes emit_patch() needs to generate instructions */
242	248	#define X86_PATCH_SIZE 5
243	249	/* Number of bytes that will be skipped on tailcall */
244		−#define X86_TAIL_CALL_OFFSET 11
	250	+#define X86_TAIL_CALL_OFFSET (11 + ENDBR_INSN_SIZE)
245	251
246	252	static void push_callee_regs(u8 *pprog, bool callee_regs_used)
247	253	{
@@ -286,6 +292,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, · 38 unmodified lines
286	292	/* BPF trampoline can be made to work without these nops,
287	293	* but let's waste 5 bytes for now and optimize later
288	294	*/
	295	+ EMIT_ENDBR();
289	296	memcpy(prog, x86_nops[5], X86_PATCH_SIZE);
290	297	prog += X86_PATCH_SIZE;
291	298	if (!ebpf_from_cbpf) {
@@ -296,6 +303,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, · 4 unmodified lines
296	303	}
297	304	EMIT1(0x55); /* push rbp */
298	305	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
	306	+
	307	+ /* X86_TAIL_CALL_OFFSET is here */
	308	+ EMIT_ENDBR();
	309	+
299	310	/* sub rsp, rounded_stack_depth */
300	311	if (stack_depth)
301	312	EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
@@ -380,6 +391,13 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, · 78 unmodified lines
380	391	/* BPF poking in modules is not supported */
381	392	return -EINVAL;
382	393
	394	+ /*
	395	+ * See emit_prologue(), for IBT builds the trampoline hook is preceded
	396	+ * with an ENDBR instruction.
	397	+ */
	398	+ if (is_endbr((u32 )ip))
	399	+ ip += ENDBR_INSN_SIZE;
	400	+
383	401	return __bpf_arch_text_poke(ip, t, old_addr, new_addr);
384	402	}
385	403
@@ -2013,14 +2031,18 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 1627 unmodified lines
2013	2031
2014	2032	ip_off = stack_size;
2015	2033
2016		− if (flags & BPF_TRAMP_F_SKIP_FRAME)
	2034	+ if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2017	2035	/* skip patched call instruction and point orig_call to actual
2018	2036	* body of the kernel function.
2019	2037	*/
	2038	+ if (is_endbr((u32 )orig_call))
	2039	+ orig_call += ENDBR_INSN_SIZE;
2020	2040	orig_call += X86_PATCH_SIZE;
	2041	+ }
2021	2042
2022	2043	prog = image;
2023	2044
	2045	+ EMIT_ENDBR();
2024	2046	EMIT1(0x55); /* push rbp */
2025	2047	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
2026	2048	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */

arch/x86/platform/efi/Makefile+0−1

@@ -1,5 +1,4 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2		−OBJECT_FILES_NON_STANDARD_efi_thunk_$(BITS).o := y
3	2	KASAN_SANITIZE := n
4	3	GCOV_PROFILE := n
5	4

arch/x86/platform/efi/efi_thunk_64.S+4−2

@@ -20,12 +20,14 @@ · 19 unmodified lines
20	20	*/
21	21
22	22	#include <linux/linkage.h>
	23	+#include <linux/objtool.h>
23	24	#include <asm/page_types.h>
24	25	#include <asm/segment.h>
25	26
26	27	.text
27	28	.code64
28		−SYM_CODE_START(__efi64_thunk)
	29	+SYM_FUNC_START(__efi64_thunk)
	30	+STACK_FRAME_NON_STANDARD __efi64_thunk
29	31	push %rbp
30	32	push %rbx
31	33
@@ -79,7 +81,7 @@ SYM_CODE_START(__efi64_thunk) · 47 unmodified lines
79	81	2: pushl $__KERNEL_CS
80	82	pushl %ebp
81	83	lret
82		−SYM_CODE_END(__efi64_thunk)
	84	+SYM_FUNC_END(__efi64_thunk)
83	85
84	86	.bss
85	87	.balign 8

arch/x86/xen/enlighten_pv.c+9−1

@@ -624,6 +624,9 @@ static struct trap_array_entry trap_array[] = { · 623 unmodified lines
624	624	TRAP_ENTRY(exc_coprocessor_error, false ),
625	625	TRAP_ENTRY(exc_alignment_check, false ),
626	626	TRAP_ENTRY(exc_simd_coprocessor_error, false ),
	627	+#ifdef CONFIG_X86_KERNEL_IBT
	628	+ TRAP_ENTRY(exc_control_protection, false ),
	629	+#endif
627	630	};
628	631
629	632	static bool __ref get_trap_addr(void **addr, unsigned int ist)
@@ -1177,6 +1180,8 @@ static void __init xen_domu_set_legacy_features(void) · 547 unmodified lines
1177	1180	x86_platform.legacy.rtc = 0;
1178	1181	}
1179	1182
	1183	+extern void early_xen_iret_patch(void);
	1184	+
1180	1185	/* First C function to be called on Xen boot */
1181	1186	asmlinkage __visible void __init xen_start_kernel(void)
1182	1187	{
@@ -1187,6 +1192,10 @@ asmlinkage __visible void __init xen_start_kernel(void) · 4 unmodified lines
1187	1192	if (!xen_start_info)
1188	1193	return;
1189	1194
	1195	+ __text_gen_insn(&early_xen_iret_patch,
	1196	+ JMP32_INSN_OPCODE, &early_xen_iret_patch, &xen_iret,
	1197	+ JMP32_INSN_SIZE);
	1198	+
1190	1199	xen_domain_type = XEN_PV_DOMAIN;
1191	1200	xen_start_flags = xen_start_info->flags;
1192	1201
@@ -1195,7 +1204,6 @@ asmlinkage __visible void __init xen_start_kernel(void) · 2 unmodified lines
1195	1204	/* Install Xen paravirt ops */
1196	1205	pv_info = xen_info;
1197	1206	pv_ops.cpu = xen_cpu_ops.cpu;
1198		− paravirt_iret = xen_iret;
1199	1207	xen_init_irq_ops();
1200	1208
1201	1209	/*

arch/x86/xen/xen-asm.S+10−0

@@ -122,6 +122,7 @@ SYM_FUNC_END(xen_read_cr2_direct); · 121 unmodified lines
122	122	.macro xen_pv_trap name
123	123	SYM_CODE_START(xen_\name)
124	124	UNWIND_HINT_EMPTY
	125	+ ENDBR
125	126	pop %rcx
126	127	pop %r11
127	128	jmp \name
@@ -147,6 +148,9 @@ xen_pv_trap asm_exc_page_fault · 19 unmodified lines
147	148	xen_pv_trap asm_exc_spurious_interrupt_bug
148	149	xen_pv_trap asm_exc_coprocessor_error
149	150	xen_pv_trap asm_exc_alignment_check
	151	+#ifdef CONFIG_X86_KERNEL_IBT
	152	+xen_pv_trap asm_exc_control_protection
	153	+#endif
150	154	#ifdef CONFIG_X86_MCE
151	155	xen_pv_trap asm_xenpv_exc_machine_check
152	156	#endif /* CONFIG_X86_MCE */
@@ -162,6 +166,7 @@ SYM_CODE_START(xen_early_idt_handler_array) · 9 unmodified lines
162	166	i = 0
163	167	.rept NUM_EXCEPTION_VECTORS
164	168	UNWIND_HINT_EMPTY
	169	+ ENDBR
165	170	pop %rcx
166	171	pop %r11
167	172	jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
@@ -189,6 +194,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 · 21 unmodified lines
189	194	*/
190	195	SYM_CODE_START(xen_iret)
191	196	UNWIND_HINT_EMPTY
	197	+ ANNOTATE_NOENDBR
192	198	pushq $0
193	199	jmp hypercall_iret
194	200	SYM_CODE_END(xen_iret)
@@ -230,6 +236,7 @@ SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode) · 35 unmodified lines
230	236	/* Normal 64-bit system call target */
231	237	SYM_CODE_START(xen_syscall_target)
232	238	UNWIND_HINT_EMPTY
	239	+ ENDBR
233	240	popq %rcx
234	241	popq %r11
235	242
@@ -249,6 +256,7 @@ SYM_CODE_END(xen_syscall_target) · 13 unmodified lines
249	256	/* 32-bit compat syscall target */
250	257	SYM_CODE_START(xen_syscall32_target)
251	258	UNWIND_HINT_EMPTY
	259	+ ENDBR
252	260	popq %rcx
253	261	popq %r11
254	262
@@ -266,6 +274,7 @@ SYM_CODE_END(xen_syscall32_target) · 11 unmodified lines
266	274	/* 32-bit compat sysenter target */
267	275	SYM_CODE_START(xen_sysenter_target)
268	276	UNWIND_HINT_EMPTY
	277	+ ENDBR
269	278	/*
270	279	* NB: Xen is polite and clears TF from EFLAGS for us. This means
271	280	* that we don't need to guard against single step exceptions here.
@@ -289,6 +298,7 @@ SYM_CODE_END(xen_sysenter_target) · 17 unmodified lines
289	298	SYM_CODE_START(xen_syscall32_target)
290	299	SYM_CODE_START(xen_sysenter_target)
291	300	UNWIND_HINT_EMPTY
	301	+ ENDBR
292	302	lea 16(%rsp), %rsp /* strip %rcx, %r11 */
293	303	mov $-ENOSYS, %rax
294	304	pushq $0

arch/x86/xen/xen-head.S+7−2

@@ -25,8 +25,12 @@ · 24 unmodified lines
25	25	SYM_CODE_START(hypercall_page)
26	26	.rept (PAGE_SIZE / 32)
27	27	UNWIND_HINT_FUNC
28		− .skip 31, 0x90
29		− RET
	28	+ ANNOTATE_NOENDBR
	29	+ ret
	30	+ /*
	31	+ * Xen will write the hypercall page, and sort out ENDBR.
	32	+ */
	33	+ .skip 31, 0xcc
30	34	.endr
31	35
32	36	#define HYPERCALL(n) \
@@ -74,6 +78,7 @@ SYM_CODE_END(startup_xen) · 41 unmodified lines
74	78	.pushsection .text
75	79	SYM_CODE_START(asm_cpu_bringup_and_idle)
76	80	UNWIND_HINT_EMPTY
	81	+ ENDBR
77	82
78	83	call cpu_bringup_and_idle
79	84	SYM_CODE_END(asm_cpu_bringup_and_idle)

fs/fuse/ioctl.c+1−1

@@ -170,7 +170,7 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, · 169 unmodified lines
170	170	#else
171	171	if (flags & FUSE_IOCTL_COMPAT) {
172	172	inarg.flags \|= FUSE_IOCTL_32BIT;
173		−#ifdef CONFIG_X86_X32
	173	+#ifdef CONFIG_X86_X32_ABI
174	174	if (in_x32_syscall())
175	175	inarg.flags \|= FUSE_IOCTL_COMPAT_X32;
176	176	#endif

fs/xfs/xfs_ioctl32.c+1−1

@@ -217,7 +217,7 @@ xfs_compat_ioc_fsbulkstat( · 216 unmodified lines
217	217	inumbers_fmt_pf inumbers_func = xfs_fsinumbers_fmt_compat;
218	218	bulkstat_one_fmt_pf bs_one_func = xfs_fsbulkstat_one_fmt_compat;
219	219
220		−#ifdef CONFIG_X86_X32
	220	+#ifdef CONFIG_X86_X32_ABI
221	221	if (in_x32_syscall()) {
222	222	/*
223	223	* ... but on x32 the input xfs_fsop_bulkreq has pointers

include/asm-generic/vmlinux.lds.h+4−0

@@ -394,6 +394,7 @@ · 393 unmodified lines
394	394	KEEP(*(__jump_table)) \
395	395	__stop___jump_table = .;
396	396
	397	+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
397	398	#define STATIC_CALL_DATA \
398	399	. = ALIGN(8); \
399	400	__start_static_call_sites = .; \
@@ -402,6 +403,9 @@ · 2 unmodified lines
402	403	__start_static_call_tramp_key = .; \
403	404	KEEP(*(.static_call_tramp_key)) \
404	405	__stop_static_call_tramp_key = .;
	406	+#else
	407	+#define STATIC_CALL_DATA
	408	+#endif
405	409
406	410	/*
407	411	* Allow architectures to handle ro_after_init data on their

include/linux/cfi.h+10−1

@@ -34,8 +34,17 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr · 33 unmodified lines
34	34
35	35	#else /* !CONFIG_CFI_CLANG */
36	36
37		−#define __CFI_ADDRESSABLE(fn, __attr)
	37	+#ifdef CONFIG_X86_KERNEL_IBT
	38	+
	39	+#define __CFI_ADDRESSABLE(fn, __attr) \
	40	+ const void __cfi_jt_ ## fn __visible __attr = (void )&fn
	41	+
	42	+#endif /* CONFIG_X86_KERNEL_IBT */
38	43
39	44	#endif /* CONFIG_CFI_CLANG */
40	45
	46	+#ifndef __CFI_ADDRESSABLE
	47	+#define __CFI_ADDRESSABLE(fn, __attr)
	48	+#endif
	49	+
41	50	#endif /* _LINUX_CFI_H */

include/linux/compiler.h+0−7

@@ -125,18 +125,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 124 unmodified lines
125	125	})
126	126	#define annotate_unreachable() __annotate_unreachable(__COUNTER__)
127	127
128		−#define ASM_REACHABLE \
129		− "998:\n\t" \
130		− ".pushsection .discard.reachable\n\t" \
131		− ".long 998b - .\n\t" \
132		− ".popsection\n\t"
133		−
134	128	/* Annotate a C jump table to allow objtool to follow the code flow */
135	129	#define __annotate_jump_table __section(".rodata..c_jump_table")
136	130
137	131	#else
138	132	#define annotate_unreachable()
139		−# define ASM_REACHABLE
140	133	#define __annotate_jump_table
141	134	#endif
142	135

include/linux/cred.h+1−1

@@ -176,7 +176,7 @@ extern int set_cred_ucounts(struct cred *); · 175 unmodified lines
176	176	* check for validity of credentials
177	177	*/
178	178	#ifdef CONFIG_DEBUG_CREDENTIALS
179		−extern void __invalid_creds(const struct cred , const char , unsigned);
	179	+extern void __noreturn __invalid_creds(const struct cred , const char , unsigned);
180	180	extern void __validate_process_creds(struct task_struct *,
181	181	const char *, unsigned);
182	182

include/linux/kprobes.h+2−1

@@ -265,7 +265,6 @@ extern int arch_init_kprobes(void); · 264 unmodified lines
265	265	extern void kprobes_inc_nmissed_count(struct kprobe *p);
266	266	extern bool arch_within_kprobe_blacklist(unsigned long addr);
267	267	extern int arch_populate_kprobe_blacklist(void);
268		−extern bool arch_kprobe_on_func_entry(unsigned long offset);
269	268	extern int kprobe_on_func_entry(kprobe_opcode_t addr, const char sym, unsigned long offset);
270	269
271	270	extern bool within_kprobe_blacklist(unsigned long addr);
@@ -384,6 +383,8 @@ static inline struct kprobe_ctlblk *get_kprobe_ctlblk(void) · 112 unmodified lines
384	383	}
385	384
386	385	kprobe_opcode_t kprobe_lookup_name(const char name, unsigned int offset);
	386	+kprobe_opcode_t arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, bool on_func_entry);
	387	+
387	388	int register_kprobe(struct kprobe *p);
388	389	void unregister_kprobe(struct kprobe *p);
389	390	int register_kprobes(struct kprobe **kps, int num);

include/linux/objtool.h+32−0

@@ -77,6 +77,18 @@ struct unwind_hint { · 76 unmodified lines
77	77	#define STACK_FRAME_NON_STANDARD_FP(func)
78	78	#endif
79	79
	80	+#define ANNOTATE_NOENDBR \
	81	+ "986: \n\t" \
	82	+ ".pushsection .discard.noendbr\n\t" \
	83	+ _ASM_PTR " 986b\n\t" \
	84	+ ".popsection\n\t"
	85	+
	86	+#define ASM_REACHABLE \
	87	+ "998:\n\t" \
	88	+ ".pushsection .discard.reachable\n\t" \
	89	+ ".long 998b - .\n\t" \
	90	+ ".popsection\n\t"
	91	+
80	92	#else /* __ASSEMBLY__ */
81	93
82	94	/*
@@ -129,6 +141,20 @@ struct unwind_hint { · 46 unmodified lines
129	141	.popsection
130	142	.endm
131	143
	144	+.macro ANNOTATE_NOENDBR
	145	+.Lhere_\@:
	146	+ .pushsection .discard.noendbr
	147	+ .quad .Lhere_\@
	148	+ .popsection
	149	+.endm
	150	+
	151	+.macro REACHABLE
	152	+.Lhere_\@:
	153	+ .pushsection .discard.reachable
	154	+ .long .Lhere_\@ - .
	155	+ .popsection
	156	+.endm
	157	+
132	158	#endif /* __ASSEMBLY__ */
133	159
134	160	#else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +165,18 @@ struct unwind_hint { · 4 unmodified lines
139	165	"\n\t"
140	166	#define STACK_FRAME_NON_STANDARD(func)
141	167	#define STACK_FRAME_NON_STANDARD_FP(func)
	168	+#define ANNOTATE_NOENDBR
	169	+#define ASM_REACHABLE
142	170	#else
143	171	#define ANNOTATE_INTRA_FUNCTION_CALL
144	172	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0
145	173	.endm
146	174	.macro STACK_FRAME_NON_STANDARD func:req
147	175	.endm
	176	+.macro ANNOTATE_NOENDBR
	177	+.endm
	178	+.macro REACHABLE
	179	+.endm
148	180	#endif
149	181
150	182	#endif /* CONFIG_STACK_VALIDATION */

include/linux/sched/task.h+1−1

@@ -79,7 +79,7 @@ static inline void exit_thread(struct task_struct *tsk) · 78 unmodified lines
79	79	{
80	80	}
81	81	#endif
82		−extern void do_group_exit(int);
	82	+extern __noreturn void do_group_exit(int);
83	83
84	84	extern void exit_files(struct task_struct *);
85	85	extern void exit_itimers(struct signal_struct *);

kernel/bpf/trampoline.c+4−16

@@ -117,18 +117,6 @@ static void bpf_trampoline_module_put(struct bpf_trampoline *tr) · 116 unmodified lines
117	117	tr->mod = NULL;
118	118	}
119	119
120		−static int is_ftrace_location(void *ip)
121		−{
122		− long addr;
123		−
124		− addr = ftrace_location((long)ip);
125		− if (!addr)
126		− return 0;
127		− if (WARN_ON_ONCE(addr != (long)ip))
128		− return -EFAULT;
129		− return 1;
130		−}
131		−
132	120	static int unregister_fentry(struct bpf_trampoline tr, void old_addr)
133	121	{
134	122	void *ip = tr->func.addr;
@@ -160,12 +148,12 @@ static int modify_fentry(struct bpf_trampoline tr, void old_addr, void *new_ad · 25 unmodified lines
160	148	static int register_fentry(struct bpf_trampoline tr, void new_addr)
161	149	{
162	150	void *ip = tr->func.addr;
	151	+ unsigned long faddr;
163	152	int ret;
164	153
165		− ret = is_ftrace_location(ip);
166		− if (ret < 0)
167		− return ret;
168		− tr->func.ftrace_managed = ret;
	154	+ faddr = ftrace_location((unsigned long)ip);
	155	+ if (faddr)
	156	+ tr->func.ftrace_managed = true;
169	157
170	158	if (bpf_trampoline_module_get(tr))
171	159	return -ENOENT;

kernel/cred.c+1−1

@@ -870,7 +870,7 @@ static void dump_invalid_creds(const struct cred cred, const char label, · 869 unmodified lines
870	870	/*
871	871	* report use of invalid credentials
872	872	*/
873		−void __invalid_creds(const struct cred cred, const char file, unsigned line)
	873	+void __noreturn __invalid_creds(const struct cred cred, const char file, unsigned line)
874	874	{
875	875	printk(KERN_ERR "CRED: Invalid credentials\n");
876	876	printk(KERN_ERR "CRED: At %s:%u\n", file, line);

kernel/exit.c+1−1

@@ -895,7 +895,7 @@ SYSCALL_DEFINE1(exit, int, error_code) · 894 unmodified lines
895	895	* Take down every thread in the group. This is called by fatal signals
896	896	* as well as by sys_exit_group (below).
897	897	*/
898		−void
	898	+void __noreturn
899	899	do_group_exit(int exit_code)
900	900	{
901	901	struct signal_struct *sig = current->signal;

kernel/kprobes.c+54−20

@@ -1488,25 +1488,69 @@ bool within_kprobe_blacklist(unsigned long addr) · 1487 unmodified lines
1488	1488	return false;
1489	1489	}
1490	1490
	1491	+/*
	1492	+ * arch_adjust_kprobe_addr - adjust the address
	1493	+ * @addr: symbol base address
	1494	+ * @offset: offset within the symbol
	1495	+ * @on_func_entry: was this @addr+@offset on the function entry
	1496	+ *
	1497	+ * Typically returns @addr + @offset, except for special cases where the
	1498	+ * function might be prefixed by a CFI landing pad, in that case any offset
	1499	+ * inside the landing pad is mapped to the first 'real' instruction of the
	1500	+ * symbol.
	1501	+ *
	1502	+ * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
	1503	+ * instruction at +0.
	1504	+ */
	1505	+kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
	1506	+ unsigned long offset,
	1507	+ bool *on_func_entry)
	1508	+{
	1509	+ *on_func_entry = !offset;
	1510	+ return (kprobe_opcode_t *)(addr + offset);
	1511	+}
	1512	+
1491	1513	/*
1492	1514	* If 'symbol_name' is specified, look it up and add the 'offset'
1493	1515	* to it. This way, we can specify a relative address to a symbol.
1494	1516	* This returns encoded errors if it fails to look up symbol or invalid
1495	1517	* combination of parameters.
1496	1518	*/
1497		−static kprobe_opcode_t _kprobe_addr(kprobe_opcode_t addr,
1498		− const char *symbol_name, unsigned int offset)
	1519	+static kprobe_opcode_t *
	1520	+_kprobe_addr(kprobe_opcode_t addr, const char symbol_name,
	1521	+ unsigned long offset, bool *on_func_entry)
1499	1522	{
1500	1523	if ((symbol_name && addr) \|\| (!symbol_name && !addr))
1501	1524	goto invalid;
1502	1525
1503	1526	if (symbol_name) {
	1527	+ /*
	1528	+ * Input: @sym + @offset
	1529	+ * Output: @addr + @offset
	1530	+ *
	1531	+ * NOTE: kprobe_lookup_name() does NOT fold the offset
	1532	+ * argument into it's output!
	1533	+ */
1504	1534	addr = kprobe_lookup_name(symbol_name, offset);
1505	1535	if (!addr)
1506	1536	return ERR_PTR(-ENOENT);
1507	1537	}
1508	1538
1509		− addr = (kprobe_opcode_t )(((char )addr) + offset);
	1539	+ /*
	1540	+ * So here we have @addr + @offset, displace it into a new
	1541	+ * @addr' + @offset' where @addr' is the symbol start address.
	1542	+ */
	1543	+ addr = (void *)addr + offset;
	1544	+ if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
	1545	+ return ERR_PTR(-ENOENT);
	1546	+ addr = (void *)addr - offset;
	1547	+
	1548	+ /*
	1549	+ * Then ask the architecture to re-combine them, taking care of
	1550	+ * magical function entry details while telling us if this was indeed
	1551	+ * at the start of the function.
	1552	+ */
	1553	+ addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
1510	1554	if (addr)
1511	1555	return addr;
1512	1556
@@ -1516,7 +1560,8 @@ invalid: · 3 unmodified lines
1516	1560
1517	1561	static kprobe_opcode_t kprobe_addr(struct kprobe p)
1518	1562	{
1519		− return _kprobe_addr(p->addr, p->symbol_name, p->offset);
	1563	+ bool on_func_entry;
	1564	+ return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
1520	1565	}
1521	1566
1522	1567	/*
@@ -1562,14 +1607,10 @@ static inline int warn_kprobe_rereg(struct kprobe *p) · 39 unmodified lines
1562	1607
1563	1608	static int check_ftrace_location(struct kprobe *p)
1564	1609	{
1565		− unsigned long ftrace_addr;
	1610	+ unsigned long addr = (unsigned long)p->addr;
1566	1611
1567		− ftrace_addr = ftrace_location((unsigned long)p->addr);
1568		− if (ftrace_addr) {
	1612	+ if (ftrace_location(addr) == addr) {
1569	1613	#ifdef CONFIG_KPROBES_ON_FTRACE
1570		− /* Given address is not on the instruction boundary */
1571		− if ((unsigned long)p->addr != ftrace_addr)
1572		− return -EILSEQ;
1573	1614	p->flags \|= KPROBE_FLAG_FTRACE;
1574	1615	#else /* !CONFIG_KPROBES_ON_FTRACE */
1575	1616	return -EINVAL;
@@ -2047,11 +2088,6 @@ static int pre_handler_kretprobe(struct kprobe p, struct pt_regs regs) · 471 unmodified lines
2047	2088	}
2048	2089	NOKPROBE_SYMBOL(pre_handler_kretprobe);
2049	2090
2050		−bool __weak arch_kprobe_on_func_entry(unsigned long offset)
2051		−{
2052		− return !offset;
2053		−}
2054		−
2055	2091	/**
2056	2092	* kprobe_on_func_entry() -- check whether given address is function entry
2057	2093	* @addr: Target address
@@ -2067,15 +2103,13 @@ bool __weak arch_kprobe_on_func_entry(unsigned long offset) · 9 unmodified lines
2067	2103	*/
2068	2104	int kprobe_on_func_entry(kprobe_opcode_t addr, const char sym, unsigned long offset)
2069	2105	{
2070		− kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
	2106	+ bool on_func_entry;
	2107	+ kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
2071	2108
2072	2109	if (IS_ERR(kp_addr))
2073	2110	return PTR_ERR(kp_addr);
2074	2111
2075		− if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset))
2076		− return -ENOENT;
2077		−
2078		− if (!arch_kprobe_on_func_entry(offset))
	2112	+ if (!on_func_entry)
2079	2113	return -EINVAL;
2080	2114
2081	2115	return 0;

kernel/livepatch/patch.c+2−17

@@ -124,19 +124,6 @@ unlock: · 123 unmodified lines
124	124	ftrace_test_recursion_unlock(bit);
125	125	}
126	126
127		−/*
128		− * Convert a function address into the appropriate ftrace location.
129		− *
130		− * Usually this is just the address of the function, but on some architectures
131		− * it's more complicated so allow them to provide a custom behaviour.
132		− */
133		−#ifndef klp_get_ftrace_location
134		−static unsigned long klp_get_ftrace_location(unsigned long faddr)
135		−{
136		− return faddr;
137		−}
138		−#endif
139		−
140	127	static void klp_unpatch_func(struct klp_func *func)
141	128	{
142	129	struct klp_ops *ops;
@@ -153,8 +140,7 @@ static void klp_unpatch_func(struct klp_func *func) · 10 unmodified lines
153	140	if (list_is_singular(&ops->func_stack)) {
154	141	unsigned long ftrace_loc;
155	142
156		− ftrace_loc =
157		− klp_get_ftrace_location((unsigned long)func->old_func);
	143	+ ftrace_loc = ftrace_location((unsigned long)func->old_func);
158	144	if (WARN_ON(!ftrace_loc))
159	145	return;
160	146
@@ -186,8 +172,7 @@ static int klp_patch_func(struct klp_func *func) · 25 unmodified lines
186	172	if (!ops) {
187	173	unsigned long ftrace_loc;
188	174
189		− ftrace_loc =
190		− klp_get_ftrace_location((unsigned long)func->old_func);
	175	+ ftrace_loc = ftrace_location((unsigned long)func->old_func);
191	176	if (!ftrace_loc) {
192	177	pr_err("failed to find location for function '%s'\n",
193	178	func->old_name);

kernel/trace/ftrace.c+40−8

@@ -1568,17 +1568,34 @@ unsigned long ftrace_location_range(unsigned long start, unsigned long end) · 1567 unmodified lines
1568	1568	}
1569	1569
1570	1570	/**
1571		− * ftrace_location - return true if the ip giving is a traced location
	1571	+ * ftrace_location - return the ftrace location
1572	1572	* @ip: the instruction pointer to check
1573	1573	*
1574		− * Returns rec->ip if @ip given is a pointer to a ftrace location.
1575		− * That is, the instruction that is either a NOP or call to
1576		− * the function tracer. It checks the ftrace internal tables to
1577		− * determine if the address belongs or not.
	1574	+ * If @ip matches the ftrace location, return @ip.
	1575	+ * If @ip matches sym+0, return sym's ftrace location.
	1576	+ * Otherwise, return 0.
1578	1577	*/
1579	1578	unsigned long ftrace_location(unsigned long ip)
1580	1579	{
1581		− return ftrace_location_range(ip, ip);
	1580	+ struct dyn_ftrace *rec;
	1581	+ unsigned long offset;
	1582	+ unsigned long size;
	1583	+
	1584	+ rec = lookup_rec(ip, ip);
	1585	+ if (!rec) {
	1586	+ if (!kallsyms_lookup_size_offset(ip, &size, &offset))
	1587	+ goto out;
	1588	+
	1589	+ /* map sym+0 to __fentry__ */
	1590	+ if (!offset)
	1591	+ rec = lookup_rec(ip, ip + size - 1);
	1592	+ }
	1593	+
	1594	+ if (rec)
	1595	+ return rec->ip;
	1596	+
	1597	+out:
	1598	+ return 0;
1582	1599	}
1583	1600
1584	1601	/**
@@ -4962,7 +4979,8 @@ __ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) · 3377 unmodified lines
4962	4979	{
4963	4980	struct ftrace_func_entry *entry;
4964	4981
4965		− if (!ftrace_location(ip))
	4982	+ ip = ftrace_location(ip);
	4983	+ if (!ip)
4966	4984	return -EINVAL;
4967	4985
4968	4986	if (remove) {
@@ -5131,11 +5149,16 @@ int register_ftrace_direct(unsigned long ip, unsigned long addr) · 162 unmodified lines
5131	5149	struct ftrace_func_entry *entry;
5132	5150	struct ftrace_hash *free_hash = NULL;
5133	5151	struct dyn_ftrace *rec;
5134		− int ret = -EBUSY;
	5152	+ int ret = -ENODEV;
5135	5153
5136	5154	mutex_lock(&direct_mutex);
5137	5155
	5156	+ ip = ftrace_location(ip);
	5157	+ if (!ip)
	5158	+ goto out_unlock;
	5159	+
5138	5160	/* See if there's a direct function at @ip already */
	5161	+ ret = -EBUSY;
5139	5162	if (ftrace_find_rec_direct(ip))
5140	5163	goto out_unlock;
5141	5164
@@ -5243,6 +5266,10 @@ int unregister_ftrace_direct(unsigned long ip, unsigned long addr) · 101 unmodified lines
5243	5266
5244	5267	mutex_lock(&direct_mutex);
5245	5268
	5269	+ ip = ftrace_location(ip);
	5270	+ if (!ip)
	5271	+ goto out_unlock;
	5272	+
5246	5273	entry = find_direct_entry(&ip, NULL);
5247	5274	if (!entry)
5248	5275	goto out_unlock;
@@ -5375,6 +5402,11 @@ int modify_ftrace_direct(unsigned long ip, · 126 unmodified lines
5375	5402	mutex_lock(&direct_mutex);
5376	5403
5377	5404	mutex_lock(&ftrace_lock);
	5405	+
	5406	+ ip = ftrace_location(ip);
	5407	+ if (!ip)
	5408	+ goto out_unlock;
	5409	+
5378	5410	entry = find_direct_entry(&ip, &rec);
5379	5411	if (!entry)
5380	5412	goto out_unlock;

samples/ftrace/ftrace-direct-modify.c+5−0

@@ -24,20 +24,25 @@ static unsigned long my_ip = (unsigned long)schedule; · 23 unmodified lines
24	24
25	25	#ifdef CONFIG_X86_64
26	26
	27	+#include <asm/ibt.h>
	28	+
27	29	asm (
28	30	" .pushsection .text, \"ax\", @progbits\n"
29	31	" .type my_tramp1, @function\n"
30	32	" .globl my_tramp1\n"
31	33	" my_tramp1:"
	34	+ ASM_ENDBR
32	35	" pushq %rbp\n"
33	36	" movq %rsp, %rbp\n"
34	37	" call my_direct_func1\n"
35	38	" leave\n"
36	39	" .size my_tramp1, .-my_tramp1\n"
37	40	ASM_RET
	41	+
38	42	" .type my_tramp2, @function\n"
39	43	" .globl my_tramp2\n"
40	44	" my_tramp2:"
	45	+ ASM_ENDBR
41	46	" pushq %rbp\n"
42	47	" movq %rsp, %rbp\n"
43	48	" call my_direct_func2\n"

samples/ftrace/ftrace-direct-multi-modify.c+7−3

@@ -22,11 +22,14 @@ extern void my_tramp2(void *); · 21 unmodified lines
22	22
23	23	#ifdef CONFIG_X86_64
24	24
	25	+#include <asm/ibt.h>
	26	+
25	27	asm (
26	28	" .pushsection .text, \"ax\", @progbits\n"
27	29	" .type my_tramp1, @function\n"
28	30	" .globl my_tramp1\n"
29	31	" my_tramp1:"
	32	+ ASM_ENDBR
30	33	" pushq %rbp\n"
31	34	" movq %rsp, %rbp\n"
32	35	" pushq %rdi\n"
@@ -34,12 +37,13 @@ asm ( · 1 unmodified line
34	37	" call my_direct_func1\n"
35	38	" popq %rdi\n"
36	39	" leave\n"
37		−" ret\n"
	40	+ ASM_RET
38	41	" .size my_tramp1, .-my_tramp1\n"
	42	+
39	43	" .type my_tramp2, @function\n"
40		−"\n"
41	44	" .globl my_tramp2\n"
42	45	" my_tramp2:"
	46	+ ASM_ENDBR
43	47	" pushq %rbp\n"
44	48	" movq %rsp, %rbp\n"
45	49	" pushq %rdi\n"
@@ -47,7 +51,7 @@ asm ( · 1 unmodified line
47	51	" call my_direct_func2\n"
48	52	" popq %rdi\n"
49	53	" leave\n"
50		−" ret\n"
	54	+ ASM_RET
51	55	" .size my_tramp2, .-my_tramp2\n"
52	56	" .popsection\n"
53	57	);

samples/ftrace/ftrace-direct-multi.c+4−1

@@ -17,11 +17,14 @@ extern void my_tramp(void *); · 16 unmodified lines
17	17
18	18	#ifdef CONFIG_X86_64
19	19
	20	+#include <asm/ibt.h>
	21	+
20	22	asm (
21	23	" .pushsection .text, \"ax\", @progbits\n"
22	24	" .type my_tramp, @function\n"
23	25	" .globl my_tramp\n"
24	26	" my_tramp:"
	27	+ ASM_ENDBR
25	28	" pushq %rbp\n"
26	29	" movq %rsp, %rbp\n"
27	30	" pushq %rdi\n"
@@ -29,7 +32,7 @@ asm ( · 1 unmodified line
29	32	" call my_direct_func\n"
30	33	" popq %rdi\n"
31	34	" leave\n"
32		−" ret\n"
	35	+ ASM_RET
33	36	" .size my_tramp, .-my_tramp\n"
34	37	" .popsection\n"
35	38	);

samples/ftrace/ftrace-direct-too.c+3−0

@@ -19,11 +19,14 @@ extern void my_tramp(void *); · 18 unmodified lines
19	19
20	20	#ifdef CONFIG_X86_64
21	21
	22	+#include <asm/ibt.h>
	23	+
22	24	asm (
23	25	" .pushsection .text, \"ax\", @progbits\n"
24	26	" .type my_tramp, @function\n"
25	27	" .globl my_tramp\n"
26	28	" my_tramp:"
	29	+ ASM_ENDBR
27	30	" pushq %rbp\n"
28	31	" movq %rsp, %rbp\n"
29	32	" pushq %rdi\n"

samples/ftrace/ftrace-direct.c+3−0

@@ -16,11 +16,14 @@ extern void my_tramp(void *); · 15 unmodified lines
16	16
17	17	#ifdef CONFIG_X86_64
18	18
	19	+#include <asm/ibt.h>
	20	+
19	21	asm (
20	22	" .pushsection .text, \"ax\", @progbits\n"
21	23	" .type my_tramp, @function\n"
22	24	" .globl my_tramp\n"
23	25	" my_tramp:"
	26	+ ASM_ENDBR
24	27	" pushq %rbp\n"
25	28	" movq %rsp, %rbp\n"
26	29	" pushq %rdi\n"

scripts/Makefile.build+14−13

@@ -88,8 +88,8 @@ endif · 87 unmodified lines
88	88
89	89	targets-for-modules := $(patsubst %.o, %.mod, $(filter %.o, $(obj-m)))
90	90
91		−ifdef CONFIG_LTO_CLANG
92		−targets-for-modules += $(patsubst %.o, %.lto.o, $(filter %.o, $(obj-m)))
	91	+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
	92	+targets-for-modules += $(patsubst %.o, %.prelink.o, $(filter %.o, $(obj-m)))
93	93	endif
94	94
95	95	ifdef need-modorder
@@ -230,6 +230,7 @@ objtool := $(objtree)/tools/objtool/objtool · 134 unmodified lines
230	230	objtool_args = \
231	231	$(if $(CONFIG_UNWINDER_ORC),orc generate,check) \
232	232	$(if $(part-of-module), --module) \
	233	+ $(if $(CONFIG_X86_KERNEL_IBT), --lto --ibt) \
233	234	$(if $(CONFIG_FRAME_POINTER),, --no-fp) \
234	235	$(if $(CONFIG_GCOV_KERNEL)$(CONFIG_LTO_CLANG), --no-unreachable)\
235	236	$(if $(CONFIG_RETPOLINE), --retpoline) \
@@ -242,7 +243,7 @@ cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(o · 6 unmodified lines
242	243
243	244	endif # CONFIG_STACK_VALIDATION
244	245
245		−ifdef CONFIG_LTO_CLANG
	246	+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
246	247
247	248	# Skip objtool for LLVM bitcode
248	249	$(obj)/%.o: objtool-enabled :=
@@ -288,24 +289,24 @@ $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE · 39 unmodified lines
288	289	$(call if_changed_rule,cc_o_c)
289	290	$(call cmd,force_checksrc)
290	291
291		−ifdef CONFIG_LTO_CLANG
	292	+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
292	293	# Module .o files may contain LLVM bitcode, compile them into native code
293	294	# before ELF processing
294		−quiet_cmd_cc_lto_link_modules = LTO [M] $@
295		−cmd_cc_lto_link_modules = \
	295	+quiet_cmd_cc_prelink_modules = LD [M] $@
	296	+ cmd_cc_prelink_modules = \
296	297	$(LD) $(ld_flags) -r -o $@ \
297		− $(shell [ -s $(@:.lto.o=.o.symversions) ] && \
298		− echo -T $(@:.lto.o=.o.symversions)) \
	298	+ $(shell [ -s $(@:.prelink.o=.o.symversions) ] && \
	299	+ echo -T $(@:.prelink.o=.o.symversions)) \
299	300	--whole-archive $(filter-out FORCE,$^) \
300	301	$(cmd_objtool)
301	302
302	303	# objtool was skipped for LLVM bitcode, run it now that we have compiled
303	304	# modules into native code
304		−$(obj)/%.lto.o: objtool-enabled = y
305		−$(obj)/%.lto.o: part-of-module := y
	305	+$(obj)/%.prelink.o: objtool-enabled = y
	306	+$(obj)/%.prelink.o: part-of-module := y
306	307
307		−$(obj)/%.lto.o: $(obj)/%.o FORCE
308		− $(call if_changed,cc_lto_link_modules)
	308	+$(obj)/%.prelink.o: $(obj)/%.o FORCE
	309	+ $(call if_changed,cc_prelink_modules)
309	310	endif
310	311
311	312	cmd_mod = { \
@@ -469,7 +470,7 @@ $(obj)/lib.a: $(lib-y) FORCE · 157 unmodified lines
469	470	# Do not replace $(filter %.o,^) with $(real-prereqs). When a single object
470	471	# module is turned into a multi object module, $^ will contain header file
471	472	# dependencies recorded in the .*.cmd file.
472		−ifdef CONFIG_LTO_CLANG
	473	+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
473	474	quiet_cmd_link_multi-m = AR [M] $@
474	475	cmd_link_multi-m = \
475	476	$(cmd_update_lto_symversions); \

scripts/Makefile.lib+2−2

@@ -225,11 +225,11 @@ dtc_cpp_flags = -Wp,-MMD,$(depfile).pre.tmp -nostdinc \ · 224 unmodified lines
225	225	$(addprefix -I,$(DTC_INCLUDE)) \
226	226	-undef -D__DTS__
227	227
228		−ifeq ($(CONFIG_LTO_CLANG),y)
	228	+ifneq ($(CONFIG_LTO_CLANG)$(CONFIG_X86_KERNEL_IBT),)
229	229	# With CONFIG_LTO_CLANG, .o files in modules might be LLVM bitcode, so we
230	230	# need to run LTO to compile them into native code (.lto.o) before further
231	231	# processing.
232		−mod-prelink-ext := .lto
	232	+mod-prelink-ext := .prelink
233	233	endif
234	234
235	235	# Useful for describing the dependency of composite objects

scripts/link-vmlinux.sh+9−3

@@ -108,14 +108,20 @@ objtool_link() · 107 unmodified lines
108	108	local objtoolcmd;
109	109	local objtoolopt;
110	110
111		− if is_enabled CONFIG_LTO_CLANG && is_enabled CONFIG_STACK_VALIDATION; then
	111	+ if is_enabled CONFIG_STACK_VALIDATION && \
	112	+ ( is_enabled CONFIG_LTO_CLANG \|\| is_enabled CONFIG_X86_KERNEL_IBT ); then
	113	+
112	114	# Don't perform vmlinux validation unless explicitly requested,
113	115	# but run objtool on vmlinux.o now that we have an object file.
114	116	if is_enabled CONFIG_UNWINDER_ORC; then
115	117	objtoolcmd="orc generate"
116	118	fi
117	119
118		− objtoolopt="${objtoolopt} --duplicate"
	120	+ objtoolopt="${objtoolopt} --lto"
	121	+
	122	+ if is_enabled CONFIG_X86_KERNEL_IBT; then
	123	+ objtoolopt="${objtoolopt} --ibt"
	124	+ fi
119	125
120	126	if is_enabled CONFIG_FTRACE_MCOUNT_USE_OBJTOOL; then
121	127	objtoolopt="${objtoolopt} --mcount"
@@ -168,7 +174,7 @@ vmlinux_link() · 46 unmodified lines
168	174	# skip output file argument
169	175	shift
170	176
171		− if is_enabled CONFIG_LTO_CLANG; then
	177	+ if is_enabled CONFIG_LTO_CLANG \|\| is_enabled CONFIG_X86_KERNEL_IBT; then
172	178	# Use vmlinux.o instead of performing the slow LTO link again.
173	179	objs=vmlinux.o
174	180	libs=

scripts/mod/modpost.c+6−6

@@ -1996,9 +1996,9 @@ static char remove_dot(char s) · 1995 unmodified lines
1996	1996	if (m && (s[n + m] == '.' \|\| s[n + m] == 0))
1997	1997	s[n] = 0;
1998	1998
1999		− /* strip trailing .lto */
2000		− if (strends(s, ".lto"))
2001		− s[strlen(s) - 4] = '\0';
	1999	+ /* strip trailing .prelink */
	2000	+ if (strends(s, ".prelink"))
	2001	+ s[strlen(s) - 8] = '\0';
2002	2002	}
2003	2003	return s;
2004	2004	}
@@ -2022,9 +2022,9 @@ static void read_symbols(const char *modname) · 17 unmodified lines
2022	2022	/* strip trailing .o */
2023	2023	tmp = NOFAIL(strdup(modname));
2024	2024	tmp[strlen(tmp) - 2] = '\0';
2025		− /* strip trailing .lto */
2026		− if (strends(tmp, ".lto"))
2027		− tmp[strlen(tmp) - 4] = '\0';
	2025	+ /* strip trailing .prelink */
	2026	+ if (strends(tmp, ".prelink"))
	2027	+ tmp[strlen(tmp) - 8] = '\0';
2028	2028	mod = new_module(tmp);
2029	2029	free(tmp);
2030	2030	}

sound/core/control_compat.c+8−8

@@ -150,7 +150,7 @@ struct snd_ctl_elem_value32 { · 149 unmodified lines
150	150	unsigned char reserved[128];
151	151	};
152	152
153		−#ifdef CONFIG_X86_X32
	153	+#ifdef CONFIG_X86_X32_ABI
154	154	/* x32 has a different alignment for 64bit values from ia32 */
155	155	struct snd_ctl_elem_value_x32 {
156	156	struct snd_ctl_elem_id id;
@@ -162,7 +162,7 @@ struct snd_ctl_elem_value_x32 { · 5 unmodified lines
162	162	} value;
163	163	unsigned char reserved[128];
164	164	};
165		−#endif /* CONFIG_X86_X32 */
	165	+#endif /* CONFIG_X86_X32_ABI */
166	166
167	167	/* get the value type and count of the control */
168	168	static int get_ctl_type(struct snd_card card, struct snd_ctl_elem_id id,
@@ -347,7 +347,7 @@ static int snd_ctl_elem_write_user_compat(struct snd_ctl_file *file, · 178 unmodified lines
347	347	return ctl_elem_write_user(file, data32, &data32->value);
348	348	}
349	349
350		−#ifdef CONFIG_X86_X32
	350	+#ifdef CONFIG_X86_X32_ABI
351	351	static int snd_ctl_elem_read_user_x32(struct snd_card *card,
352	352	struct snd_ctl_elem_value_x32 __user *data32)
353	353	{
@@ -359,7 +359,7 @@ static int snd_ctl_elem_write_user_x32(struct snd_ctl_file *file, · 5 unmodified lines
359	359	{
360	360	return ctl_elem_write_user(file, data32, &data32->value);
361	361	}
362		−#endif /* CONFIG_X86_X32 */
	362	+#endif /* CONFIG_X86_X32_ABI */
363	363
364	364	/* add or replace a user control */
365	365	static int snd_ctl_elem_add_compat(struct snd_ctl_file *file,
@@ -418,10 +418,10 @@ enum { · 52 unmodified lines
418	418	SNDRV_CTL_IOCTL_ELEM_WRITE32 = _IOWR('U', 0x13, struct snd_ctl_elem_value32),
419	419	SNDRV_CTL_IOCTL_ELEM_ADD32 = _IOWR('U', 0x17, struct snd_ctl_elem_info32),
420	420	SNDRV_CTL_IOCTL_ELEM_REPLACE32 = _IOWR('U', 0x18, struct snd_ctl_elem_info32),
421		−#ifdef CONFIG_X86_X32
	421	+#ifdef CONFIG_X86_X32_ABI
422	422	SNDRV_CTL_IOCTL_ELEM_READ_X32 = _IOWR('U', 0x12, struct snd_ctl_elem_value_x32),
423	423	SNDRV_CTL_IOCTL_ELEM_WRITE_X32 = _IOWR('U', 0x13, struct snd_ctl_elem_value_x32),
424		−#endif /* CONFIG_X86_X32 */
	424	+#endif /* CONFIG_X86_X32_ABI */
425	425	};
426	426
427	427	static inline long snd_ctl_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)
@@ -460,12 +460,12 @@ static inline long snd_ctl_ioctl_compat(struct file *file, unsigned int cmd, uns · 32 unmodified lines
460	460	return snd_ctl_elem_add_compat(ctl, argp, 0);
461	461	case SNDRV_CTL_IOCTL_ELEM_REPLACE32:
462	462	return snd_ctl_elem_add_compat(ctl, argp, 1);
463		−#ifdef CONFIG_X86_X32
	463	+#ifdef CONFIG_X86_X32_ABI
464	464	case SNDRV_CTL_IOCTL_ELEM_READ_X32:
465	465	return snd_ctl_elem_read_user_x32(ctl->card, argp);
466	466	case SNDRV_CTL_IOCTL_ELEM_WRITE_X32:
467	467	return snd_ctl_elem_write_user_x32(ctl, argp);
468		−#endif /* CONFIG_X86_X32 */
	468	+#endif /* CONFIG_X86_X32_ABI */
469	469	}
470	470
471	471	down_read(&snd_ioctl_rwsem);

sound/core/pcm_compat.c+10−10

@@ -147,13 +147,13 @@ static int snd_pcm_ioctl_channel_info_compat(struct snd_pcm_substream *substream · 146 unmodified lines
147	147	return err;
148	148	}
149	149
150		−#ifdef CONFIG_X86_X32
	150	+#ifdef CONFIG_X86_X32_ABI
151	151	/* X32 ABI has the same struct as x86-64 for snd_pcm_channel_info */
152	152	static int snd_pcm_channel_info_user(struct snd_pcm_substream *substream,
153	153	struct snd_pcm_channel_info __user *src);
154	154	#define snd_pcm_ioctl_channel_info_x32(s, p) \
155	155	snd_pcm_channel_info_user(s, p)
156		−#endif /* CONFIG_X86_X32 */
	156	+#endif /* CONFIG_X86_X32_ABI */
157	157
158	158	struct compat_snd_pcm_status64 {
159	159	snd_pcm_state_t state;
@@ -375,7 +375,7 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, · 215 unmodified lines
375	375	return err;
376	376	}
377	377
378		−#ifdef CONFIG_X86_X32
	378	+#ifdef CONFIG_X86_X32_ABI
379	379	/* X32 ABI has 64bit timespec and 64bit alignment */
380	380	struct snd_pcm_mmap_status_x32 {
381	381	snd_pcm_state_t state;
@@ -468,7 +468,7 @@ static int snd_pcm_ioctl_sync_ptr_x32(struct snd_pcm_substream *substream, · 86 unmodified lines
468	468
469	469	return 0;
470	470	}
471		−#endif /* CONFIG_X86_X32 */
	471	+#endif /* CONFIG_X86_X32_ABI */
472	472
473	473	#ifdef __BIG_ENDIAN
474	474	typedef char __pad_before_u32[4];
@@ -560,10 +560,10 @@ enum { · 85 unmodified lines
560	560	SNDRV_PCM_IOCTL_READN_FRAMES32 = _IOR('A', 0x53, struct snd_xfern32),
561	561	SNDRV_PCM_IOCTL_STATUS_COMPAT64 = _IOR('A', 0x20, struct compat_snd_pcm_status64),
562	562	SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64 = _IOWR('A', 0x24, struct compat_snd_pcm_status64),
563		−#ifdef CONFIG_X86_X32
	563	+#ifdef CONFIG_X86_X32_ABI
564	564	SNDRV_PCM_IOCTL_CHANNEL_INFO_X32 = _IOR('A', 0x32, struct snd_pcm_channel_info),
565	565	SNDRV_PCM_IOCTL_SYNC_PTR_X32 = _IOWR('A', 0x23, struct snd_pcm_sync_ptr_x32),
566		−#endif /* CONFIG_X86_X32 */
	566	+#endif /* CONFIG_X86_X32_ABI */
567	567	};
568	568
569	569	static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg)
@@ -607,10 +607,10 @@ static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned l · 37 unmodified lines
607	607	case __SNDRV_PCM_IOCTL_SYNC_PTR32:
608	608	return snd_pcm_common_ioctl(file, substream, cmd, argp);
609	609	case __SNDRV_PCM_IOCTL_SYNC_PTR64:
610		−#ifdef CONFIG_X86_X32
	610	+#ifdef CONFIG_X86_X32_ABI
611	611	if (in_x32_syscall())
612	612	return snd_pcm_ioctl_sync_ptr_x32(substream, argp);
613		−#endif /* CONFIG_X86_X32 */
	613	+#endif /* CONFIG_X86_X32_ABI */
614	614	return snd_pcm_ioctl_sync_ptr_buggy(substream, argp);
615	615	case SNDRV_PCM_IOCTL_HW_REFINE32:
616	616	return snd_pcm_ioctl_hw_params_compat(substream, 1, argp);
@@ -642,10 +642,10 @@ static long snd_pcm_ioctl_compat(struct file *file, unsigned int cmd, unsigned l · 25 unmodified lines
642	642	return snd_pcm_status_user_compat64(substream, argp, false);
643	643	case SNDRV_PCM_IOCTL_STATUS_EXT_COMPAT64:
644	644	return snd_pcm_status_user_compat64(substream, argp, true);
645		−#ifdef CONFIG_X86_X32
	645	+#ifdef CONFIG_X86_X32_ABI
646	646	case SNDRV_PCM_IOCTL_CHANNEL_INFO_X32:
647	647	return snd_pcm_ioctl_channel_info_x32(substream, argp);
648		−#endif /* CONFIG_X86_X32 */
	648	+#endif /* CONFIG_X86_X32_ABI */
649	649	}
650	650
651	651	return -ENOIOCTLCMD;

tools/include/linux/objtool.h+32−0

@@ -77,6 +77,18 @@ struct unwind_hint { · 76 unmodified lines
77	77	#define STACK_FRAME_NON_STANDARD_FP(func)
78	78	#endif
79	79
	80	+#define ANNOTATE_NOENDBR \
	81	+ "986: \n\t" \
	82	+ ".pushsection .discard.noendbr\n\t" \
	83	+ _ASM_PTR " 986b\n\t" \
	84	+ ".popsection\n\t"
	85	+
	86	+#define ASM_REACHABLE \
	87	+ "998:\n\t" \
	88	+ ".pushsection .discard.reachable\n\t" \
	89	+ ".long 998b - .\n\t" \
	90	+ ".popsection\n\t"
	91	+
80	92	#else /* __ASSEMBLY__ */
81	93
82	94	/*
@@ -129,6 +141,20 @@ struct unwind_hint { · 46 unmodified lines
129	141	.popsection
130	142	.endm
131	143
	144	+.macro ANNOTATE_NOENDBR
	145	+.Lhere_\@:
	146	+ .pushsection .discard.noendbr
	147	+ .quad .Lhere_\@
	148	+ .popsection
	149	+.endm
	150	+
	151	+.macro REACHABLE
	152	+.Lhere_\@:
	153	+ .pushsection .discard.reachable
	154	+ .long .Lhere_\@ - .
	155	+ .popsection
	156	+.endm
	157	+
132	158	#endif /* __ASSEMBLY__ */
133	159
134	160	#else /* !CONFIG_STACK_VALIDATION */
@@ -139,12 +165,18 @@ struct unwind_hint { · 4 unmodified lines
139	165	"\n\t"
140	166	#define STACK_FRAME_NON_STANDARD(func)
141	167	#define STACK_FRAME_NON_STANDARD_FP(func)
	168	+#define ANNOTATE_NOENDBR
	169	+#define ASM_REACHABLE
142	170	#else
143	171	#define ANNOTATE_INTRA_FUNCTION_CALL
144	172	.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0
145	173	.endm
146	174	.macro STACK_FRAME_NON_STANDARD func:req
147	175	.endm
	176	+.macro ANNOTATE_NOENDBR
	177	+.endm
	178	+.macro REACHABLE
	179	+.endm
148	180	#endif
149	181
150	182	#endif /* CONFIG_STACK_VALIDATION */

tools/objtool/arch/x86/decode.c+29−5

@@ -103,6 +103,18 @@ unsigned long arch_jump_destination(struct instruction *insn) · 102 unmodified lines
103	103	#define rm_is_mem(reg) (mod_is_mem() && !is_RIP() && rm_is(reg))
104	104	#define rm_is_reg(reg) (mod_is_reg() && modrm_rm == (reg))
105	105
	106	+static bool has_notrack_prefix(struct insn *insn)
	107	+{
	108	+ int i;
	109	+
	110	+ for (i = 0; i < insn->prefixes.nbytes; i++) {
	111	+ if (insn->prefixes.bytes[i] == 0x3e)
	112	+ return true;
	113	+ }
	114	+
	115	+ return false;
	116	+}
	117	+
106	118	int arch_decode_instruction(struct objtool_file file, const struct section sec,
107	119	unsigned long offset, unsigned int maxlen,
108	120	unsigned int len, enum insn_type type,
@@ -112,7 +124,7 @@ int arch_decode_instruction(struct objtool_file file, const struct section sec · 3 unmodified lines
112	124	const struct elf *elf = file->elf;
113	125	struct insn insn;
114	126	int x86_64, ret;
115		− unsigned char op1, op2, op3,
	127	+ unsigned char op1, op2, op3, prefix,
116	128	rex = 0, rex_b = 0, rex_r = 0, rex_w = 0, rex_x = 0,
117	129	modrm = 0, modrm_mod = 0, modrm_rm = 0, modrm_reg = 0,
118	130	sib = 0, /* sib_scale = 0, */ sib_index = 0, sib_base = 0;
@@ -137,6 +149,8 @@ int arch_decode_instruction(struct objtool_file file, const struct section sec · 18 unmodified lines
137	149	if (insn.vex_prefix.nbytes)
138	150	return 0;
139	151
	152	+ prefix = insn.prefixes.bytes[0];
	153	+
140	154	op1 = insn.opcode.bytes[0];
141	155	op2 = insn.opcode.bytes[1];
142	156	op3 = insn.opcode.bytes[2];
@@ -492,6 +506,12 @@ int arch_decode_instruction(struct objtool_file file, const struct section sec · 349 unmodified lines
492	506	/* nopl/nopw */
493	507	*type = INSN_NOP;
494	508
	509	+ } else if (op2 == 0x1e) {
	510	+
	511	+ if (prefix == 0xf3 && (modrm == 0xfa \|\| modrm == 0xfb))
	512	+ *type = INSN_ENDBR;
	513	+
	514	+
495	515	} else if (op2 == 0x38 && op3 == 0xf8) {
496	516	if (insn.prefixes.nbytes == 1 &&
497	517	insn.prefixes.bytes[0] == 0xf2) {
@@ -636,20 +656,24 @@ int arch_decode_instruction(struct objtool_file file, const struct section sec · 138 unmodified lines
636	656	break;
637	657
638	658	case 0xff:
639		− if (modrm_reg == 2 \|\| modrm_reg == 3)
	659	+ if (modrm_reg == 2 \|\| modrm_reg == 3) {
640	660
641	661	*type = INSN_CALL_DYNAMIC;
	662	+ if (has_notrack_prefix(&insn))
	663	+ WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
642	664
643		− else if (modrm_reg == 4)
	665	+ } else if (modrm_reg == 4) {
644	666
645	667	*type = INSN_JUMP_DYNAMIC;
	668	+ if (has_notrack_prefix(&insn))
	669	+ WARN("notrack prefix found at %s:0x%lx", sec->name, offset);
646	670
647		− else if (modrm_reg == 5)
	671	+ } else if (modrm_reg == 5) {
648	672
649	673	/* jmpf */
650	674	*type = INSN_CONTEXT_SWITCH;
651	675
652		− else if (modrm_reg == 6) {
	676	+ } else if (modrm_reg == 6) {
653	677
654	678	/* push from mem */
655	679	ADD_OP(op) {

tools/objtool/builtin-check.c+5−2

@@ -20,7 +20,8 @@ · 19 unmodified lines
20	20	#include <objtool/objtool.h>
21	21
22	22	bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
23		− validate_dup, vmlinux, mcount, noinstr, backup, sls;
	23	+ lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
	24	+ ibt;
24	25
25	26	static const char * const check_usage[] = {
26	27	"objtool check [<options>] file.o",
@@ -40,12 +41,14 @@ const struct option check_options[] = { · 13 unmodified lines
40	41	OPT_BOOLEAN('b', "backtrace", &backtrace, "unwind on error"),
41	42	OPT_BOOLEAN('a', "uaccess", &uaccess, "enable uaccess checking"),
42	43	OPT_BOOLEAN('s', "stats", &stats, "print statistics"),
43		− OPT_BOOLEAN('d', "duplicate", &validate_dup, "duplicate validation for vmlinux.o"),
	44	+ OPT_BOOLEAN(0, "lto", &lto, "whole-archive like runs"),
44	45	OPT_BOOLEAN('n', "noinstr", &noinstr, "noinstr validation for vmlinux.o"),
45	46	OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"),
46	47	OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"),
47	48	OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"),
48	49	OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"),
	50	+ OPT_BOOLEAN(0, "dry-run", &dryrun, "don't write the modifications"),
	51	+ OPT_BOOLEAN(0, "ibt", &ibt, "validate ENDBR placement"),
49	52	OPT_END(),
50	53	};
51	54

tools/objtool/check.c+376−24

@@ -181,6 +181,9 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 180 unmodified lines
181	181	"kunit_try_catch_throw",
182	182	"xen_start_kernel",
183	183	"cpu_bringup_and_idle",
	184	+ "do_group_exit",
	185	+ "stop_this_cpu",
	186	+ "__invalid_creds",
184	187	};
185	188
186	189	if (!func)
@@ -380,6 +383,7 @@ static int decode_instructions(struct objtool_file *file) · 193 unmodified lines
380	383	memset(insn, 0, sizeof(*insn));
381	384	INIT_LIST_HEAD(&insn->alts);
382	385	INIT_LIST_HEAD(&insn->stack_ops);
	386	+ INIT_LIST_HEAD(&insn->call_node);
383	387
384	388	insn->sec = sec;
385	389	insn->offset = offset;
@@ -392,6 +396,14 @@ static int decode_instructions(struct objtool_file *file) · 6 unmodified lines
392	396	if (ret)
393	397	goto err;
394	398
	399	+ /*
	400	+ * By default, "ud2" is a dead end unless otherwise
	401	+ * annotated, because GCC 7 inserts it for certain
	402	+ * divide-by-zero cases.
	403	+ */
	404	+ if (insn->type == INSN_BUG)
	405	+ insn->dead_end = true;
	406	+
395	407	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
396	408	list_add_tail(&insn->list, &file->insn_list);
397	409	nr_insns++;
@@ -407,8 +419,17 @@ static int decode_instructions(struct objtool_file *file) · 9 unmodified lines
407	419	return -1;
408	420	}
409	421
410		− sym_for_each_insn(file, func, insn)
	422	+ sym_for_each_insn(file, func, insn) {
411	423	insn->func = func;
	424	+ if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
	425	+ if (insn->offset == insn->func->offset) {
	426	+ list_add_tail(&insn->call_node, &file->endbr_list);
	427	+ file->nr_endbr++;
	428	+ } else {
	429	+ file->nr_endbr_int++;
	430	+ }
	431	+ }
	432	+ }
412	433	}
413	434	}
414	435
@@ -520,14 +541,6 @@ static int add_dead_ends(struct objtool_file *file) · 105 unmodified lines
520	541	struct reloc *reloc;
521	542	struct instruction *insn;
522	543
523		− /*
524		− * By default, "ud2" is a dead end unless otherwise annotated, because
525		− * GCC 7 inserts it for certain divide-by-zero cases.
526		− */
527		− for_each_insn(file, insn)
528		− if (insn->type == INSN_BUG)
529		− insn->dead_end = true;
530		−
531	544	/*
532	545	* Check for manually annotated dead ends.
533	546	*/
@@ -731,6 +744,58 @@ static int create_retpoline_sites_sections(struct objtool_file *file) · 197 unmodified lines
731	744	return 0;
732	745	}
733	746
	747	+static int create_ibt_endbr_seal_sections(struct objtool_file *file)
	748	+{
	749	+ struct instruction *insn;
	750	+ struct section *sec;
	751	+ int idx;
	752	+
	753	+ sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
	754	+ if (sec) {
	755	+ WARN("file already has .ibt_endbr_seal, skipping");
	756	+ return 0;
	757	+ }
	758	+
	759	+ idx = 0;
	760	+ list_for_each_entry(insn, &file->endbr_list, call_node)
	761	+ idx++;
	762	+
	763	+ if (stats) {
	764	+ printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
	765	+ printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
	766	+ printf("ibt: superfluous ENDBR: %d\n", idx);
	767	+ }
	768	+
	769	+ if (!idx)
	770	+ return 0;
	771	+
	772	+ sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
	773	+ sizeof(int), idx);
	774	+ if (!sec) {
	775	+ WARN("elf_create_section: .ibt_endbr_seal");
	776	+ return -1;
	777	+ }
	778	+
	779	+ idx = 0;
	780	+ list_for_each_entry(insn, &file->endbr_list, call_node) {
	781	+
	782	+ int site = (int )sec->data->d_buf + idx;
	783	+ *site = 0;
	784	+
	785	+ if (elf_add_reloc_to_insn(file->elf, sec,
	786	+ idx * sizeof(int),
	787	+ R_X86_64_PC32,
	788	+ insn->sec, insn->offset)) {
	789	+ WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
	790	+ return -1;
	791	+ }
	792	+
	793	+ idx++;
	794	+ }
	795	+
	796	+ return 0;
	797	+}
	798	+
734	799	static int create_mcount_loc_sections(struct objtool_file *file)
735	800	{
736	801	struct section *sec;
@@ -1111,6 +1176,9 @@ static void annotate_call_site(struct objtool_file *file, · 374 unmodified lines
1111	1176	list_add_tail(&insn->call_node, &file->mcount_loc_list);
1112	1177	return;
1113	1178	}
	1179	+
	1180	+ if (!sibling && dead_end_function(file, sym))
	1181	+ insn->dead_end = true;
1114	1182	}
1115	1183
1116	1184	static void add_call_dest(struct objtool_file file, struct instruction insn,
@@ -1165,6 +1233,19 @@ static void add_retpoline_call(struct objtool_file file, struct instruction in · 48 unmodified lines
1165	1233
1166	1234	annotate_call_site(file, insn, false);
1167	1235	}
	1236	+
	1237	+static bool same_function(struct instruction insn1, struct instruction insn2)
	1238	+{
	1239	+ return insn1->func->pfunc == insn2->func->pfunc;
	1240	+}
	1241	+
	1242	+static bool is_first_func_insn(struct instruction *insn)
	1243	+{
	1244	+ return insn->offset == insn->func->offset \|\|
	1245	+ (insn->type == INSN_ENDBR &&
	1246	+ insn->offset == insn->func->offset + insn->len);
	1247	+}
	1248	+
1168	1249	/*
1169	1250	* Find the destination instructions for all jumps.
1170	1251	*/
@@ -1245,8 +1326,8 @@ static int add_jump_destinations(struct objtool_file *file) · 74 unmodified lines
1245	1326	insn->func->cfunc = insn->jump_dest->func;
1246	1327	insn->jump_dest->func->pfunc = insn->func;
1247	1328
1248		− } else if (insn->jump_dest->func->pfunc != insn->func->pfunc &&
1249		− insn->jump_dest->offset == insn->jump_dest->func->offset) {
	1329	+ } else if (!same_function(insn, insn->jump_dest) &&
	1330	+ is_first_func_insn(insn->jump_dest)) {
1250	1331	/* internal sibling call (without reloc) */
1251	1332	add_call_dest(file, insn, insn->jump_dest->func, true);
1252	1333	}
@@ -1836,6 +1917,16 @@ static int read_unwind_hints(struct objtool_file *file) · 583 unmodified lines
1836	1917
1837	1918	insn->hint = true;
1838	1919
	1920	+ if (ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
	1921	+ struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
	1922	+
	1923	+ if (sym && sym->bind == STB_GLOBAL &&
	1924	+ insn->type != INSN_ENDBR && !insn->noendbr) {
	1925	+ WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
	1926	+ insn->sec, insn->offset);
	1927	+ }
	1928	+ }
	1929	+
1839	1930	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
1840	1931	insn->cfi = &func_cfi;
1841	1932	continue;
@@ -1860,6 +1951,32 @@ static int read_unwind_hints(struct objtool_file *file) · 18 unmodified lines
1860	1951	return 0;
1861	1952	}
1862	1953
	1954	+static int read_noendbr_hints(struct objtool_file *file)
	1955	+{
	1956	+ struct section *sec;
	1957	+ struct instruction *insn;
	1958	+ struct reloc *reloc;
	1959	+
	1960	+ sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
	1961	+ if (!sec)
	1962	+ return 0;
	1963	+
	1964	+ list_for_each_entry(reloc, &sec->reloc_list, list) {
	1965	+ insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
	1966	+ if (!insn) {
	1967	+ WARN("bad .discard.noendbr entry");
	1968	+ return -1;
	1969	+ }
	1970	+
	1971	+ if (insn->type == INSN_ENDBR)
	1972	+ WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset);
	1973	+
	1974	+ insn->noendbr = 1;
	1975	+ }
	1976	+
	1977	+ return 0;
	1978	+}
	1979	+
1863	1980	static int read_retpoline_hints(struct objtool_file *file)
1864	1981	{
1865	1982	struct section *sec;
@@ -2086,10 +2203,6 @@ static int decode_sections(struct objtool_file *file) · 220 unmodified lines
2086	2203	if (ret)
2087	2204	return ret;
2088	2205
2089		− ret = add_dead_ends(file);
2090		− if (ret)
2091		− return ret;
2092		−
2093	2206	add_ignores(file);
2094	2207	add_uaccess_safe(file);
2095	2208
@@ -2097,6 +2210,13 @@ static int decode_sections(struct objtool_file *file) · 1 unmodified line
2097	2210	if (ret)
2098	2211	return ret;
2099	2212
	2213	+ /*
	2214	+ * Must be before read_unwind_hints() since that needs insn->noendbr.
	2215	+ */
	2216	+ ret = read_noendbr_hints(file);
	2217	+ if (ret)
	2218	+ return ret;
	2219	+
2100	2220	/*
2101	2221	* Must be before add_{jump_call}_destination.
2102	2222	*/
@@ -2128,6 +2248,14 @@ static int decode_sections(struct objtool_file *file) · 25 unmodified lines
2128	2248	if (ret)
2129	2249	return ret;
2130	2250
	2251	+ /*
	2252	+ * Must be after add_call_destinations() such that it can override
	2253	+ * dead_end_function() marks.
	2254	+ */
	2255	+ ret = add_dead_ends(file);
	2256	+ if (ret)
	2257	+ return ret;
	2258	+
2131	2259	ret = add_jump_table_alts(file);
2132	2260	if (ret)
2133	2261	return ret;
@@ -3026,6 +3154,115 @@ static struct instruction next_insn_to_validate(struct objtool_file file, · 892 unmodified lines
3026	3154	return next_insn_same_sec(file, insn);
3027	3155	}
3028	3156
	3157	+static struct instruction *
	3158	+validate_ibt_reloc(struct objtool_file file, struct reloc reloc)
	3159	+{
	3160	+ struct instruction *dest;
	3161	+ struct section *sec;
	3162	+ unsigned long off;
	3163	+
	3164	+ sec = reloc->sym->sec;
	3165	+ off = reloc->sym->offset;
	3166	+
	3167	+ if ((reloc->sec->base->sh.sh_flags & SHF_EXECINSTR) &&
	3168	+ (reloc->type == R_X86_64_PC32 \|\| reloc->type == R_X86_64_PLT32))
	3169	+ off += arch_dest_reloc_offset(reloc->addend);
	3170	+ else
	3171	+ off += reloc->addend;
	3172	+
	3173	+ dest = find_insn(file, sec, off);
	3174	+ if (!dest)
	3175	+ return NULL;
	3176	+
	3177	+ if (dest->type == INSN_ENDBR) {
	3178	+ if (!list_empty(&dest->call_node))
	3179	+ list_del_init(&dest->call_node);
	3180	+
	3181	+ return NULL;
	3182	+ }
	3183	+
	3184	+ if (reloc->sym->static_call_tramp)
	3185	+ return NULL;
	3186	+
	3187	+ return dest;
	3188	+}
	3189	+
	3190	+static void warn_noendbr(const char msg, struct section sec, unsigned long offset,
	3191	+ struct instruction *dest)
	3192	+{
	3193	+ WARN_FUNC("%srelocation to !ENDBR: %s+0x%lx", sec, offset, msg,
	3194	+ dest->func ? dest->func->name : dest->sec->name,
	3195	+ dest->func ? dest->offset - dest->func->offset : dest->offset);
	3196	+}
	3197	+
	3198	+static void validate_ibt_dest(struct objtool_file file, struct instruction insn,
	3199	+ struct instruction *dest)
	3200	+{
	3201	+ if (dest->func && dest->func == insn->func) {
	3202	+ /*
	3203	+ * Anything from->to self is either _THIS_IP_ or IRET-to-self.
	3204	+ *
	3205	+ * There is no sane way to annotate _THIS_IP_ since the compiler treats the
	3206	+ * relocation as a constant and is happy to fold in offsets, skewing any
	3207	+ * annotation we do, leading to vast amounts of false-positives.
	3208	+ *
	3209	+ * There's also compiler generated _THIS_IP_ through KCOV and
	3210	+ * such which we have no hope of annotating.
	3211	+ *
	3212	+ * As such, blanket accept self-references without issue.
	3213	+ */
	3214	+ return;
	3215	+ }
	3216	+
	3217	+ if (dest->noendbr)
	3218	+ return;
	3219	+
	3220	+ warn_noendbr("", insn->sec, insn->offset, dest);
	3221	+}
	3222	+
	3223	+static void validate_ibt_insn(struct objtool_file file, struct instruction insn)
	3224	+{
	3225	+ struct instruction *dest;
	3226	+ struct reloc *reloc;
	3227	+
	3228	+ switch (insn->type) {
	3229	+ case INSN_CALL:
	3230	+ case INSN_CALL_DYNAMIC:
	3231	+ case INSN_JUMP_CONDITIONAL:
	3232	+ case INSN_JUMP_UNCONDITIONAL:
	3233	+ case INSN_JUMP_DYNAMIC:
	3234	+ case INSN_JUMP_DYNAMIC_CONDITIONAL:
	3235	+ case INSN_RETURN:
	3236	+ /*
	3237	+ * We're looking for code references setting up indirect code
	3238	+ * flow. As such, ignore direct code flow and the actual
	3239	+ * dynamic branches.
	3240	+ */
	3241	+ return;
	3242	+
	3243	+ case INSN_NOP:
	3244	+ /*
	3245	+ * handle_group_alt() will create INSN_NOP instruction that
	3246	+ * don't belong to any section, ignore all NOP since they won't
	3247	+ * carry a (useful) relocation anyway.
	3248	+ */
	3249	+ return;
	3250	+
	3251	+ default:
	3252	+ break;
	3253	+ }
	3254	+
	3255	+ for (reloc = insn_reloc(file, insn);
	3256	+ reloc;
	3257	+ reloc = find_reloc_by_dest_range(file->elf, insn->sec,
	3258	+ reloc->offset + 1,
	3259	+ (insn->offset + insn->len) - (reloc->offset + 1))) {
	3260	+ dest = validate_ibt_reloc(file, reloc);
	3261	+ if (dest)
	3262	+ validate_ibt_dest(file, insn, dest);
	3263	+ }
	3264	+}
	3265	+
3029	3266	/*
3030	3267	* Follow the branch starting at the given instruction, and recursively follow
3031	3268	* any other branches (jumps). Meanwhile, track the frame pointer state at
@@ -3115,9 +3352,8 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 83 unmodified lines
3115	3352	switch (insn->type) {
3116	3353
3117	3354	case INSN_RETURN:
3118		− if (next_insn && next_insn->type == INSN_TRAP) {
3119		− next_insn->ignore = true;
3120		− } else if (sls && !insn->retpoline_safe) {
	3355	+ if (sls && !insn->retpoline_safe &&
	3356	+ next_insn && next_insn->type != INSN_TRAP) {
3121	3357	WARN_FUNC("missing int3 after ret",
3122	3358	insn->sec, insn->offset);
3123	3359	}
@@ -3136,7 +3372,7 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 12 unmodified lines
3136	3372	return 1;
3137	3373	}
3138	3374
3139		− if (dead_end_function(file, insn->call_dest))
	3375	+ if (insn->dead_end)
3140	3376	return 0;
3141	3377
3142	3378	break;
@@ -3164,9 +3400,8 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 21 unmodified lines
3164	3400	break;
3165	3401
3166	3402	case INSN_JUMP_DYNAMIC:
3167		− if (next_insn && next_insn->type == INSN_TRAP) {
3168		− next_insn->ignore = true;
3169		− } else if (sls && !insn->retpoline_safe) {
	3403	+ if (sls && !insn->retpoline_safe &&
	3404	+ next_insn && next_insn->type != INSN_TRAP) {
3170	3405	WARN_FUNC("missing int3 after indirect jump",
3171	3406	insn->sec, insn->offset);
3172	3407	}
@@ -3237,6 +3472,9 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 64 unmodified lines
3237	3472	break;
3238	3473	}
3239	3474
	3475	+ if (ibt)
	3476	+ validate_ibt_insn(file, insn);
	3477	+
3240	3478	if (insn->dead_end)
3241	3479	return 0;
3242	3480
@@ -3337,7 +3575,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 94 unmodified lines
3337	3575	int i;
3338	3576	struct instruction *prev_insn;
3339	3577
3340		− if (insn->ignore \|\| insn->type == INSN_NOP)
	3578	+ if (insn->ignore \|\| insn->type == INSN_NOP \|\| insn->type == INSN_TRAP)
3341	3579	return true;
3342	3580
3343	3581	/*
@@ -3348,6 +3586,49 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 4 unmodified lines
3348	3586	!strcmp(insn->sec->name, ".altinstr_aux"))
3349	3587	return true;
3350	3588
	3589	+ /*
	3590	+ * Whole archive runs might encounder dead code from weak symbols.
	3591	+ * This is where the linker will have dropped the weak symbol in
	3592	+ * favour of a regular symbol, but leaves the code in place.
	3593	+ *
	3594	+ * In this case we'll find a piece of code (whole function) that is not
	3595	+ * covered by a !section symbol. Ignore them.
	3596	+ */
	3597	+ if (!insn->func && lto) {
	3598	+ int size = find_symbol_hole_containing(insn->sec, insn->offset);
	3599	+ unsigned long end = insn->offset + size;
	3600	+
	3601	+ if (!size) /* not a hole */
	3602	+ return false;
	3603	+
	3604	+ if (size < 0) /* hole until the end */
	3605	+ return true;
	3606	+
	3607	+ sec_for_each_insn_continue(file, insn) {
	3608	+ /*
	3609	+ * If we reach a visited instruction at or before the
	3610	+ * end of the hole, ignore the unreachable.
	3611	+ */
	3612	+ if (insn->visited)
	3613	+ return true;
	3614	+
	3615	+ if (insn->offset >= end)
	3616	+ break;
	3617	+
	3618	+ /*
	3619	+ * If this hole jumps to a .cold function, mark it ignore too.
	3620	+ */
	3621	+ if (insn->jump_dest && insn->jump_dest->func &&
	3622	+ strstr(insn->jump_dest->func->name, ".cold")) {
	3623	+ struct instruction *dest = insn->jump_dest;
	3624	+ func_for_each_insn(file, dest->func, dest)
	3625	+ dest->ignore = true;
	3626	+ }
	3627	+ }
	3628	+
	3629	+ return false;
	3630	+ }
	3631	+
3351	3632	if (!insn->func)
3352	3633	return false;
3353	3634
@@ -3479,6 +3760,53 @@ static int validate_functions(struct objtool_file *file) · 125 unmodified lines
3479	3760	return warnings;
3480	3761	}
3481	3762
	3763	+static int validate_ibt(struct objtool_file *file)
	3764	+{
	3765	+ struct section *sec;
	3766	+ struct reloc *reloc;
	3767	+
	3768	+ for_each_sec(file, sec) {
	3769	+ bool is_data;
	3770	+
	3771	+ /* already done in validate_branch() */
	3772	+ if (sec->sh.sh_flags & SHF_EXECINSTR)
	3773	+ continue;
	3774	+
	3775	+ if (!sec->reloc)
	3776	+ continue;
	3777	+
	3778	+ if (!strncmp(sec->name, ".orc", 4))
	3779	+ continue;
	3780	+
	3781	+ if (!strncmp(sec->name, ".discard", 8))
	3782	+ continue;
	3783	+
	3784	+ if (!strncmp(sec->name, ".debug", 6))
	3785	+ continue;
	3786	+
	3787	+ if (!strcmp(sec->name, "_error_injection_whitelist"))
	3788	+ continue;
	3789	+
	3790	+ if (!strcmp(sec->name, "_kprobe_blacklist"))
	3791	+ continue;
	3792	+
	3793	+ is_data = strstr(sec->name, ".data") \|\| strstr(sec->name, ".rodata");
	3794	+
	3795	+ list_for_each_entry(reloc, &sec->reloc->reloc_list, list) {
	3796	+ struct instruction *dest;
	3797	+
	3798	+ dest = validate_ibt_reloc(file, reloc);
	3799	+ if (is_data && dest && !dest->noendbr) {
	3800	+ warn_noendbr("data ", reloc->sym->sec,
	3801	+ reloc->sym->offset + reloc->addend,
	3802	+ dest);
	3803	+ }
	3804	+ }
	3805	+ }
	3806	+
	3807	+ return 0;
	3808	+}
	3809	+
3482	3810	static int validate_reachable_instructions(struct objtool_file *file)
3483	3811	{
3484	3812	struct instruction *insn;
@@ -3501,6 +3829,16 @@ int check(struct objtool_file *file) · 16 unmodified lines
3501	3829	{
3502	3830	int ret, warnings = 0;
3503	3831
	3832	+ if (lto && !(vmlinux \|\| module)) {
	3833	+ fprintf(stderr, "--lto requires: --vmlinux or --module\n");
	3834	+ return 1;
	3835	+ }
	3836	+
	3837	+ if (ibt && !lto) {
	3838	+ fprintf(stderr, "--ibt requires: --lto\n");
	3839	+ return 1;
	3840	+ }
	3841	+
3504	3842	arch_initial_func_cfi_state(&initial_func_cfi);
3505	3843	init_cfi_state(&init_cfi);
3506	3844	init_cfi_state(&func_cfi);
@@ -3521,7 +3859,7 @@ int check(struct objtool_file *file) · 14 unmodified lines
3521	3859	if (list_empty(&file->insn_list))
3522	3860	goto out;
3523	3861
3524		− if (vmlinux && !validate_dup) {
	3862	+ if (vmlinux && !lto) {
3525	3863	ret = validate_vmlinux_functions(file);
3526	3864	if (ret < 0)
3527	3865	goto out;
@@ -3547,6 +3885,13 @@ int check(struct objtool_file *file) · 19 unmodified lines
3547	3885	goto out;
3548	3886	warnings += ret;
3549	3887
	3888	+ if (ibt) {
	3889	+ ret = validate_ibt(file);
	3890	+ if (ret < 0)
	3891	+ goto out;
	3892	+ warnings += ret;
	3893	+ }
	3894	+
3550	3895	if (!warnings) {
3551	3896	ret = validate_reachable_instructions(file);
3552	3897	if (ret < 0)
@@ -3573,6 +3918,13 @@ int check(struct objtool_file *file) · 20 unmodified lines
3573	3918	warnings += ret;
3574	3919	}
3575	3920
	3921	+ if (ibt) {
	3922	+ ret = create_ibt_endbr_seal_sections(file);
	3923	+ if (ret < 0)
	3924	+ goto out;
	3925	+ warnings += ret;
	3926	+ }
	3927	+
3576	3928	if (stats) {
3577	3929	printf("nr_insns_visited: %ld\n", nr_insns_visited);
3578	3930	printf("nr_cfi: %ld\n", nr_cfi);

tools/objtool/elf.c+63−0

@@ -83,6 +83,31 @@ static int symbol_by_offset(const void key, const struct rb_node node) · 82 unmodified lines
83	83	return 0;
84	84	}
85	85
	86	+struct symbol_hole {
	87	+ unsigned long key;
	88	+ const struct symbol *sym;
	89	+};
	90	+
	91	+/*
	92	+ * Find !section symbol where @offset is after it.
	93	+ */
	94	+static int symbol_hole_by_offset(const void key, const struct rb_node node)
	95	+{
	96	+ const struct symbol *s = rb_entry(node, struct symbol, node);
	97	+ struct symbol_hole sh = (void )key;
	98	+
	99	+ if (sh->key < s->offset)
	100	+ return -1;
	101	+
	102	+ if (sh->key >= s->offset + s->len) {
	103	+ if (s->type != STT_SECTION)
	104	+ sh->sym = s;
	105	+ return 1;
	106	+ }
	107	+
	108	+ return 0;
	109	+}
	110	+
86	111	struct section find_section_by_name(const struct elf elf, const char *name)
87	112	{
88	113	struct section *sec;
@@ -162,6 +187,41 @@ struct symbol find_symbol_containing(const struct section sec, unsigned long o · 73 unmodified lines
162	187	return NULL;
163	188	}
164	189
	190	+/*
	191	+ * Returns size of hole starting at @offset.
	192	+ */
	193	+int find_symbol_hole_containing(const struct section *sec, unsigned long offset)
	194	+{
	195	+ struct symbol_hole hole = {
	196	+ .key = offset,
	197	+ .sym = NULL,
	198	+ };
	199	+ struct rb_node *n;
	200	+ struct symbol *s;
	201	+
	202	+ /*
	203	+ * Find the rightmost symbol for which @offset is after it.
	204	+ */
	205	+ n = rb_find(&hole, &sec->symbol_tree, symbol_hole_by_offset);
	206	+
	207	+ /* found a symbol that contains @offset */
	208	+ if (n)
	209	+ return 0; /* not a hole */
	210	+
	211	+ /* didn't find a symbol for which @offset is after it */
	212	+ if (!hole.sym)
	213	+ return 0; /* not a hole */
	214	+
	215	+ /* @offset >= sym->offset + sym->len, find symbol after it */
	216	+ n = rb_next(&hole.sym->node);
	217	+ if (!n)
	218	+ return -1; /* until end of address space */
	219	+
	220	+ /* hole until start of next symbol */
	221	+ s = rb_entry(n, struct symbol, node);
	222	+ return s->offset - offset;
	223	+}
	224	+
165	225	struct symbol find_func_containing(struct section sec, unsigned long offset)
166	226	{
167	227	struct rb_node *node;
@@ -1019,6 +1079,9 @@ int elf_write(struct elf *elf) · 851 unmodified lines
1019	1079	struct section *sec;
1020	1080	Elf_Scn *s;
1021	1081
	1082	+ if (dryrun)
	1083	+ return 0;
	1084	+
1022	1085	/* Update changed relocation sections and section headers: */
1023	1086	list_for_each_entry(sec, &elf->sections, list) {
1024	1087	if (sec->changed) {

tools/objtool/include/objtool/arch.h+1−0

@@ -27,6 +27,7 @@ enum insn_type { · 26 unmodified lines
27	27	INSN_STD,
28	28	INSN_CLD,
29	29	INSN_TRAP,
	30	+ INSN_ENDBR,
30	31	INSN_OTHER,
31	32	};
32	33

tools/objtool/include/objtool/builtin.h+2−1

@@ -9,7 +9,8 @@ · 8 unmodified lines
9	9
10	10	extern const struct option check_options[];
11	11	extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats,
12		− validate_dup, vmlinux, mcount, noinstr, backup, sls;
	12	+ lto, vmlinux, mcount, noinstr, backup, sls, dryrun,
	13	+ ibt;
13	14
14	15	extern int cmd_parse_options(int argc, const char *argv, const char const usage[]);
15	16

tools/objtool/include/objtool/check.h+10−3

@@ -45,11 +45,18 @@ struct instruction { · 44 unmodified lines
45	45	unsigned int len;
46	46	enum insn_type type;
47	47	unsigned long immediate;
48		− bool dead_end, ignore, ignore_alts;
49		− bool hint;
50		− bool retpoline_safe;
	48	+
	49	+ u8 dead_end : 1,
	50	+ ignore : 1,
	51	+ ignore_alts : 1,
	52	+ hint : 1,
	53	+ retpoline_safe : 1,
	54	+ noendbr : 1;
	55	+ /* 2 bit hole */
51	56	s8 instr;
52	57	u8 visited;
	58	+ /* u8 hole */
	59	+
53	60	struct alt_group *alt_group;
54	61	struct symbol *call_dest;
55	62	struct instruction *jump_dest;

tools/objtool/include/objtool/elf.h+1−0

@@ -152,6 +152,7 @@ struct symbol find_func_by_offset(struct section sec, unsigned long offset); · 151 unmodified lines
152	152	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset);
153	153	struct symbol find_symbol_by_name(const struct elf elf, const char *name);
154	154	struct symbol find_symbol_containing(const struct section sec, unsigned long offset);
	155	+int find_symbol_hole_containing(const struct section *sec, unsigned long offset);
155	156	struct reloc find_reloc_by_dest(const struct elf elf, struct section *sec, unsigned long offset);
156	157	struct reloc find_reloc_by_dest_range(const struct elf elf, struct section *sec,
157	158	unsigned long offset, unsigned int len);

tools/objtool/include/objtool/objtool.h+4−0

@@ -26,8 +26,12 @@ struct objtool_file { · 25 unmodified lines
26	26	struct list_head retpoline_call_list;
27	27	struct list_head static_call_list;
28	28	struct list_head mcount_loc_list;
	29	+ struct list_head endbr_list;
29	30	bool ignore_unreachables, c_file, hints, rodata;
30	31
	32	+ unsigned int nr_endbr;
	33	+ unsigned int nr_endbr_int;
	34	+
31	35	unsigned long jl_short, jl_long;
32	36	unsigned long jl_nop_short, jl_nop_long;
33	37

tools/objtool/include/objtool/warn.h+2−0

@@ -22,6 +22,8 @@ static inline char offstr(struct section sec, unsigned long offset) · 21 unmodified lines
22	22	unsigned long name_off;
23	23
24	24	func = find_func_containing(sec, offset);
	25	+ if (!func)
	26	+ func = find_symbol_containing(sec, offset);
25	27	if (func) {
26	28	name = func->name;
27	29	name_off = offset - func->offset;

tools/objtool/objtool.c+1−0

@@ -128,6 +128,7 @@ struct objtool_file objtool_open_read(const char _objname) · 127 unmodified lines
128	128	INIT_LIST_HEAD(&file.retpoline_call_list);
129	129	INIT_LIST_HEAD(&file.static_call_list);
130	130	INIT_LIST_HEAD(&file.mcount_loc_list);
	131	+ INIT_LIST_HEAD(&file.endbr_list);
131	132	file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment");
132	133	file.ignore_unreachables = no_unreachable;
133	134	file.hints = false;

Merge tag 'x86_core_for_5.18_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code