Merge tag 'x86_core_for_v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 core updates from Borislav Petkov: - Add the call depth tracking mitigation for Retbleed which has been long in the making. It is a lighterweight software-only fix for Skylake-based cores where enabling IBRS is a big hammer and causes a significant performance impact. What it basically does is, it aligns all kernel functions to 16 bytes boundary and adds a 16-byte padding before the function, objtool collects all functions' locations and when the mitigation gets applied, it patches a call accounting thunk which is used to track the call depth of the stack at any time. When that call depth reaches a magical, microarchitecture-specific value for the Return Stack Buffer, the code stuffs that RSB and avoids its underflow which could otherwise lead to the Intel variant of Retbleed. This software-only solution brings a lot of the lost performance back, as benchmarks suggest: https://lore.kernel.org/all/20220915111039.092790446@infradead.org/ That page above also contains a lot more detailed explanation of the whole mechanism - Implement a new control flow integrity scheme called FineIBT which is based on the software kCFI implementation and uses hardware IBT support where present to annotate and track indirect branches using a hash to validate them - Other misc fixes and cleanups * tag 'x86_core_for_v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (80 commits) x86/paravirt: Use common macro for creating simple asm paravirt functions x86/paravirt: Remove clobber bitmask from .parainstructions x86/debug: Include percpu.h in debugreg.h to get DECLARE_PER_CPU() et al x86/cpufeatures: Move X86_FEATURE_CALL_DEPTH from bit 18 to bit 19 of word 11, to leave space for WIP X86_FEATURE_SGX_EDECCSSA bit x86/Kconfig: Enable kernel IBT by default x86,pm: Force out-of-line memcpy() objtool: Fix weak hole vs prefix symbol objtool: Optimize elf_dirty_reloc_sym() x86/cfi: Add boot time hash randomization x86/cfi: Boot time selection of CFI scheme x86/ibt: Implement FineIBT objtool: Add --cfi to generate the .cfi_sites section x86: Add prefix symbols for function padding objtool: Add option to generate prefix symbols objtool: Avoid O(bloody terrible) behaviour -- an ode to libelf objtool: Slice up elf_create_section_symbol() kallsyms: Revert "Take callthunks into account" x86: Unconfuse CONFIG_ and X86_FEATURE_ namespaces x86/retpoline: Fix crash printing warning x86/paravirt: Fix a !PARAVIRT build warning ...

Linus Torvalds committed 3 years ago

Commit 94a85512 parents 93761c9 f1a033c103 files · +2705 −593Browse files

Unified · Split

103 files

Makefile +2 −2
arch/Kconfig +24 −0
arch/ia64/Kconfig +1 −0
arch/ia64/Makefile +1 −1
arch/um/kernel/um_arch.c +5 −0
arch/x86/Kconfig +74 −1
arch/x86/Makefile +6 −0
arch/x86/boot/compressed/head_64.S +8 −0
arch/x86/crypto/camellia-aesni-avx-asm_64.S +0 −2
arch/x86/crypto/camellia-aesni-avx2-asm_64.S +0 −4
arch/x86/crypto/cast5-avx-x86_64-asm_64.S +0 −2
arch/x86/crypto/crct10dif-pcl-asm_64.S +0 −1
arch/x86/crypto/poly1305-x86_64-cryptogams.pl +0 −1
arch/x86/crypto/serpent-avx-x86_64-asm_64.S +0 −2
arch/x86/crypto/serpent-avx2-asm_64.S +0 −2
arch/x86/crypto/sha1_ni_asm.S +0 −1
arch/x86/crypto/sha256-avx-asm.S +0 −1
arch/x86/crypto/sha256-avx2-asm.S +0 −1
arch/x86/crypto/sha256-ssse3-asm.S +0 −1
arch/x86/crypto/sha256_ni_asm.S +0 −1
arch/x86/crypto/sm3-avx-asm_64.S +0 −1
arch/x86/crypto/sm4-aesni-avx-asm_64.S +0 −7
arch/x86/crypto/sm4-aesni-avx2-asm_64.S +0 −6
arch/x86/crypto/twofish-avx-x86_64-asm_64.S +0 −2
arch/x86/entry/entry_32.S +2 −2
arch/x86/entry/entry_64.S +28 −22
arch/x86/entry/entry_64_compat.S +3 −4
arch/x86/entry/thunk_64.S +2 −2
arch/x86/entry/vdso/Makefile +8 −6
arch/x86/include/asm/alternative.h +65 −3
arch/x86/include/asm/cpufeatures.h +1 −2
arch/x86/include/asm/current.h +29 −3
arch/x86/include/asm/debugreg.h +1 −1
arch/x86/include/asm/disabled-features.h +8 −1
arch/x86/include/asm/hardirq.h +2 −1
arch/x86/include/asm/irq_stack.h +6 −6
arch/x86/include/asm/linkage.h +55 −8
arch/x86/include/asm/nospec-branch.h +170 −6
arch/x86/include/asm/paravirt.h +15 −2
arch/x86/include/asm/paravirt_types.h +19 −15
arch/x86/include/asm/preempt.h +14 −13
arch/x86/include/asm/processor.h +2 −9
arch/x86/include/asm/qspinlock_paravirt.h +20 −27
arch/x86/include/asm/smp.h +5 −7
arch/x86/include/asm/text-patching.h +1 −0
arch/x86/kernel/Makefile +2 −0
arch/x86/kernel/alternative.c +495 −34
arch/x86/kernel/asm-offsets.c +5 −0
arch/x86/kernel/asm-offsets_64.c +1 −1
arch/x86/kernel/callthunks.c +388 −0
arch/x86/kernel/cpu/Makefile +0 −3
arch/x86/kernel/cpu/bugs.c +32 −2
arch/x86/kernel/cpu/common.c +44 −53
arch/x86/kernel/dumpstack_32.c +2 −2
arch/x86/kernel/dumpstack_64.c +1 −1
arch/x86/kernel/ftrace.c +13 −7
arch/x86/kernel/ftrace_64.S +34 −3
arch/x86/kernel/head_64.S +1 −0
arch/x86/kernel/irq_32.c +5 −8
arch/x86/kernel/irq_64.c +3 −3
arch/x86/kernel/kprobes/core.c +0 −1
arch/x86/kernel/kvm.c +6 −12
arch/x86/kernel/module.c +42 −5
arch/x86/kernel/paravirt.c +2 −19
arch/x86/kernel/process_32.c +3 −3
arch/x86/kernel/process_64.c +3 −3
arch/x86/kernel/relocate_kernel_64.S +3 −2
arch/x86/kernel/setup_percpu.c +2 −5
arch/x86/kernel/smpboot.c +7 −3
arch/x86/kernel/static_call.c +2 −1
arch/x86/kernel/traps.c +2 −2
arch/x86/kernel/unwind_orc.c +20 −1
arch/x86/kernel/vmlinux.lds.S +23 −14
arch/x86/kvm/svm/vmenter.S +1 −0
arch/x86/lib/error-inject.c +1 −0
arch/x86/lib/putuser.S +49 −13
arch/x86/lib/retpoline.S +99 −8
arch/x86/net/bpf_jit_comp.c +34 −11
arch/x86/power/hibernate.c +1 −1
arch/x86/xen/enlighten_pv.c +1 −1
include/asm-generic/vmlinux.lds.h +2 −2
include/linux/linkage.h +2 −2
include/linux/static_call.h +2 −0
kernel/static_call_inline.c +18 −5
kernel/trace/trace_selftest.c +8 −1
lib/Kconfig.debug +1 −0
samples/ftrace/ftrace-direct-modify.c +3 −0
samples/ftrace/ftrace-direct-multi-modify.c +3 −0
samples/ftrace/ftrace-direct-multi.c +2 −0
samples/ftrace/ftrace-direct-too.c +2 −0
samples/ftrace/ftrace-direct.c +2 −0
scripts/Makefile.lib +3 −0
tools/include/linux/interval_tree_generic.h +187 −0
tools/objtool/arch/x86/decode.c +24 −0
tools/objtool/builtin-check.c +8 −1
tools/objtool/check.c +310 −81
tools/objtool/elf.c +199 −97
tools/objtool/include/objtool/arch.h +2 −0
tools/objtool/include/objtool/builtin.h +3 −0
tools/objtool/include/objtool/check.h +11 −1
tools/objtool/include/objtool/elf.h +7 −2
tools/objtool/include/objtool/objtool.h +1 −0
tools/objtool/objtool.c +1 −0

Makefile+2−2

@@ -1006,8 +1006,8 @@ KBUILD_CFLAGS += $(CC_FLAGS_CFI) · 1005 unmodified lines
1006	1006	export CC_FLAGS_CFI
1007	1007	endif
1008	1008
1009		−ifdef CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B
1010		−KBUILD_CFLAGS += -falign-functions=64
	1009	+ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0)
	1010	+KBUILD_CFLAGS += -falign-functions=$(CONFIG_FUNCTION_ALIGNMENT)
1011	1011	endif
1012	1012
1013	1013	# arch Makefile may override CC so keep this after arch Makefile is included

arch/Kconfig+24−0

@@ -1438,4 +1438,28 @@ source "kernel/gcov/Kconfig" · 1437 unmodified lines
1438	1438
1439	1439	source "scripts/gcc-plugins/Kconfig"
1440	1440
	1441	+config FUNCTION_ALIGNMENT_4B
	1442	+ bool
	1443	+
	1444	+config FUNCTION_ALIGNMENT_8B
	1445	+ bool
	1446	+
	1447	+config FUNCTION_ALIGNMENT_16B
	1448	+ bool
	1449	+
	1450	+config FUNCTION_ALIGNMENT_32B
	1451	+ bool
	1452	+
	1453	+config FUNCTION_ALIGNMENT_64B
	1454	+ bool
	1455	+
	1456	+config FUNCTION_ALIGNMENT
	1457	+ int
	1458	+ default 64 if FUNCTION_ALIGNMENT_64B
	1459	+ default 32 if FUNCTION_ALIGNMENT_32B
	1460	+ default 16 if FUNCTION_ALIGNMENT_16B
	1461	+ default 8 if FUNCTION_ALIGNMENT_8B
	1462	+ default 4 if FUNCTION_ALIGNMENT_4B
	1463	+ default 0
	1464	+
1441	1465	endmenu

arch/ia64/Kconfig+1−0

@@ -63,6 +63,7 @@ config IA64 · 62 unmodified lines
63	63	select NUMA if !FLATMEM
64	64	select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
65	65	select ZONE_DMA32
	66	+ select FUNCTION_ALIGNMENT_32B
66	67	default y
67	68	help
68	69	The Itanium Processor Family is Intel's 64-bit successor to

arch/ia64/Makefile+1−1

@@ -23,7 +23,7 @@ KBUILD_AFLAGS_KERNEL := -mconstant-gp · 22 unmodified lines
23	23	EXTRA :=
24	24
25	25	cflags-y := -pipe $(EXTRA) -ffixed-r13 -mfixed-range=f12-f15,f32-f127 \
26		− -falign-functions=32 -frename-registers -fno-optimize-sibling-calls
	26	+ -frename-registers -fno-optimize-sibling-calls
27	27	KBUILD_CFLAGS_KERNEL := -mconstant-gp
28	28
29	29	GAS_STATUS = $(shell $(srctree)/arch/ia64/scripts/check-gas "$(CC)" "$(OBJDUMP)")

arch/um/kernel/um_arch.c+5−0

@@ -444,6 +444,11 @@ void apply_returns(s32 start, s32 end) · 443 unmodified lines
444	444	{
445	445	}
446	446
	447	+void apply_fineibt(s32 start_retpoline, s32 end_retpoline,
	448	+ s32 start_cfi, s32 end_cfi)
	449	+{
	450	+}
	451	+
447	452	void apply_alternatives(struct alt_instr start, struct alt_instr end)
448	453	{
449	454	}

arch/x86/Kconfig+74−1

@@ -292,6 +292,8 @@ config X86 · 291 unmodified lines
292	292	select X86_FEATURE_NAMES if PROC_FS
293	293	select PROC_PID_ARCH_STATUS if PROC_FS
294	294	select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX
	295	+ select FUNCTION_ALIGNMENT_16B if X86_64 \|\| X86_ALIGNMENT_16
	296	+ select FUNCTION_ALIGNMENT_4B
295	297	imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI
296	298	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
297	299
@@ -1855,7 +1857,7 @@ config CC_HAS_IBT · 1557 unmodified lines
1855	1857
1856	1858	config X86_KERNEL_IBT
1857	1859	prompt "Indirect Branch Tracking"
1858		− bool
	1860	+ def_bool y
1859	1861	depends on X86_64 && CC_HAS_IBT && HAVE_OBJTOOL
1860	1862	# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
1861	1863	depends on !LD_IS_LLD \|\| LLD_VERSION >= 140000
@@ -2492,6 +2494,46 @@ config CC_HAS_SLS · 630 unmodified lines
2492	2494	config CC_HAS_RETURN_THUNK
2493	2495	def_bool $(cc-option,-mfunction-return=thunk-extern)
2494	2496
	2497	+config CC_HAS_ENTRY_PADDING
	2498	+ def_bool $(cc-option,-fpatchable-function-entry=16,16)
	2499	+
	2500	+config FUNCTION_PADDING_CFI
	2501	+ int
	2502	+ default 59 if FUNCTION_ALIGNMENT_64B
	2503	+ default 27 if FUNCTION_ALIGNMENT_32B
	2504	+ default 11 if FUNCTION_ALIGNMENT_16B
	2505	+ default 3 if FUNCTION_ALIGNMENT_8B
	2506	+ default 0
	2507	+
	2508	+# Basically: FUNCTION_ALIGNMENT - 5*CFI_CLANG
	2509	+# except Kconfig can't do arithmetic :/
	2510	+config FUNCTION_PADDING_BYTES
	2511	+ int
	2512	+ default FUNCTION_PADDING_CFI if CFI_CLANG
	2513	+ default FUNCTION_ALIGNMENT
	2514	+
	2515	+config CALL_PADDING
	2516	+ def_bool n
	2517	+ depends on CC_HAS_ENTRY_PADDING && OBJTOOL
	2518	+ select FUNCTION_ALIGNMENT_16B
	2519	+
	2520	+config FINEIBT
	2521	+ def_bool y
	2522	+ depends on X86_KERNEL_IBT && CFI_CLANG && RETPOLINE
	2523	+ select CALL_PADDING
	2524	+
	2525	+config HAVE_CALL_THUNKS
	2526	+ def_bool y
	2527	+ depends on CC_HAS_ENTRY_PADDING && RETHUNK && OBJTOOL
	2528	+
	2529	+config CALL_THUNKS
	2530	+ def_bool n
	2531	+ select CALL_PADDING
	2532	+
	2533	+config PREFIX_SYMBOLS
	2534	+ def_bool y
	2535	+ depends on CALL_PADDING && !CFI_CLANG
	2536	+
2495	2537	menuconfig SPECULATION_MITIGATIONS
2496	2538	bool "Mitigations for speculative execution vulnerabilities"
2497	2539	default y
@@ -2543,6 +2585,37 @@ config CPU_UNRET_ENTRY · 45 unmodified lines
2543	2585	help
2544	2586	Compile the kernel with support for the retbleed=unret mitigation.
2545	2587
	2588	+config CALL_DEPTH_TRACKING
	2589	+ bool "Mitigate RSB underflow with call depth tracking"
	2590	+ depends on CPU_SUP_INTEL && HAVE_CALL_THUNKS
	2591	+ select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
	2592	+ select CALL_THUNKS
	2593	+ default y
	2594	+ help
	2595	+ Compile the kernel with call depth tracking to mitigate the Intel
	2596	+ SKL Return-Speculation-Buffer (RSB) underflow issue. The
	2597	+ mitigation is off by default and needs to be enabled on the
	2598	+ kernel command line via the retbleed=stuff option. For
	2599	+ non-affected systems the overhead of this option is marginal as
	2600	+ the call depth tracking is using run-time generated call thunks
	2601	+ in a compiler generated padding area and call patching. This
	2602	+ increases text size by ~5%. For non affected systems this space
	2603	+ is unused. On affected SKL systems this results in a significant
	2604	+ performance gain over the IBRS mitigation.
	2605	+
	2606	+config CALL_THUNKS_DEBUG
	2607	+ bool "Enable call thunks and call depth tracking debugging"
	2608	+ depends on CALL_DEPTH_TRACKING
	2609	+ select FUNCTION_ALIGNMENT_32B
	2610	+ default n
	2611	+ help
	2612	+ Enable call/ret counters for imbalance detection and build in
	2613	+ a noisy dmesg about callthunks generation and call patching for
	2614	+ trouble shooting. The debug prints need to be enabled on the
	2615	+ kernel command line with 'debug-callthunks'.
	2616	+ Only enable this, when you are debugging call thunks as this
	2617	+ creates a noticable runtime overhead. If unsure say N.
	2618	+
2546	2619	config CPU_IBPB_ENTRY
2547	2620	bool "Enable IBPB on kernel entry"
2548	2621	depends on CPU_SUP_AMD && X86_64

arch/x86/Makefile+6−0

@@ -208,6 +208,12 @@ ifdef CONFIG_SLS · 207 unmodified lines
208	208	KBUILD_CFLAGS += -mharden-sls=all
209	209	endif
210	210
	211	+ifdef CONFIG_CALL_PADDING
	212	+PADDING_CFLAGS := -fpatchable-function-entry=$(CONFIG_FUNCTION_PADDING_BYTES),$(CONFIG_FUNCTION_PADDING_BYTES)
	213	+KBUILD_CFLAGS += $(PADDING_CFLAGS)
	214	+export PADDING_CFLAGS
	215	+endif
	216	+
211	217	KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
212	218
213	219	ifdef CONFIG_LTO_CLANG

arch/x86/boot/compressed/head_64.S+8−0

@@ -37,6 +37,14 @@ · 36 unmodified lines
37	37	#include <asm/trapnr.h>
38	38	#include "pgtable.h"
39	39
	40	+/*
	41	+ * Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
	42	+ * in assembly errors due to trying to move .org backward due to the excessive
	43	+ * alignment.
	44	+ */
	45	+#undef __ALIGN
	46	+#define __ALIGN .balign 16, 0x90
	47	+
40	48	/*
41	49	* Locally defined symbols should be marked hidden:
42	50	*/

arch/x86/crypto/camellia-aesni-avx-asm_64.S+0−2

@@ -712,7 +712,6 @@ SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) · 711 unmodified lines
712	712
713	713	.text
714	714
715		−.align 8
716	715	SYM_FUNC_START_LOCAL(__camellia_enc_blk16)
717	716	/* input:
718	717	* %rdi: ctx, CTX
@@ -799,7 +798,6 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk16) · 80 unmodified lines
799	798	jmp .Lenc_done;
800	799	SYM_FUNC_END(__camellia_enc_blk16)
801	800
802		−.align 8
803	801	SYM_FUNC_START_LOCAL(__camellia_dec_blk16)
804	802	/* input:
805	803	* %rdi: ctx, CTX

arch/x86/crypto/camellia-aesni-avx2-asm_64.S+0−4

@@ -221,7 +221,6 @@ · 220 unmodified lines
221	221	* Size optimization... with inlined roundsm32 binary would be over 5 times
222	222	* larger and would only marginally faster.
223	223	*/
224		−.align 8
225	224	SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
226	225	roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
227	226	%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
@@ -229,7 +228,6 @@ SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_c · 1 unmodified line
229	228	RET;
230	229	SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
231	230
232		−.align 8
233	231	SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
234	232	roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
235	233	%ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
@@ -748,7 +746,6 @@ SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) · 512 unmodified lines
748	746
749	747	.text
750	748
751		−.align 8
752	749	SYM_FUNC_START_LOCAL(__camellia_enc_blk32)
753	750	/* input:
754	751	* %rdi: ctx, CTX
@@ -835,7 +832,6 @@ SYM_FUNC_START_LOCAL(__camellia_enc_blk32) · 80 unmodified lines
835	832	jmp .Lenc_done;
836	833	SYM_FUNC_END(__camellia_enc_blk32)
837	834
838		−.align 8
839	835	SYM_FUNC_START_LOCAL(__camellia_dec_blk32)
840	836	/* input:
841	837	* %rdi: ctx, CTX

arch/x86/crypto/cast5-avx-x86_64-asm_64.S+0−2

@@ -208,7 +208,6 @@ · 207 unmodified lines
208	208
209	209	.text
210	210
211		−.align 16
212	211	SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
213	212	/* input:
214	213	* %rdi: ctx
@@ -282,7 +281,6 @@ SYM_FUNC_START_LOCAL(__cast5_enc_blk16) · 67 unmodified lines
282	281	RET;
283	282	SYM_FUNC_END(__cast5_enc_blk16)
284	283
285		−.align 16
286	284	SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
287	285	/* input:
288	286	* %rdi: ctx

arch/x86/crypto/crct10dif-pcl-asm_64.S+0−1

@@ -94,7 +94,6 @@ · 93 unmodified lines
94	94	#
95	95	# Assumes len >= 16.
96	96	#
97		−.align 16
98	97	SYM_FUNC_START(crc_t10dif_pcl)
99	98
100	99	movdqa .Lbswap_mask(%rip), BSWAP_MASK

arch/x86/crypto/poly1305-x86_64-cryptogams.pl+0−1

@@ -108,7 +108,6 @@ if (!$kernel) { · 107 unmodified lines
108	108	sub declare_function() {
109	109	my ($name, $align, $nargs) = @_;
110	110	if($kernel) {
111		− $code .= ".align $align\n";
112	111	$code .= "SYM_FUNC_START($name)\n";
113	112	$code .= ".L$name:\n";
114	113	} else {

arch/x86/crypto/serpent-avx-x86_64-asm_64.S+0−2

@@ -550,7 +550,6 @@ · 549 unmodified lines
550	550	#define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
551	551	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
552	552
553		−.align 8
554	553	SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx)
555	554	/* input:
556	555	* %rdi: ctx, CTX
@@ -604,7 +603,6 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx) · 47 unmodified lines
604	603	RET;
605	604	SYM_FUNC_END(__serpent_enc_blk8_avx)
606	605
607		−.align 8
608	606	SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx)
609	607	/* input:
610	608	* %rdi: ctx, CTX

arch/x86/crypto/serpent-avx2-asm_64.S+0−2

@@ -550,7 +550,6 @@ · 549 unmodified lines
550	550	#define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
551	551	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
552	552
553		−.align 8
554	553	SYM_FUNC_START_LOCAL(__serpent_enc_blk16)
555	554	/* input:
556	555	* %rdi: ctx, CTX
@@ -604,7 +603,6 @@ SYM_FUNC_START_LOCAL(__serpent_enc_blk16) · 47 unmodified lines
604	603	RET;
605	604	SYM_FUNC_END(__serpent_enc_blk16)
606	605
607		−.align 8
608	606	SYM_FUNC_START_LOCAL(__serpent_dec_blk16)
609	607	/* input:
610	608	* %rdi: ctx, CTX

arch/x86/crypto/sha1_ni_asm.S+0−1

@@ -93,7 +93,6 @@ · 92 unmodified lines
93	93	* numBlocks: Number of blocks to process
94	94	*/
95	95	.text
96		−.align 32
97	96	SYM_TYPED_FUNC_START(sha1_ni_transform)
98	97	push %rbp
99	98	mov %rsp, %rbp

arch/x86/crypto/sha256-avx-asm.S+0−1

@@ -348,7 +348,6 @@ a = TMP_ · 347 unmodified lines
348	348	########################################################################
349	349	.text
350	350	SYM_TYPED_FUNC_START(sha256_transform_avx)
351		−.align 32
352	351	pushq %rbx
353	352	pushq %r12
354	353	pushq %r13

arch/x86/crypto/sha256-avx2-asm.S+0−1

@@ -525,7 +525,6 @@ STACK_SIZE = _CTX + _CTX_SIZE · 524 unmodified lines
525	525	########################################################################
526	526	.text
527	527	SYM_TYPED_FUNC_START(sha256_transform_rorx)
528		−.align 32
529	528	pushq %rbx
530	529	pushq %r12
531	530	pushq %r13

arch/x86/crypto/sha256-ssse3-asm.S+0−1

@@ -357,7 +357,6 @@ a = TMP_ · 356 unmodified lines
357	357	########################################################################
358	358	.text
359	359	SYM_TYPED_FUNC_START(sha256_transform_ssse3)
360		−.align 32
361	360	pushq %rbx
362	361	pushq %r12
363	362	pushq %r13

arch/x86/crypto/sha256_ni_asm.S+0−1

@@ -97,7 +97,6 @@ · 96 unmodified lines
97	97	*/
98	98
99	99	.text
100		−.align 32
101	100	SYM_TYPED_FUNC_START(sha256_ni_transform)
102	101
103	102	shl $6, NUM_BLKS /* convert to bytes */

arch/x86/crypto/sm3-avx-asm_64.S+0−1

@@ -328,7 +328,6 @@ · 327 unmodified lines
328	328	* void sm3_transform_avx(struct sm3_state *state,
329	329	* const u8 *data, int nblocks);
330	330	*/
331		−.align 16
332	331	SYM_TYPED_FUNC_START(sm3_transform_avx)
333	332	/* input:
334	333	* %rdi: ctx, CTX

arch/x86/crypto/sm4-aesni-avx-asm_64.S+0−7

@@ -140,13 +140,11 @@ · 139 unmodified lines
140	140
141	141
142	142	.text
143		−.align 16
144	143
145	144	/*
146	145	* void sm4_aesni_avx_crypt4(const u32 rk, u8 dst,
147	146	* const u8 *src, int nblocks)
148	147	*/
149		−.align 8
150	148	SYM_FUNC_START(sm4_aesni_avx_crypt4)
151	149	/* input:
152	150	* %rdi: round key array, CTX
@@ -250,7 +248,6 @@ SYM_FUNC_START(sm4_aesni_avx_crypt4) · 97 unmodified lines
250	248	RET;
251	249	SYM_FUNC_END(sm4_aesni_avx_crypt4)
252	250
253		−.align 8
254	251	SYM_FUNC_START_LOCAL(__sm4_crypt_blk8)
255	252	/* input:
256	253	* %rdi: round key array, CTX
@@ -364,7 +361,6 @@ SYM_FUNC_END(__sm4_crypt_blk8) · 107 unmodified lines
364	361	* void sm4_aesni_avx_crypt8(const u32 rk, u8 dst,
365	362	* const u8 *src, int nblocks)
366	363	*/
367		−.align 8
368	364	SYM_FUNC_START(sm4_aesni_avx_crypt8)
369	365	/* input:
370	366	* %rdi: round key array, CTX
@@ -420,7 +416,6 @@ SYM_FUNC_END(sm4_aesni_avx_crypt8) · 49 unmodified lines
420	416	* void sm4_aesni_avx_ctr_enc_blk8(const u32 rk, u8 dst,
421	417	* const u8 src, u8 iv)
422	418	*/
423		−.align 8
424	419	SYM_TYPED_FUNC_START(sm4_aesni_avx_ctr_enc_blk8)
425	420	/* input:
426	421	* %rdi: round key array, CTX
@@ -495,7 +490,6 @@ SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8) · 68 unmodified lines
495	490	* void sm4_aesni_avx_cbc_dec_blk8(const u32 rk, u8 dst,
496	491	* const u8 src, u8 iv)
497	492	*/
498		−.align 8
499	493	SYM_TYPED_FUNC_START(sm4_aesni_avx_cbc_dec_blk8)
500	494	/* input:
501	495	* %rdi: round key array, CTX
@@ -545,7 +539,6 @@ SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8) · 43 unmodified lines
545	539	* void sm4_aesni_avx_cfb_dec_blk8(const u32 rk, u8 dst,
546	540	* const u8 src, u8 iv)
547	541	*/
548		−.align 8
549	542	SYM_TYPED_FUNC_START(sm4_aesni_avx_cfb_dec_blk8)
550	543	/* input:
551	544	* %rdi: round key array, CTX

arch/x86/crypto/sm4-aesni-avx2-asm_64.S+0−6

@@ -154,9 +154,6 @@ · 153 unmodified lines
154	154	.long 0xdeadbeef, 0xdeadbeef, 0xdeadbeef
155	155
156	156	.text
157		−.align 16
158		−
159		−.align 8
160	157	SYM_FUNC_START_LOCAL(__sm4_crypt_blk16)
161	158	/* input:
162	159	* %rdi: round key array, CTX
@@ -282,7 +279,6 @@ SYM_FUNC_END(__sm4_crypt_blk16) · 119 unmodified lines
282	279	* void sm4_aesni_avx2_ctr_enc_blk16(const u32 rk, u8 dst,
283	280	* const u8 src, u8 iv)
284	281	*/
285		−.align 8
286	282	SYM_TYPED_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16)
287	283	/* input:
288	284	* %rdi: round key array, CTX
@@ -395,7 +391,6 @@ SYM_FUNC_END(sm4_aesni_avx2_ctr_enc_blk16) · 106 unmodified lines
395	391	* void sm4_aesni_avx2_cbc_dec_blk16(const u32 rk, u8 dst,
396	392	* const u8 src, u8 iv)
397	393	*/
398		−.align 8
399	394	SYM_TYPED_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16)
400	395	/* input:
401	396	* %rdi: round key array, CTX
@@ -449,7 +444,6 @@ SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16) · 47 unmodified lines
449	444	* void sm4_aesni_avx2_cfb_dec_blk16(const u32 rk, u8 dst,
450	445	* const u8 src, u8 iv)
451	446	*/
452		−.align 8
453	447	SYM_TYPED_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16)
454	448	/* input:
455	449	* %rdi: round key array, CTX

arch/x86/crypto/twofish-avx-x86_64-asm_64.S+0−2

@@ -228,7 +228,6 @@ · 227 unmodified lines
228	228	vpxor x2, wkey, x2; \
229	229	vpxor x3, wkey, x3;
230	230
231		−.align 8
232	231	SYM_FUNC_START_LOCAL(__twofish_enc_blk8)
233	232	/* input:
234	233	* %rdi: ctx, CTX
@@ -270,7 +269,6 @@ SYM_FUNC_START_LOCAL(__twofish_enc_blk8) · 35 unmodified lines
270	269	RET;
271	270	SYM_FUNC_END(__twofish_enc_blk8)
272	271
273		−.align 8
274	272	SYM_FUNC_START_LOCAL(__twofish_dec_blk8)
275	273	/* input:
276	274	* %rdi: ctx, CTX

arch/x86/entry/entry_32.S+2−2

@@ -1181,7 +1181,7 @@ SYM_CODE_START(asm_exc_nmi) · 1180 unmodified lines
1181	1181	* is using the thread stack right now, so it's safe for us to use it.
1182	1182	*/
1183	1183	movl %esp, %ebx
1184		− movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
	1184	+ movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp
1185	1185	call exc_nmi
1186	1186	movl %ebx, %esp
1187	1187
@@ -1243,7 +1243,7 @@ SYM_CODE_START(rewind_stack_and_make_dead) · 55 unmodified lines
1243	1243	/* Prevent any naive code from trying to unwind to our caller. */
1244	1244	xorl %ebp, %ebp
1245	1245
1246		− movl PER_CPU_VAR(cpu_current_top_of_stack), %esi
	1246	+ movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
1247	1247	leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1248	1248
1249	1249	call make_task_dead

arch/x86/entry/entry_64.S+28−22

@@ -92,7 +92,7 @@ SYM_CODE_START(entry_SYSCALL_64) · 91 unmodified lines
92	92	/* tss.sp2 is scratch space. */
93	93	movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
94	94	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
95		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	95	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
96	96
97	97	SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
98	98	ANNOTATE_NOENDBR
@@ -252,7 +252,7 @@ SYM_FUNC_START(__switch_to_asm) · 153 unmodified lines
252	252
253	253	#ifdef CONFIG_STACKPROTECTOR
254	254	movq TASK_stack_canary(%rsi), %rbx
255		− movq %rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
	255	+ movq %rbx, PER_CPU_VAR(fixed_percpu_data) + FIXED_stack_canary
256	256	#endif
257	257
258	258	/*
@@ -284,9 +284,11 @@ SYM_FUNC_END(__switch_to_asm) · 25 unmodified lines
284	284	* r12: kernel thread arg
285	285	*/
286	286	.pushsection .text, "ax"
287		−SYM_CODE_START(ret_from_fork)
	287	+ __FUNC_ALIGN
	288	+SYM_CODE_START_NOALIGN(ret_from_fork)
288	289	UNWIND_HINT_EMPTY
289	290	ANNOTATE_NOENDBR // copy_thread
	291	+ CALL_DEPTH_ACCOUNT
290	292	movq %rax, %rdi
291	293	call schedule_tail /* rdi: 'prev' task parameter */
292	294
@@ -326,11 +328,12 @@ SYM_CODE_END(ret_from_fork) · 33 unmodified lines
326	328	#endif
327	329	.endm
328	330
329		−SYM_CODE_START_LOCAL(xen_error_entry)
	331	+SYM_CODE_START(xen_error_entry)
	332	+ ANNOTATE_NOENDBR
330	333	UNWIND_HINT_FUNC
331	334	PUSH_AND_CLEAR_REGS save_ret=1
332	335	ENCODE_FRAME_POINTER 8
333		− UNTRAIN_RET
	336	+ UNTRAIN_RET_FROM_CALL
334	337	RET
335	338	SYM_CODE_END(xen_error_entry)
336	339
@@ -600,13 +603,13 @@ SYM_CODE_END(\asmsym) · 263 unmodified lines
600	603	* shared between 32 and 64 bit and emit the __irqentry_text_* markers
601	604	* so the stacktrace boundary checks work.
602	605	*/
603		− .align 16
	606	+ __ALIGN
604	607	.globl __irqentry_text_start
605	608	__irqentry_text_start:
606	609
607	610	#include <asm/idtentry.h>
608	611
609		− .align 16
	612	+ __ALIGN
610	613	.globl __irqentry_text_end
611	614	__irqentry_text_end:
612	615	ANNOTATE_NOENDBR
@@ -828,7 +831,8 @@ EXPORT_SYMBOL(asm_load_gs_index) · 215 unmodified lines
828	831	*
829	832	* C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
830	833	*/
831		−SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
	834	+ __FUNC_ALIGN
	835	+SYM_CODE_START_LOCAL_NOALIGN(exc_xen_hypervisor_callback)
832	836
833	837	/*
834	838	* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
@@ -856,7 +860,8 @@ SYM_CODE_END(exc_xen_hypervisor_callback) · 21 unmodified lines
856	860	* We distinguish between categories by comparing each saved segment register
857	861	* with its current contents: any discrepancy means we in category 1.
858	862	*/
859		−SYM_CODE_START(xen_failsafe_callback)
	863	+ __FUNC_ALIGN
	864	+SYM_CODE_START_NOALIGN(xen_failsafe_callback)
860	865	UNWIND_HINT_EMPTY
861	866	ENDBR
862	867	movl %ds, %ecx
@@ -903,7 +908,8 @@ SYM_CODE_END(xen_failsafe_callback) · 40 unmodified lines
903	908	* R14 - old CR3
904	909	* R15 - old SPEC_CTRL
905	910	*/
906		−SYM_CODE_START_LOCAL(paranoid_entry)
	911	+SYM_CODE_START(paranoid_entry)
	912	+ ANNOTATE_NOENDBR
907	913	UNWIND_HINT_FUNC
908	914	PUSH_AND_CLEAR_REGS save_ret=1
909	915	ENCODE_FRAME_POINTER 8
@@ -972,7 +978,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) · 62 unmodified lines
972	978	* CR3 above, keep the old value in a callee saved register.
973	979	*/
974	980	IBRS_ENTER save_reg=%r15
975		− UNTRAIN_RET
	981	+ UNTRAIN_RET_FROM_CALL
976	982
977	983	RET
978	984	SYM_CODE_END(paranoid_entry)
@@ -1038,7 +1044,8 @@ SYM_CODE_END(paranoid_exit) · 59 unmodified lines
1038	1044	/*
1039	1045	* Switch GS and CR3 if needed.
1040	1046	*/
1041		−SYM_CODE_START_LOCAL(error_entry)
	1047	+SYM_CODE_START(error_entry)
	1048	+ ANNOTATE_NOENDBR
1042	1049	UNWIND_HINT_FUNC
1043	1050
1044	1051	PUSH_AND_CLEAR_REGS save_ret=1
@@ -1056,14 +1063,11 @@ SYM_CODE_START_LOCAL(error_entry) · 11 unmodified lines
1056	1063	/* We have user CR3. Change to kernel CR3. */
1057	1064	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1058	1065	IBRS_ENTER
1059		− UNTRAIN_RET
	1066	+ UNTRAIN_RET_FROM_CALL
1060	1067
1061	1068	leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
1062		−.Lerror_entry_from_usermode_after_swapgs:
1063		−
1064	1069	/* Put us onto the real thread stack. */
1065		− call sync_regs
1066		− RET
	1070	+ jmp sync_regs
1067	1071
1068	1072	/*
1069	1073	* There are two places in the kernel that can potentially fault with
@@ -1094,6 +1098,7 @@ SYM_CODE_START_LOCAL(error_entry) · 24 unmodified lines
1094	1098	*/
1095	1099	.Lerror_entry_done_lfence:
1096	1100	FENCE_SWAPGS_KERNEL_ENTRY
	1101	+ CALL_DEPTH_ACCOUNT
1097	1102	leaq 8(%rsp), %rax /* return pt_regs pointer */
1098	1103	ANNOTATE_UNRET_END
1099	1104	RET
@@ -1112,7 +1117,7 @@ SYM_CODE_START_LOCAL(error_entry) · 12 unmodified lines
1112	1117	FENCE_SWAPGS_USER_ENTRY
1113	1118	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1114	1119	IBRS_ENTER
1115		− UNTRAIN_RET
	1120	+ UNTRAIN_RET_FROM_CALL
1116	1121
1117	1122	/*
1118	1123	* Pretend that the exception came from user mode: set up pt_regs
@@ -1121,7 +1126,7 @@ SYM_CODE_START_LOCAL(error_entry) · 2 unmodified lines
1121	1126	leaq 8(%rsp), %rdi /* arg0 = pt_regs pointer */
1122	1127	call fixup_bad_iret
1123	1128	mov %rax, %rdi
1124		− jmp .Lerror_entry_from_usermode_after_swapgs
	1129	+ jmp sync_regs
1125	1130	SYM_CODE_END(error_entry)
1126	1131
1127	1132	SYM_CODE_START_LOCAL(error_return)
@@ -1206,7 +1211,7 @@ SYM_CODE_START(asm_exc_nmi) · 78 unmodified lines
1206	1211	FENCE_SWAPGS_USER_ENTRY
1207	1212	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1208	1213	movq %rsp, %rdx
1209		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	1214	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
1210	1215	UNWIND_HINT_IRET_REGS base=%rdx offset=8
1211	1216	pushq 58(%rdx) / pt_regs->ss */
1212	1217	pushq 48(%rdx) / pt_regs->rsp */
@@ -1516,12 +1521,13 @@ SYM_CODE_END(ignore_sysret) · 303 unmodified lines
1516	1521	#endif
1517	1522
1518	1523	.pushsection .text, "ax"
1519		−SYM_CODE_START(rewind_stack_and_make_dead)
	1524	+ __FUNC_ALIGN
	1525	+SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead)
1520	1526	UNWIND_HINT_FUNC
1521	1527	/* Prevent any naive code from trying to unwind to our caller. */
1522	1528	xorl %ebp, %ebp
1523	1529
1524		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
	1530	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
1525	1531	leaq -PTREGS_SIZE(%rax), %rsp
1526	1532	UNWIND_HINT_REGS
1527	1533

arch/x86/entry/entry_64_compat.S+3−4

@@ -58,7 +58,7 @@ SYM_CODE_START(entry_SYSENTER_compat) · 57 unmodified lines
58	58	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
59	59	popq %rax
60	60
61		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	61	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
62	62
63	63	/* Construct struct pt_regs on stack */
64	64	pushq $__USER_DS /* pt_regs->ss */
@@ -128,7 +128,6 @@ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) · 63 unmodified lines
128	128	popfq
129	129	jmp .Lsysenter_flags_fixed
130	130	SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL)
131		− ANNOTATE_NOENDBR // is_sysenter_singlestep
132	131	SYM_CODE_END(entry_SYSENTER_compat)
133	132
134	133	/*
@@ -191,7 +190,7 @@ SYM_CODE_START(entry_SYSCALL_compat) · 56 unmodified lines
191	190	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
192	191
193	192	/* Switch to the kernel stack */
194		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	193	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
195	194
196	195	SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
197	196	ANNOTATE_NOENDBR
@@ -332,7 +331,7 @@ SYM_CODE_START(entry_INT80_compat) · 134 unmodified lines
332	331	ALTERNATIVE "", "jmp .Lint80_keep_stack", X86_FEATURE_XENPV
333	332
334	333	movq %rsp, %rax
335		− movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
	334	+ movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
336	335
337	336	pushq 58(%rax) / regs->ss */
338	337	pushq 48(%rax) / regs->rsp */

arch/x86/entry/thunk_64.S+2−2

@@ -11,7 +11,7 @@ · 10 unmodified lines
11	11
12	12	/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
13	13	.macro THUNK name, func
14		−SYM_FUNC_START_NOALIGN(\name)
	14	+SYM_FUNC_START(\name)
15	15	pushq %rbp
16	16	movq %rsp, %rbp
17	17
@@ -36,7 +36,7 @@ SYM_FUNC_END(\name) · 18 unmodified lines
36	36	EXPORT_SYMBOL(preempt_schedule_thunk)
37	37	EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
38	38
39		−SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore)
	39	+SYM_CODE_START_LOCAL(__thunk_restore)
40	40	popq %r11
41	41	popq %r10
42	42	popq %r9

arch/x86/entry/vdso/Makefile+8−6

@@ -33,11 +33,12 @@ vobjs32-y += vdso32/vclock_gettime.o · 32 unmodified lines
33	33	vobjs-$(CONFIG_X86_SGX) += vsgx.o
34	34
35	35	# files to link into kernel
36		−obj-y += vma.o extable.o
37		−KASAN_SANITIZE_vma.o := y
38		−UBSAN_SANITIZE_vma.o := y
39		−KCSAN_SANITIZE_vma.o := y
40		−OBJECT_FILES_NON_STANDARD_vma.o := n
	36	+obj-y += vma.o extable.o
	37	+KASAN_SANITIZE_vma.o := y
	38	+UBSAN_SANITIZE_vma.o := y
	39	+KCSAN_SANITIZE_vma.o := y
	40	+OBJECT_FILES_NON_STANDARD_vma.o := n
	41	+OBJECT_FILES_NON_STANDARD_extable.o := n
41	42
42	43	# vDSO images to build
43	44	vdso_img-$(VDSO64-y) += 64
@@ -94,7 +95,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) · 50 unmodified lines
94	95	endif
95	96	endif
96	97
97		−$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
	98	+$(vobjs): KBUILD_CFLAGS := $(filter-out $(PADDING_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
98	99	$(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
99	100
100	101	#
@@ -157,6 +158,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) · 56 unmodified lines
157	158	KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
158	159	KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
159	160	KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))
	161	+KBUILD_CFLAGS_32 := $(filter-out $(PADDING_CFLAGS),$(KBUILD_CFLAGS_32))
160	162	KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic
161	163	KBUILD_CFLAGS_32 += -fno-stack-protector
162	164	KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)

arch/x86/include/asm/alternative.h+65−3

@@ -78,8 +78,43 @@ extern void apply_alternatives(struct alt_instr start, struct alt_instr end); · 77 unmodified lines
78	78	extern void apply_retpolines(s32 start, s32 end);
79	79	extern void apply_returns(s32 start, s32 end);
80	80	extern void apply_ibt_endbr(s32 start, s32 end);
	81	+extern void apply_fineibt(s32 start_retpoline, s32 end_retpoine,
	82	+ s32 start_cfi, s32 end_cfi);
81	83
82	84	struct module;
	85	+struct paravirt_patch_site;
	86	+
	87	+struct callthunk_sites {
	88	+ s32 call_start, call_end;
	89	+ struct paravirt_patch_site pv_start, pv_end;
	90	+};
	91	+
	92	+#ifdef CONFIG_CALL_THUNKS
	93	+extern void callthunks_patch_builtin_calls(void);
	94	+extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
	95	+ struct module *mod);
	96	+extern void callthunks_translate_call_dest(void dest);
	97	+extern bool is_callthunk(void *addr);
	98	+extern int x86_call_depth_emit_accounting(u8 *pprog, void func);
	99	+#else
	100	+static __always_inline void callthunks_patch_builtin_calls(void) {}
	101	+static __always_inline void
	102	+callthunks_patch_module_calls(struct callthunk_sites *sites,
	103	+ struct module *mod) {}
	104	+static __always_inline void callthunks_translate_call_dest(void dest)
	105	+{
	106	+ return dest;
	107	+}
	108	+static __always_inline bool is_callthunk(void *addr)
	109	+{
	110	+ return false;
	111	+}
	112	+static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
	113	+ void *func)
	114	+{
	115	+ return 0;
	116	+}
	117	+#endif
83	118
84	119	#ifdef CONFIG_SMP
85	120	extern void alternatives_smp_module_add(struct module mod, char name,
@@ -347,6 +382,7 @@ static inline int alternatives_text_reserved(void start, void end) · 261 unmodified lines
347	382	#define old_len 141b-140b
348	383	#define new_len1 144f-143f
349	384	#define new_len2 145f-144f
	385	+#define new_len3 146f-145f
350	386
351	387	/*
352	388	* gas compatible max based on the idea from:
@@ -354,7 +390,8 @@ static inline int alternatives_text_reserved(void start, void end) · 1 unmodified line
354	390	*
355	391	* The additional "-" is needed because gas uses a "true" value of -1.
356	392	*/
357		−#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
	393	+#define alt_max_2(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
	394	+#define alt_max_3(a, b, c) (alt_max_2(alt_max_2(a, b), c))
358	395
359	396
360	397	/*
@@ -366,13 +403,36 @@ static inline int alternatives_text_reserved(void start, void end) · 5 unmodified lines
366	403	140:
367	404	\oldinstr
368	405	141:
369		− .skip -((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \
370		− (alt_max_short(new_len1, new_len2) - (old_len)),0x90
	406	+ .skip -((alt_max_2(new_len1, new_len2) - (old_len)) > 0) * \
	407	+ (alt_max_2(new_len1, new_len2) - (old_len)),0x90
	408	+142:
	409	+
	410	+ .pushsection .altinstructions,"a"
	411	+ altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
	412	+ altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
	413	+ .popsection
	414	+
	415	+ .pushsection .altinstr_replacement,"ax"
	416	+143:
	417	+ \newinstr1
	418	+144:
	419	+ \newinstr2
	420	+145:
	421	+ .popsection
	422	+.endm
	423	+
	424	+.macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3
	425	+140:
	426	+ \oldinstr
	427	+141:
	428	+ .skip -((alt_max_3(new_len1, new_len2, new_len3) - (old_len)) > 0) * \
	429	+ (alt_max_3(new_len1, new_len2, new_len3) - (old_len)),0x90
371	430	142:
372	431
373	432	.pushsection .altinstructions,"a"
374	433	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
375	434	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
	435	+ altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f
376	436	.popsection
377	437
378	438	.pushsection .altinstr_replacement,"ax"
@@ -381,6 +441,8 @@ static inline int alternatives_text_reserved(void start, void end) · 2 unmodified lines
381	441	144:
382	442	\newinstr2
383	443	145:
	444	+ \newinstr3
	445	+146:
384	446	.popsection
385	447	.endm
386	448

arch/x86/include/asm/cpufeatures.h+1−2

@@ -305,8 +305,7 @@ · 304 unmodified lines
305	305	#define X86_FEATURE_USE_IBPB_FW (1132+16) / "" Use IBPB during runtime firmware calls */
306	306	#define X86_FEATURE_RSB_VMEXIT_LITE (1132+17) / "" Fill RSB on VM exit when EIBRS is enabled */
307	307	#define X86_FEATURE_SGX_EDECCSSA (1132+18) / "" SGX EDECCSSA user leaf function */
308		−
309		−
	308	+#define X86_FEATURE_CALL_DEPTH (1132+19) / "" Call depth tracking for RSB stuffing */
310	309	#define X86_FEATURE_MSR_TSX_CTRL (1132+20) / "" MSR IA32_TSX_CTRL (Intel) implemented */
311	310
312	311	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */

arch/x86/include/asm/current.h+29−3

@@ -3,16 +3,42 @@ · 2 unmodified lines
3	3	#define _ASM_X86_CURRENT_H
4	4
5	5	#include <linux/compiler.h>
6		−#include <asm/percpu.h>
7	6
8	7	#ifndef __ASSEMBLY__
	8	+
	9	+#include <linux/cache.h>
	10	+#include <asm/percpu.h>
	11	+
9	12	struct task_struct;
10	13
11		−DECLARE_PER_CPU(struct task_struct *, current_task);
	14	+struct pcpu_hot {
	15	+ union {
	16	+ struct {
	17	+ struct task_struct *current_task;
	18	+ int preempt_count;
	19	+ int cpu_number;
	20	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	21	+ u64 call_depth;
	22	+#endif
	23	+ unsigned long top_of_stack;
	24	+ void *hardirq_stack_ptr;
	25	+ u16 softirq_pending;
	26	+#ifdef CONFIG_X86_64
	27	+ bool hardirq_stack_inuse;
	28	+#else
	29	+ void *softirq_stack_ptr;
	30	+#endif
	31	+ };
	32	+ u8 pad[64];
	33	+ };
	34	+};
	35	+static_assert(sizeof(struct pcpu_hot) == 64);
	36	+
	37	+DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
12	38
13	39	static __always_inline struct task_struct *get_current(void)
14	40	{
15		− return this_cpu_read_stable(current_task);
	41	+ return this_cpu_read_stable(pcpu_hot.current_task);
16	42	}
17	43
18	44	#define current get_current()

arch/x86/include/asm/debugreg.h+1−1

@@ -2,8 +2,8 @@ · 1 unmodified line
2	2	#ifndef _ASM_X86_DEBUGREG_H
3	3	#define _ASM_X86_DEBUGREG_H
4	4
5		−
6	5	#include <linux/bug.h>
	6	+#include <linux/percpu.h>
7	7	#include <uapi/asm/debugreg.h>
8	8
9	9	DECLARE_PER_CPU(unsigned long, cpu_dr7);

arch/x86/include/asm/disabled-features.h+8−1

@@ -69,6 +69,12 @@ · 68 unmodified lines
69	69	# define DISABLE_UNRET (1 << (X86_FEATURE_UNRET & 31))
70	70	#endif
71	71
	72	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	73	+# define DISABLE_CALL_DEPTH_TRACKING 0
	74	+#else
	75	+# define DISABLE_CALL_DEPTH_TRACKING (1 << (X86_FEATURE_CALL_DEPTH & 31))
	76	+#endif
	77	+
72	78	#ifdef CONFIG_INTEL_IOMMU_SVM
73	79	# define DISABLE_ENQCMD 0
74	80	#else
@@ -107,7 +113,8 @@ · 32 unmodified lines
107	113	#define DISABLED_MASK8 (DISABLE_XENPV\|DISABLE_TDX_GUEST)
108	114	#define DISABLED_MASK9 (DISABLE_SGX)
109	115	#define DISABLED_MASK10 0
110		−#define DISABLED_MASK11 (DISABLE_RETPOLINE\|DISABLE_RETHUNK\|DISABLE_UNRET)
	116	+#define DISABLED_MASK11 (DISABLE_RETPOLINE\|DISABLE_RETHUNK\|DISABLE_UNRET\| \
	117	+ DISABLE_CALL_DEPTH_TRACKING)
111	118	#define DISABLED_MASK12 0
112	119	#define DISABLED_MASK13 0
113	120	#define DISABLED_MASK14 0

arch/x86/include/asm/hardirq.h+2−1

@@ -3,9 +3,9 @@ · 2 unmodified lines
3	3	#define _ASM_X86_HARDIRQ_H
4	4
5	5	#include <linux/threads.h>
	6	+#include <asm/current.h>
6	7
7	8	typedef struct {
8		− u16 __softirq_pending;
9	9	#if IS_ENABLED(CONFIG_KVM_INTEL)
10	10	u8 kvm_cpu_l1tf_flush_l1d;
11	11	#endif
@@ -60,6 +60,7 @@ extern u64 arch_irq_stat_cpu(unsigned int cpu); · 48 unmodified lines
60	60	extern u64 arch_irq_stat(void);
61	61	#define arch_irq_stat arch_irq_stat
62	62
	63	+#define local_softirq_pending_ref pcpu_hot.softirq_pending
63	64
64	65	#if IS_ENABLED(CONFIG_KVM_INTEL)
65	66	static inline void kvm_set_cpu_l1tf_flush_l1d(void)

arch/x86/include/asm/irq_stack.h+6−6

@@ -116,7 +116,7 @@ · 115 unmodified lines
116	116	ASM_CALL_ARG2
117	117
118	118	#define call_on_irqstack(func, asm_call, argconstr...) \
119		− call_on_stack(__this_cpu_read(hardirq_stack_ptr), \
	119	+ call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \
120	120	func, asm_call, argconstr)
121	121
122	122	/* Macros to assert type correctness for run__on_irqstack macros /
@@ -135,7 +135,7 @@ · 12 unmodified lines
135	135	* User mode entry and interrupt on the irq stack do not \
136	136	* switch stacks. If from user mode the task stack is empty. \
137	137	*/ \
138		− if (user_mode(regs) \|\| __this_cpu_read(hardirq_stack_inuse)) { \
	138	+ if (user_mode(regs) \|\| __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
139	139	irq_enter_rcu(); \
140	140	func(c_args); \
141	141	irq_exit_rcu(); \
@@ -146,9 +146,9 @@ · 4 unmodified lines
146	146	* places. Invoke the stack switch macro with the call \
147	147	* sequence which matches the above direct invocation. \
148	148	*/ \
149		− __this_cpu_write(hardirq_stack_inuse, true); \
	149	+ __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
150	150	call_on_irqstack(func, asm_call, constr); \
151		− __this_cpu_write(hardirq_stack_inuse, false); \
	151	+ __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
152	152	} \
153	153	}
154	154
@@ -212,9 +212,9 @@ · 57 unmodified lines
212	212	*/
213	213	#define do_softirq_own_stack() \
214	214	{ \
215		− __this_cpu_write(hardirq_stack_inuse, true); \
	215	+ __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
216	216	call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \
217		− __this_cpu_write(hardirq_stack_inuse, false); \
	217	+ __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
218	218	}
219	219
220	220	#endif

arch/x86/include/asm/linkage.h+55−8

@@ -12,13 +12,26 @@ · 11 unmodified lines
12	12	#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
13	13	#endif /* CONFIG_X86_32 */
14	14
15		−#ifdef __ASSEMBLY__
16		−
17		−#if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_ALIGNMENT_16)
18		−#define __ALIGN .p2align 4, 0x90
	15	+#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT, 0x90;
19	16	#define __ALIGN_STR __stringify(__ALIGN)
	17	+
	18	+#if defined(CONFIG_CALL_PADDING) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
	19	+#define FUNCTION_PADDING .skip CONFIG_FUNCTION_ALIGNMENT, 0x90;
	20	+#else
	21	+#define FUNCTION_PADDING
	22	+#endif
	23	+
	24	+#if (CONFIG_FUNCTION_ALIGNMENT > 8) && !defined(__DISABLE_EXPORTS) && !defined(BULID_VDSO)
	25	+# define __FUNC_ALIGN __ALIGN; FUNCTION_PADDING
	26	+#else
	27	+# define __FUNC_ALIGN __ALIGN
20	28	#endif
21	29
	30	+#define ASM_FUNC_ALIGN __stringify(__FUNC_ALIGN)
	31	+#define SYM_F_ALIGN __FUNC_ALIGN
	32	+
	33	+#ifdef __ASSEMBLY__
	34	+
22	35	#if defined(CONFIG_RETHUNK) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
23	36	#define RET jmp __x86_return_thunk
24	37	#else /* CONFIG_RETPOLINE */
@@ -43,11 +56,45 @@ · 18 unmodified lines
43	56
44	57	#endif /* __ASSEMBLY__ */
45	58
	59	+/*
	60	+ * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the
	61	+ * CFI symbol layout changes.
	62	+ *
	63	+ * Without CALL_THUNKS:
	64	+ *
	65	+ * .align FUNCTION_ALIGNMENT
	66	+ * __cfi_##name:
	67	+ * .skip FUNCTION_PADDING, 0x90
	68	+ * .byte 0xb8
	69	+ * .long __kcfi_typeid_##name
	70	+ * name:
	71	+ *
	72	+ * With CALL_THUNKS:
	73	+ *
	74	+ * .align FUNCTION_ALIGNMENT
	75	+ * __cfi_##name:
	76	+ * .byte 0xb8
	77	+ * .long __kcfi_typeid_##name
	78	+ * .skip FUNCTION_PADDING, 0x90
	79	+ * name:
	80	+ *
	81	+ * In both cases the whole thing is FUNCTION_ALIGNMENT aligned and sized.
	82	+ */
	83	+
	84	+#ifdef CONFIG_CALL_PADDING
	85	+#define CFI_PRE_PADDING
	86	+#define CFI_POST_PADDING .skip CONFIG_FUNCTION_PADDING_BYTES, 0x90;
	87	+#else
	88	+#define CFI_PRE_PADDING .skip CONFIG_FUNCTION_PADDING_BYTES, 0x90;
	89	+#define CFI_POST_PADDING
	90	+#endif
	91	+
46	92	#define __CFI_TYPE(name) \
47	93	SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \
48		− .fill 11, 1, 0x90 ASM_NL \
	94	+ CFI_PRE_PADDING \
49	95	.byte 0xb8 ASM_NL \
50	96	.long __kcfi_typeid_##name ASM_NL \
	97	+ CFI_POST_PADDING \
51	98	SYM_FUNC_END(__cfi_##name)
52	99
53	100	/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
@@ -57,7 +104,7 @@ · 3 unmodified lines
57	104
58	105	/* SYM_FUNC_START -- use for global functions */
59	106	#define SYM_FUNC_START(name) \
60		− SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
	107	+ SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \
61	108	ENDBR
62	109
63	110	/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */
@@ -67,7 +114,7 @@ · 3 unmodified lines
67	114
68	115	/* SYM_FUNC_START_LOCAL -- use for local functions */
69	116	#define SYM_FUNC_START_LOCAL(name) \
70		− SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \
	117	+ SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN) \
71	118	ENDBR
72	119
73	120	/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */
@@ -77,7 +124,7 @@ · 3 unmodified lines
77	124
78	125	/* SYM_FUNC_START_WEAK -- use for weak functions */
79	126	#define SYM_FUNC_START_WEAK(name) \
80		− SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \
	127	+ SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN) \
81	128	ENDBR
82	129
83	130	/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */

arch/x86/include/asm/nospec-branch.h+170−6

@@ -12,8 +12,104 @@ · 11 unmodified lines
12	12	#include <asm/msr-index.h>
13	13	#include <asm/unwind_hints.h>
14	14	#include <asm/percpu.h>
	15	+#include <asm/current.h>
15	16
16		−#define RETPOLINE_THUNK_SIZE 32
	17	+/*
	18	+ * Call depth tracking for Intel SKL CPUs to address the RSB underflow
	19	+ * issue in software.
	20	+ *
	21	+ * The tracking does not use a counter. It uses uses arithmetic shift
	22	+ * right on call entry and logical shift left on return.
	23	+ *
	24	+ * The depth tracking variable is initialized to 0x8000.... when the call
	25	+ * depth is zero. The arithmetic shift right sign extends the MSB and
	26	+ * saturates after the 12th call. The shift count is 5 for both directions
	27	+ * so the tracking covers 12 nested calls.
	28	+ *
	29	+ * Call
	30	+ * 0: 0x8000000000000000 0x0000000000000000
	31	+ * 1: 0xfc00000000000000 0xf000000000000000
	32	+ * ...
	33	+ * 11: 0xfffffffffffffff8 0xfffffffffffffc00
	34	+ * 12: 0xffffffffffffffff 0xffffffffffffffe0
	35	+ *
	36	+ * After a return buffer fill the depth is credited 12 calls before the
	37	+ * next stuffing has to take place.
	38	+ *
	39	+ * There is a inaccuracy for situations like this:
	40	+ *
	41	+ * 10 calls
	42	+ * 5 returns
	43	+ * 3 calls
	44	+ * 4 returns
	45	+ * 3 calls
	46	+ * ....
	47	+ *
	48	+ * The shift count might cause this to be off by one in either direction,
	49	+ * but there is still a cushion vs. the RSB depth. The algorithm does not
	50	+ * claim to be perfect and it can be speculated around by the CPU, but it
	51	+ * is considered that it obfuscates the problem enough to make exploitation
	52	+ * extremly difficult.
	53	+ */
	54	+#define RET_DEPTH_SHIFT 5
	55	+#define RSB_RET_STUFF_LOOPS 16
	56	+#define RET_DEPTH_INIT 0x8000000000000000ULL
	57	+#define RET_DEPTH_INIT_FROM_CALL 0xfc00000000000000ULL
	58	+#define RET_DEPTH_CREDIT 0xffffffffffffffffULL
	59	+
	60	+#ifdef CONFIG_CALL_THUNKS_DEBUG
	61	+# define CALL_THUNKS_DEBUG_INC_CALLS \
	62	+ incq %gs:__x86_call_count;
	63	+# define CALL_THUNKS_DEBUG_INC_RETS \
	64	+ incq %gs:__x86_ret_count;
	65	+# define CALL_THUNKS_DEBUG_INC_STUFFS \
	66	+ incq %gs:__x86_stuffs_count;
	67	+# define CALL_THUNKS_DEBUG_INC_CTXSW \
	68	+ incq %gs:__x86_ctxsw_count;
	69	+#else
	70	+# define CALL_THUNKS_DEBUG_INC_CALLS
	71	+# define CALL_THUNKS_DEBUG_INC_RETS
	72	+# define CALL_THUNKS_DEBUG_INC_STUFFS
	73	+# define CALL_THUNKS_DEBUG_INC_CTXSW
	74	+#endif
	75	+
	76	+#if defined(CONFIG_CALL_DEPTH_TRACKING) && !defined(COMPILE_OFFSETS)
	77	+
	78	+#include <asm/asm-offsets.h>
	79	+
	80	+#define CREDIT_CALL_DEPTH \
	81	+ movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
	82	+
	83	+#define ASM_CREDIT_CALL_DEPTH \
	84	+ movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
	85	+
	86	+#define RESET_CALL_DEPTH \
	87	+ mov $0x80, %rax; \
	88	+ shl $56, %rax; \
	89	+ movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
	90	+
	91	+#define RESET_CALL_DEPTH_FROM_CALL \
	92	+ mov $0xfc, %rax; \
	93	+ shl $56, %rax; \
	94	+ movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
	95	+ CALL_THUNKS_DEBUG_INC_CALLS
	96	+
	97	+#define INCREMENT_CALL_DEPTH \
	98	+ sarq $5, %gs:pcpu_hot + X86_call_depth; \
	99	+ CALL_THUNKS_DEBUG_INC_CALLS
	100	+
	101	+#define ASM_INCREMENT_CALL_DEPTH \
	102	+ sarq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
	103	+ CALL_THUNKS_DEBUG_INC_CALLS
	104	+
	105	+#else
	106	+#define CREDIT_CALL_DEPTH
	107	+#define ASM_CREDIT_CALL_DEPTH
	108	+#define RESET_CALL_DEPTH
	109	+#define INCREMENT_CALL_DEPTH
	110	+#define ASM_INCREMENT_CALL_DEPTH
	111	+#define RESET_CALL_DEPTH_FROM_CALL
	112	+#endif
17	113
18	114	/*
19	115	* Fill the CPU return stack buffer.
@@ -32,6 +128,7 @@ · 12 unmodified lines
32	128	* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
33	129	*/
34	130
	131	+#define RETPOLINE_THUNK_SIZE 32
35	132	#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
36	133
37	134	/*
@@ -60,7 +157,9 @@ · 22 unmodified lines
60	157	dec reg; \
61	158	jnz 771b; \
62	159	/* barrier for jnz misprediction */ \
63		− lfence;
	160	+ lfence; \
	161	+ ASM_CREDIT_CALL_DEPTH \
	162	+ CALL_THUNKS_DEBUG_INC_CTXSW
64	163	#else
65	164	/*
66	165	* i386 doesn't unconditionally have LFENCE, as such it can't
@@ -185,11 +284,32 @@ · 118 unmodified lines
185	284	* where we have a stack but before any RET instruction.
186	285	*/
187	286	.macro UNTRAIN_RET
188		−#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY)
	287	+#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY) \|\| \
	288	+ defined(CONFIG_CALL_DEPTH_TRACKING)
189	289	ANNOTATE_UNRET_END
190		− ALTERNATIVE_2 "", \
191		− CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
192		− "call entry_ibpb", X86_FEATURE_ENTRY_IBPB
	290	+ ALTERNATIVE_3 "", \
	291	+ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
	292	+ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
	293	+ __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
	294	+#endif
	295	+.endm
	296	+
	297	+.macro UNTRAIN_RET_FROM_CALL
	298	+#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY) \|\| \
	299	+ defined(CONFIG_CALL_DEPTH_TRACKING)
	300	+ ANNOTATE_UNRET_END
	301	+ ALTERNATIVE_3 "", \
	302	+ CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
	303	+ "call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
	304	+ __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH
	305	+#endif
	306	+.endm
	307	+
	308	+
	309	+.macro CALL_DEPTH_ACCOUNT
	310	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	311	+ ALTERNATIVE "", \
	312	+ __stringify(ASM_INCREMENT_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
193	313	#endif
194	314	.endm
195	315
@@ -203,11 +323,45 @@ · 7 unmodified lines
203	323
204	324	typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
205	325	extern retpoline_thunk_t __x86_indirect_thunk_array[];
	326	+extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
	327	+extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
206	328
207	329	extern void __x86_return_thunk(void);
208	330	extern void zen_untrain_ret(void);
209	331	extern void entry_ibpb(void);
210	332
	333	+#ifdef CONFIG_CALL_THUNKS
	334	+extern void (*x86_return_thunk)(void);
	335	+#else
	336	+#define x86_return_thunk (&__x86_return_thunk)
	337	+#endif
	338	+
	339	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	340	+extern void __x86_return_skl(void);
	341	+
	342	+static inline void x86_set_skl_return_thunk(void)
	343	+{
	344	+ x86_return_thunk = &__x86_return_skl;
	345	+}
	346	+
	347	+#define CALL_DEPTH_ACCOUNT \
	348	+ ALTERNATIVE("", \
	349	+ __stringify(INCREMENT_CALL_DEPTH), \
	350	+ X86_FEATURE_CALL_DEPTH)
	351	+
	352	+#ifdef CONFIG_CALL_THUNKS_DEBUG
	353	+DECLARE_PER_CPU(u64, __x86_call_count);
	354	+DECLARE_PER_CPU(u64, __x86_ret_count);
	355	+DECLARE_PER_CPU(u64, __x86_stuffs_count);
	356	+DECLARE_PER_CPU(u64, __x86_ctxsw_count);
	357	+#endif
	358	+#else
	359	+static inline void x86_set_skl_return_thunk(void) {}
	360	+
	361	+#define CALL_DEPTH_ACCOUNT ""
	362	+
	363	+#endif
	364	+
211	365	#ifdef CONFIG_RETPOLINE
212	366
213	367	#define GEN(reg) \
@@ -215,6 +369,16 @@ extern void entry_ibpb(void); · 1 unmodified line
215	369	#include <asm/GEN-for-each-reg.h>
216	370	#undef GEN
217	371
	372	+#define GEN(reg) \
	373	+ extern retpoline_thunk_t __x86_indirect_call_thunk_ ## reg;
	374	+#include <asm/GEN-for-each-reg.h>
	375	+#undef GEN
	376	+
	377	+#define GEN(reg) \
	378	+ extern retpoline_thunk_t __x86_indirect_jump_thunk_ ## reg;
	379	+#include <asm/GEN-for-each-reg.h>
	380	+#undef GEN
	381	+
218	382	#ifdef CONFIG_X86_64
219	383
220	384	/*

arch/x86/include/asm/paravirt.h+15−2

@@ -4,13 +4,13 @@ · 3 unmodified lines
4	4	/* Various instructions on x86 need to be replaced for
5	5	* para-virtualization: those hooks are defined here. */
6	6
	7	+#include <asm/paravirt_types.h>
	8	+
7	9	#ifdef CONFIG_PARAVIRT
8	10	#include <asm/pgtable_types.h>
9	11	#include <asm/asm.h>
10	12	#include <asm/nospec-branch.h>
11	13
12		−#include <asm/paravirt_types.h>
13		−
14	14	#ifndef __ASSEMBLY__
15	15	#include <linux/bug.h>
16	16	#include <linux/types.h>
@@ -665,6 +665,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); · 648 unmodified lines
665	665	asm(".pushsection " section ", \"ax\";" \
666	666	".globl " PV_THUNK_NAME(func) ";" \
667	667	".type " PV_THUNK_NAME(func) ", @function;" \
	668	+ ASM_FUNC_ALIGN \
668	669	PV_THUNK_NAME(func) ":" \
669	670	ASM_ENDBR \
670	671	FRAME_BEGIN \
@@ -730,6 +731,18 @@ static __always_inline unsigned long arch_local_irq_save(void) · 59 unmodified lines
730	731	#undef PVOP_VCALL4
731	732	#undef PVOP_CALL4
732	733
	734	+#define DEFINE_PARAVIRT_ASM(func, instr, sec) \
	735	+ asm (".pushsection " #sec ", \"ax\"\n" \
	736	+ ".global " #func "\n\t" \
	737	+ ".type " #func ", @function\n\t" \
	738	+ ASM_FUNC_ALIGN "\n" \
	739	+ #func ":\n\t" \
	740	+ ASM_ENDBR \
	741	+ instr "\n\t" \
	742	+ ASM_RET \
	743	+ ".size " #func ", . - " #func "\n\t" \
	744	+ ".popsection")
	745	+
733	746	extern void default_banner(void);
734	747
735	748	#else /* __ASSEMBLY__ */

arch/x86/include/asm/paravirt_types.h+19−15

@@ -2,6 +2,24 @@ · 1 unmodified line
2	2	#ifndef _ASM_X86_PARAVIRT_TYPES_H
3	3	#define _ASM_X86_PARAVIRT_TYPES_H
4	4
	5	+#ifndef __ASSEMBLY__
	6	+/* These all sit in the .parainstructions section to tell us what to patch. */
	7	+struct paravirt_patch_site {
	8	+ u8 instr; / original instructions */
	9	+ u8 type; /* type of this instruction */
	10	+ u8 len; /* length of original instruction */
	11	+};
	12	+
	13	+/* Lazy mode for batching updates / context switch */
	14	+enum paravirt_lazy_mode {
	15	+ PARAVIRT_LAZY_NONE,
	16	+ PARAVIRT_LAZY_MMU,
	17	+ PARAVIRT_LAZY_CPU,
	18	+};
	19	+#endif
	20	+
	21	+#ifdef CONFIG_PARAVIRT
	22	+
5	23	#ifndef __ASSEMBLY__
6	24
7	25	#include <asm/desc_defs.h>
@@ -534,13 +552,6 @@ int paravirt_disable_iospace(void); · 526 unmodified lines
534	552	__PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
535	553	PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
536	554
537		−/* Lazy mode for batching updates / context switch */
538		−enum paravirt_lazy_mode {
539		− PARAVIRT_LAZY_NONE,
540		− PARAVIRT_LAZY_MMU,
541		− PARAVIRT_LAZY_CPU,
542		−};
543		−
544	555	enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
545	556	void paravirt_start_context_switch(struct task_struct *prev);
546	557	void paravirt_end_context_switch(struct task_struct *next);
@@ -556,16 +567,9 @@ unsigned long paravirt_ret0(void); · 9 unmodified lines
556	567
557	568	#define paravirt_nop ((void *)_paravirt_nop)
558	569
559		−/* These all sit in the .parainstructions section to tell us what to patch. */
560		−struct paravirt_patch_site {
561		− u8 instr; / original instructions */
562		− u8 type; /* type of this instruction */
563		− u8 len; /* length of original instruction */
564		−};
565		−
566	570	extern struct paravirt_patch_site __parainstructions[],
567	571	__parainstructions_end[];
568	572
569	573	#endif /* __ASSEMBLY__ */
570		−
	574	+#endif /* CONFIG_PARAVIRT */
571	575	#endif /* _ASM_X86_PARAVIRT_TYPES_H */

arch/x86/include/asm/preempt.h+14−13

@@ -4,11 +4,11 @@ · 3 unmodified lines
4	4
5	5	#include <asm/rmwcc.h>
6	6	#include <asm/percpu.h>
	7	+#include <asm/current.h>
	8	+
7	9	#include <linux/thread_info.h>
8	10	#include <linux/static_call_types.h>
9	11
10		−DECLARE_PER_CPU(int, __preempt_count);
11		−
12	12	/* We use the MSB mostly because its available */
13	13	#define PREEMPT_NEED_RESCHED 0x80000000
14	14
@@ -24,7 +24,7 @@ DECLARE_PER_CPU(int, __preempt_count); · 9 unmodified lines
24	24	*/
25	25	static __always_inline int preempt_count(void)
26	26	{
27		− return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
	27	+ return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;
28	28	}
29	29
30	30	static __always_inline void preempt_count_set(int pc)
@@ -32,10 +32,10 @@ static __always_inline void preempt_count_set(int pc) · 1 unmodified line
32	32	int old, new;
33	33
34	34	do {
35		− old = raw_cpu_read_4(__preempt_count);
	35	+ old = raw_cpu_read_4(pcpu_hot.preempt_count);
36	36	new = (old & PREEMPT_NEED_RESCHED) \|
37	37	(pc & ~PREEMPT_NEED_RESCHED);
38		− } while (raw_cpu_cmpxchg_4(__preempt_count, old, new) != old);
	38	+ } while (raw_cpu_cmpxchg_4(pcpu_hot.preempt_count, old, new) != old);
39	39	}
40	40
41	41	/*
@@ -44,7 +44,7 @@ static __always_inline void preempt_count_set(int pc) · 2 unmodified lines
44	44	#define init_task_preempt_count(p) do { } while (0)
45	45
46	46	#define init_idle_preempt_count(p, cpu) do { \
47		− per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
	47	+ per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \
48	48	} while (0)
49	49
50	50	/*
@@ -58,17 +58,17 @@ static __always_inline void preempt_count_set(int pc) · 7 unmodified lines
58	58
59	59	static __always_inline void set_preempt_need_resched(void)
60	60	{
61		− raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
	61	+ raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);
62	62	}
63	63
64	64	static __always_inline void clear_preempt_need_resched(void)
65	65	{
66		− raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
	66	+ raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);
67	67	}
68	68
69	69	static __always_inline bool test_preempt_need_resched(void)
70	70	{
71		− return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
	71	+ return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);
72	72	}
73	73
74	74	/*
@@ -77,12 +77,12 @@ static __always_inline bool test_preempt_need_resched(void) · 2 unmodified lines
77	77
78	78	static __always_inline void __preempt_count_add(int val)
79	79	{
80		− raw_cpu_add_4(__preempt_count, val);
	80	+ raw_cpu_add_4(pcpu_hot.preempt_count, val);
81	81	}
82	82
83	83	static __always_inline void __preempt_count_sub(int val)
84	84	{
85		− raw_cpu_add_4(__preempt_count, -val);
	85	+ raw_cpu_add_4(pcpu_hot.preempt_count, -val);
86	86	}
87	87
88	88	/*
@@ -92,7 +92,8 @@ static __always_inline void __preempt_count_sub(int val) · 3 unmodified lines
92	92	*/
93	93	static __always_inline bool __preempt_count_dec_and_test(void)
94	94	{
95		− return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
	95	+ return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e,
	96	+ __percpu_arg([var]));
96	97	}
97	98
98	99	/*
@@ -100,7 +101,7 @@ static __always_inline bool __preempt_count_dec_and_test(void) · 1 unmodified line
100	101	*/
101	102	static __always_inline bool should_resched(int preempt_offset)
102	103	{
103		− return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
	104	+ return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);
104	105	}
105	106
106	107	#ifdef CONFIG_PREEMPTION

arch/x86/include/asm/processor.h+2−9

@@ -377,8 +377,6 @@ struct irq_stack { · 376 unmodified lines
377	377	char stack[IRQ_STACK_SIZE];
378	378	} __aligned(IRQ_STACK_SIZE);
379	379
380		−DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
381		−
382	380	#ifdef CONFIG_X86_64
383	381	struct fixed_percpu_data {
384	382	/*
@@ -401,8 +399,6 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) · 16 unmodified lines
401	399	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
402	400	}
403	401
404		−DECLARE_PER_CPU(void *, hardirq_stack_ptr);
405		−DECLARE_PER_CPU(bool, hardirq_stack_inuse);
406	402	extern asmlinkage void ignore_sysret(void);
407	403
408	404	/* Save actual FS/GS selectors and bases to current->thread */
@@ -411,8 +407,6 @@ void current_save_fsgs(void); · 2 unmodified lines
411	407	#ifdef CONFIG_STACKPROTECTOR
412	408	DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
413	409	#endif
414		−DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
415		−DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
416	410	#endif /* !X86_64 */
417	411
418	412	struct perf_event;
@@ -517,7 +511,7 @@ static __always_inline unsigned long current_top_of_stack(void) · 98 unmodified lines
517	511	* and around vm86 mode and sp0 on x86_64 is special because of the
518	512	* entry trampoline.
519	513	*/
520		− return this_cpu_read_stable(cpu_current_top_of_stack);
	514	+ return this_cpu_read_stable(pcpu_hot.top_of_stack);
521	515	}
522	516
523	517	static __always_inline bool on_thread_stack(void)
@@ -554,10 +548,9 @@ extern int sysenter_setup(void); · 30 unmodified lines
554	548	/* Defined in head.S */
555	549	extern struct desc_ptr early_gdt_descr;
556	550
557		−extern void switch_to_new_gdt(int);
	551	+extern void switch_gdt_and_percpu_base(int);
558	552	extern void load_direct_gdt(int);
559	553	extern void load_fixmap_gdt(int);
560		−extern void load_percpu_segment(int);
561	554	extern void cpu_init(void);
562	555	extern void cpu_init_secondary(void);
563	556	extern void cpu_init_exception_handling(void);

arch/x86/include/asm/qspinlock_paravirt.h+20−27

@@ -14,8 +14,6 @@ · 13 unmodified lines
14	14
15	15	__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
16	16	#define __pv_queued_spin_unlock __pv_queued_spin_unlock
17		−#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
18		−#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
19	17
20	18	/*
21	19	* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
@@ -37,32 +35,27 @@ __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text"); · 15 unmodified lines
37	35	* rsi = lockval (second argument)
38	36	* rdx = internal variable (set to 0)
39	37	*/
40		−asm (".pushsection .spinlock.text, \"ax\";"
41		− ".globl " PV_UNLOCK ";"
42		− ".type " PV_UNLOCK ", @function;"
43		− ".align 4,0x90;"
44		− PV_UNLOCK ": "
45		− ASM_ENDBR
46		− FRAME_BEGIN
47		− "push %rdx;"
48		− "mov $0x1,%eax;"
49		− "xor %edx,%edx;"
50		− LOCK_PREFIX "cmpxchg %dl,(%rdi);"
51		− "cmp $0x1,%al;"
52		− "jne .slowpath;"
53		− "pop %rdx;"
	38	+#define PV_UNLOCK_ASM \
	39	+ FRAME_BEGIN \
	40	+ "push %rdx\n\t" \
	41	+ "mov $0x1,%eax\n\t" \
	42	+ "xor %edx,%edx\n\t" \
	43	+ LOCK_PREFIX "cmpxchg %dl,(%rdi)\n\t" \
	44	+ "cmp $0x1,%al\n\t" \
	45	+ "jne .slowpath\n\t" \
	46	+ "pop %rdx\n\t" \
	47	+ FRAME_END \
	48	+ ASM_RET \
	49	+ ".slowpath:\n\t" \
	50	+ "push %rsi\n\t" \
	51	+ "movzbl %al,%esi\n\t" \
	52	+ "call __raw_callee_save___pv_queued_spin_unlock_slowpath\n\t" \
	53	+ "pop %rsi\n\t" \
	54	+ "pop %rdx\n\t" \
54	55	FRAME_END
55		− ASM_RET
56		− ".slowpath: "
57		− "push %rsi;"
58		− "movzbl %al,%esi;"
59		− "call " PV_UNLOCK_SLOWPATH ";"
60		− "pop %rsi;"
61		− "pop %rdx;"
62		− FRAME_END
63		− ASM_RET
64		− ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
65		− ".popsection");
	56	+
	57	+DEFINE_PARAVIRT_ASM(__raw_callee_save___pv_queued_spin_unlock,
	58	+ PV_UNLOCK_ASM, .spinlock.text);
66	59
67	60	#else /* CONFIG_64BIT */
68	61

arch/x86/include/asm/smp.h+5−7

@@ -3,10 +3,10 @@ · 2 unmodified lines
3	3	#define _ASM_X86_SMP_H
4	4	#ifndef __ASSEMBLY__
5	5	#include <linux/cpumask.h>
6		−#include <asm/percpu.h>
7	6
8		−#include <asm/thread_info.h>
9	7	#include <asm/cpumask.h>
	8	+#include <asm/current.h>
	9	+#include <asm/thread_info.h>
10	10
11	11	extern int smp_num_siblings;
12	12	extern unsigned int num_processors;
@@ -19,7 +19,6 @@ DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); · 6 unmodified lines
19	19	DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_l2c_shared_map);
20	20	DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
21	21	DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id);
22		−DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
23	22
24	23	DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
25	24	DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid);
@@ -150,11 +149,10 @@ __visible void smp_call_function_single_interrupt(struct pt_regs *r); · 124 unmodified lines
150	149
151	150	/*
152	151	* This function is needed by all SMP systems. It must _always_ be valid
153		− * from the initial startup. We map APIC_BASE very early in page_setup(),
154		− * so this is correct in the x86 case.
	152	+ * from the initial startup.
155	153	*/
156		−#define raw_smp_processor_id() this_cpu_read(cpu_number)
157		−#define __smp_processor_id() __this_cpu_read(cpu_number)
	154	+#define raw_smp_processor_id() this_cpu_read(pcpu_hot.cpu_number)
	155	+#define __smp_processor_id() __this_cpu_read(pcpu_hot.cpu_number)
158	156
159	157	#ifdef CONFIG_X86_32
160	158	extern int safe_smp_processor_id(void);

arch/x86/include/asm/text-patching.h+1−0

@@ -45,6 +45,7 @@ extern void text_poke(void addr, const void *opcode, size_t len); · 44 unmodified lines
45	45	extern void text_poke_sync(void);
46	46	extern void text_poke_kgdb(void addr, const void *opcode, size_t len);
47	47	extern void text_poke_copy(void addr, const void *opcode, size_t len);
	48	+extern void text_poke_copy_locked(void addr, const void *opcode, size_t len, bool core_ok);
48	49	extern void text_poke_set(void addr, int c, size_t len);
49	50	extern int poke_int3_handler(struct pt_regs *regs);
50	51	extern void text_poke_bp(void addr, const void opcode, size_t len, const void *emulate);

arch/x86/kernel/Makefile+2−0

@@ -143,6 +143,8 @@ obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o · 142 unmodified lines
143	143
144	144	obj-$(CONFIG_CFI_CLANG) += cfi.o
145	145
	146	+obj-$(CONFIG_CALL_THUNKS) += callthunks.o
	147	+
146	148	###
147	149	# 64 bit specific files
148	150	ifeq ($(CONFIG_X86_64),y)

arch/x86/kernel/alternative.c+495−34

@@ -116,6 +116,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len) · 115 unmodified lines
116	116
117	117	extern s32 __retpoline_sites[], __retpoline_sites_end[];
118	118	extern s32 __return_sites[], __return_sites_end[];
	119	+extern s32 __cfi_sites[], __cfi_sites_end[];
119	120	extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
120	121	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
121	122	extern s32 __smp_locks[], __smp_locks_end[];
@@ -377,6 +378,56 @@ static int emit_indirect(int op, int reg, u8 *bytes) · 255 unmodified lines
377	378	return i;
378	379	}
379	380
	381	+static inline bool is_jcc32(struct insn *insn)
	382	+{
	383	+ /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */
	384	+ return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;
	385	+}
	386	+
	387	+static int emit_call_track_retpoline(void addr, struct insn insn, int reg, u8 *bytes)
	388	+{
	389	+ u8 op = insn->opcode.bytes[0];
	390	+ int i = 0;
	391	+
	392	+ /*
	393	+ * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
	394	+ * tail-calls. Deal with them.
	395	+ */
	396	+ if (is_jcc32(insn)) {
	397	+ bytes[i++] = op;
	398	+ op = insn->opcode.bytes[1];
	399	+ goto clang_jcc;
	400	+ }
	401	+
	402	+ if (insn->length == 6)
	403	+ bytes[i++] = 0x2e; /* CS-prefix */
	404	+
	405	+ switch (op) {
	406	+ case CALL_INSN_OPCODE:
	407	+ __text_gen_insn(bytes+i, op, addr+i,
	408	+ __x86_indirect_call_thunk_array[reg],
	409	+ CALL_INSN_SIZE);
	410	+ i += CALL_INSN_SIZE;
	411	+ break;
	412	+
	413	+ case JMP32_INSN_OPCODE:
	414	+clang_jcc:
	415	+ __text_gen_insn(bytes+i, op, addr+i,
	416	+ __x86_indirect_jump_thunk_array[reg],
	417	+ JMP32_INSN_SIZE);
	418	+ i += JMP32_INSN_SIZE;
	419	+ break;
	420	+
	421	+ default:
	422	+ WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
	423	+ return -1;
	424	+ }
	425	+
	426	+ WARN_ON_ONCE(i != insn->length);
	427	+
	428	+ return i;
	429	+}
	430	+
380	431	/*
381	432	* Rewrite the compiler generated retpoline thunk calls.
382	433	*
@@ -409,8 +460,12 @@ static int patch_retpoline(void addr, struct insn insn, u8 *bytes) · 26 unmodified lines
409	460	BUG_ON(reg == 4);
410	461
411	462	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
412		− !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE))
	463	+ !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
	464	+ if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
	465	+ return emit_call_track_retpoline(addr, insn, reg, bytes);
	466	+
413	467	return -1;
	468	+ }
414	469
415	470	op = insn->opcode.bytes[0];
416	471
@@ -427,8 +482,7 @@ static int patch_retpoline(void addr, struct insn insn, u8 *bytes) · 10 unmodified lines
427	482	* [ NOP ]
428	483	* 1:
429	484	*/
430		− /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */
431		− if (op == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80) {
	485	+ if (is_jcc32(insn)) {
432	486	cc = insn->opcode.bytes[1] & 0xf;
433	487	cc ^= 1; /* invert condition */
434	488
@@ -518,6 +572,11 @@ void __init_or_module noinline apply_retpolines(s32 start, s32 end) · 83 unmodified lines
518	572	}
519	573
520	574	#ifdef CONFIG_RETHUNK
	575	+
	576	+#ifdef CONFIG_CALL_THUNKS
	577	+void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
	578	+#endif
	579	+
521	580	/*
522	581	* Rewrite the compiler generated return thunk tail-calls.
523	582	*
@@ -533,14 +592,18 @@ static int patch_return(void addr, struct insn insn, u8 *bytes) · 9 unmodified lines
533	592	{
534	593	int i = 0;
535	594
536		− if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
537		− return -1;
	595	+ if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
	596	+ if (x86_return_thunk == __x86_return_thunk)
	597	+ return -1;
538	598
539		− bytes[i++] = RET_INSN_OPCODE;
	599	+ i = JMP32_INSN_SIZE;
	600	+ __text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i);
	601	+ } else {
	602	+ bytes[i++] = RET_INSN_OPCODE;
	603	+ }
540	604
541	605	for (; i < insn->length;)
542	606	bytes[i++] = INT3_INSN_OPCODE;
543		−
544	607	return i;
545	608	}
546	609
@@ -594,6 +657,28 @@ void __init_or_module noinline apply_returns(s32 start, s32 end) { } · 47 unmodified lines
594	657
595	658	#ifdef CONFIG_X86_KERNEL_IBT
596	659
	660	+static void poison_endbr(void *addr, bool warn)
	661	+{
	662	+ u32 endbr, poison = gen_endbr_poison();
	663	+
	664	+ if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
	665	+ return;
	666	+
	667	+ if (!is_endbr(endbr)) {
	668	+ WARN_ON_ONCE(warn);
	669	+ return;
	670	+ }
	671	+
	672	+ DPRINTK("ENDBR at: %pS (%px)", addr, addr);
	673	+
	674	+ /*
	675	+ * When we have IBT, the lack of ENDBR will trigger #CP
	676	+ */
	677	+ DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
	678	+ DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
	679	+ text_poke_early(addr, &poison, 4);
	680	+}
	681	+
597	682	/*
598	683	* Generated by: objtool --ibt
599	684	*/
@@ -602,31 +687,391 @@ void __init_or_module noinline apply_ibt_endbr(s32 start, s32 end) · 2 unmodified lines
602	687	s32 *s;
603	688
604	689	for (s = start; s < end; s++) {
605		− u32 endbr, poison = gen_endbr_poison();
606	690	void addr = (void )s + *s;
607	691
608		− if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
	692	+ poison_endbr(addr, true);
	693	+ if (IS_ENABLED(CONFIG_FINEIBT))
	694	+ poison_endbr(addr - 16, false);
	695	+ }
	696	+}
	697	+
	698	+#else
	699	+
	700	+void __init_or_module apply_ibt_endbr(s32 start, s32 end) { }
	701	+
	702	+#endif /* CONFIG_X86_KERNEL_IBT */
	703	+
	704	+#ifdef CONFIG_FINEIBT
	705	+
	706	+enum cfi_mode {
	707	+ CFI_DEFAULT,
	708	+ CFI_OFF,
	709	+ CFI_KCFI,
	710	+ CFI_FINEIBT,
	711	+};
	712	+
	713	+static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT;
	714	+static bool cfi_rand __ro_after_init = true;
	715	+static u32 cfi_seed __ro_after_init;
	716	+
	717	+/*
	718	+ * Re-hash the CFI hash with a boot-time seed while making sure the result is
	719	+ * not a valid ENDBR instruction.
	720	+ */
	721	+static u32 cfi_rehash(u32 hash)
	722	+{
	723	+ hash ^= cfi_seed;
	724	+ while (unlikely(is_endbr(hash) \|\| is_endbr(-hash))) {
	725	+ bool lsb = hash & 1;
	726	+ hash >>= 1;
	727	+ if (lsb)
	728	+ hash ^= 0x80200003;
	729	+ }
	730	+ return hash;
	731	+}
	732	+
	733	+static __init int cfi_parse_cmdline(char *str)
	734	+{
	735	+ if (!str)
	736	+ return -EINVAL;
	737	+
	738	+ while (str) {
	739	+ char *next = strchr(str, ',');
	740	+ if (next) {
	741	+ *next = 0;
	742	+ next++;
	743	+ }
	744	+
	745	+ if (!strcmp(str, "auto")) {
	746	+ cfi_mode = CFI_DEFAULT;
	747	+ } else if (!strcmp(str, "off")) {
	748	+ cfi_mode = CFI_OFF;
	749	+ cfi_rand = false;
	750	+ } else if (!strcmp(str, "kcfi")) {
	751	+ cfi_mode = CFI_KCFI;
	752	+ } else if (!strcmp(str, "fineibt")) {
	753	+ cfi_mode = CFI_FINEIBT;
	754	+ } else if (!strcmp(str, "norand")) {
	755	+ cfi_rand = false;
	756	+ } else {
	757	+ pr_err("Ignoring unknown cfi option (%s).", str);
	758	+ }
	759	+
	760	+ str = next;
	761	+ }
	762	+
	763	+ return 0;
	764	+}
	765	+early_param("cfi", cfi_parse_cmdline);
	766	+
	767	+/*
	768	+ * kCFI FineIBT
	769	+ *
	770	+ * __cfi_\func: __cfi_\func:
	771	+ * movl $0x12345678,%eax // 5 endbr64 // 4
	772	+ * nop subl $0x12345678,%r10d // 7
	773	+ * nop jz 1f // 2
	774	+ * nop ud2 // 2
	775	+ * nop 1: nop // 1
	776	+ * nop
	777	+ * nop
	778	+ * nop
	779	+ * nop
	780	+ * nop
	781	+ * nop
	782	+ * nop
	783	+ *
	784	+ *
	785	+ * caller: caller:
	786	+ * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
	787	+ * addl $-15(%r11),%r10d // 4 sub $16,%r11 // 4
	788	+ * je 1f // 2 nop4 // 4
	789	+ * ud2 // 2
	790	+ * 1: call __x86_indirect_thunk_r11 // 5 call *%r11; nop2; // 5
	791	+ *
	792	+ */
	793	+
	794	+asm( ".pushsection .rodata \n"
	795	+ "fineibt_preamble_start: \n"
	796	+ " endbr64 \n"
	797	+ " subl $0x12345678, %r10d \n"
	798	+ " je fineibt_preamble_end \n"
	799	+ " ud2 \n"
	800	+ " nop \n"
	801	+ "fineibt_preamble_end: \n"
	802	+ ".popsection\n"
	803	+);
	804	+
	805	+extern u8 fineibt_preamble_start[];
	806	+extern u8 fineibt_preamble_end[];
	807	+
	808	+#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
	809	+#define fineibt_preamble_hash 7
	810	+
	811	+asm( ".pushsection .rodata \n"
	812	+ "fineibt_caller_start: \n"
	813	+ " movl $0x12345678, %r10d \n"
	814	+ " sub $16, %r11 \n"
	815	+ ASM_NOP4
	816	+ "fineibt_caller_end: \n"
	817	+ ".popsection \n"
	818	+);
	819	+
	820	+extern u8 fineibt_caller_start[];
	821	+extern u8 fineibt_caller_end[];
	822	+
	823	+#define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
	824	+#define fineibt_caller_hash 2
	825	+
	826	+#define fineibt_caller_jmp (fineibt_caller_size - 2)
	827	+
	828	+static u32 decode_preamble_hash(void *addr)
	829	+{
	830	+ u8 *p = addr;
	831	+
	832	+ /* b8 78 56 34 12 mov $0x12345678,%eax */
	833	+ if (p[0] == 0xb8)
	834	+ return (u32 )(addr + 1);
	835	+
	836	+ return 0; /* invalid hash value */
	837	+}
	838	+
	839	+static u32 decode_caller_hash(void *addr)
	840	+{
	841	+ u8 *p = addr;
	842	+
	843	+ /* 41 ba 78 56 34 12 mov $0x12345678,%r10d */
	844	+ if (p[0] == 0x41 && p[1] == 0xba)
	845	+ return -(u32 )(addr + 2);
	846	+
	847	+ /* e8 0c 78 56 34 12 jmp.d8 +12 */
	848	+ if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
	849	+ return -(u32 )(addr + 2);
	850	+
	851	+ return 0; /* invalid hash value */
	852	+}
	853	+
	854	+/* .retpoline_sites */
	855	+static int cfi_disable_callers(s32 start, s32 end)
	856	+{
	857	+ /*
	858	+ * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
	859	+ * in tact for later usage. Also see decode_caller_hash() and
	860	+ * cfi_rewrite_callers().
	861	+ */
	862	+ const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp };
	863	+ s32 *s;
	864	+
	865	+ for (s = start; s < end; s++) {
	866	+ void addr = (void )s + *s;
	867	+ u32 hash;
	868	+
	869	+ addr -= fineibt_caller_size;
	870	+ hash = decode_caller_hash(addr);
	871	+ if (!hash) /* nocfi callers */
609	872	continue;
610	873
611		− if (WARN_ON_ONCE(!is_endbr(endbr)))
	874	+ text_poke_early(addr, jmp, 2);
	875	+ }
	876	+
	877	+ return 0;
	878	+}
	879	+
	880	+static int cfi_enable_callers(s32 start, s32 end)
	881	+{
	882	+ /*
	883	+ * Re-enable kCFI, undo what cfi_disable_callers() did.
	884	+ */
	885	+ const u8 mov[] = { 0x41, 0xba };
	886	+ s32 *s;
	887	+
	888	+ for (s = start; s < end; s++) {
	889	+ void addr = (void )s + *s;
	890	+ u32 hash;
	891	+
	892	+ addr -= fineibt_caller_size;
	893	+ hash = decode_caller_hash(addr);
	894	+ if (!hash) /* nocfi callers */
612	895	continue;
613	896
614		− DPRINTK("ENDBR at: %pS (%px)", addr, addr);
	897	+ text_poke_early(addr, mov, 2);
	898	+ }
615	899
616		− /*
617		− * When we have IBT, the lack of ENDBR will trigger #CP
618		− */
619		− DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
620		− DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
621		− text_poke_early(addr, &poison, 4);
	900	+ return 0;
	901	+}
	902	+
	903	+/* .cfi_sites */
	904	+static int cfi_rand_preamble(s32 start, s32 end)
	905	+{
	906	+ s32 *s;
	907	+
	908	+ for (s = start; s < end; s++) {
	909	+ void addr = (void )s + *s;
	910	+ u32 hash;
	911	+
	912	+ hash = decode_preamble_hash(addr);
	913	+ if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
	914	+ addr, addr, 5, addr))
	915	+ return -EINVAL;
	916	+
	917	+ hash = cfi_rehash(hash);
	918	+ text_poke_early(addr + 1, &hash, 4);
	919	+ }
	920	+
	921	+ return 0;
	922	+}
	923	+
	924	+static int cfi_rewrite_preamble(s32 start, s32 end)
	925	+{
	926	+ s32 *s;
	927	+
	928	+ for (s = start; s < end; s++) {
	929	+ void addr = (void )s + *s;
	930	+ u32 hash;
	931	+
	932	+ hash = decode_preamble_hash(addr);
	933	+ if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
	934	+ addr, addr, 5, addr))
	935	+ return -EINVAL;
	936	+
	937	+ text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
	938	+ WARN_ON((u32 )(addr + fineibt_preamble_hash) != 0x12345678);
	939	+ text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
	940	+ }
	941	+
	942	+ return 0;
	943	+}
	944	+
	945	+/* .retpoline_sites */
	946	+static int cfi_rand_callers(s32 start, s32 end)
	947	+{
	948	+ s32 *s;
	949	+
	950	+ for (s = start; s < end; s++) {
	951	+ void addr = (void )s + *s;
	952	+ u32 hash;
	953	+
	954	+ addr -= fineibt_caller_size;
	955	+ hash = decode_caller_hash(addr);
	956	+ if (hash) {
	957	+ hash = -cfi_rehash(hash);
	958	+ text_poke_early(addr + 2, &hash, 4);
	959	+ }
	960	+ }
	961	+
	962	+ return 0;
	963	+}
	964	+
	965	+static int cfi_rewrite_callers(s32 start, s32 end)
	966	+{
	967	+ s32 *s;
	968	+
	969	+ for (s = start; s < end; s++) {
	970	+ void addr = (void )s + *s;
	971	+ u32 hash;
	972	+
	973	+ addr -= fineibt_caller_size;
	974	+ hash = decode_caller_hash(addr);
	975	+ if (hash) {
	976	+ text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
	977	+ WARN_ON((u32 )(addr + fineibt_caller_hash) != 0x12345678);
	978	+ text_poke_early(addr + fineibt_caller_hash, &hash, 4);
	979	+ }
	980	+ /* rely on apply_retpolines() */
	981	+ }
	982	+
	983	+ return 0;
	984	+}
	985	+
	986	+static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
	987	+ s32 start_cfi, s32 end_cfi, bool builtin)
	988	+{
	989	+ int ret;
	990	+
	991	+ if (WARN_ONCE(fineibt_preamble_size != 16,
	992	+ "FineIBT preamble wrong size: %ld", fineibt_preamble_size))
	993	+ return;
	994	+
	995	+ if (cfi_mode == CFI_DEFAULT) {
	996	+ cfi_mode = CFI_KCFI;
	997	+ if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
	998	+ cfi_mode = CFI_FINEIBT;
	999	+ }
	1000	+
	1001	+ /*
	1002	+ * Rewrite the callers to not use the __cfi_ stubs, such that we might
	1003	+ * rewrite them. This disables all CFI. If this succeeds but any of the
	1004	+ * later stages fails, we're without CFI.
	1005	+ */
	1006	+ ret = cfi_disable_callers(start_retpoline, end_retpoline);
	1007	+ if (ret)
	1008	+ goto err;
	1009	+
	1010	+ if (cfi_rand) {
	1011	+ if (builtin)
	1012	+ cfi_seed = get_random_u32();
	1013	+
	1014	+ ret = cfi_rand_preamble(start_cfi, end_cfi);
	1015	+ if (ret)
	1016	+ goto err;
	1017	+
	1018	+ ret = cfi_rand_callers(start_retpoline, end_retpoline);
	1019	+ if (ret)
	1020	+ goto err;
	1021	+ }
	1022	+
	1023	+ switch (cfi_mode) {
	1024	+ case CFI_OFF:
	1025	+ if (builtin)
	1026	+ pr_info("Disabling CFI\n");
	1027	+ return;
	1028	+
	1029	+ case CFI_KCFI:
	1030	+ ret = cfi_enable_callers(start_retpoline, end_retpoline);
	1031	+ if (ret)
	1032	+ goto err;
	1033	+
	1034	+ if (builtin)
	1035	+ pr_info("Using kCFI\n");
	1036	+ return;
	1037	+
	1038	+ case CFI_FINEIBT:
	1039	+ ret = cfi_rewrite_preamble(start_cfi, end_cfi);
	1040	+ if (ret)
	1041	+ goto err;
	1042	+
	1043	+ ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
	1044	+ if (ret)
	1045	+ goto err;
	1046	+
	1047	+ if (builtin)
	1048	+ pr_info("Using FineIBT CFI\n");
	1049	+ return;
	1050	+
	1051	+ default:
	1052	+ break;
622	1053	}
	1054	+
	1055	+err:
	1056	+ pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n");
623	1057	}
624	1058
625	1059	#else
626	1060
627		−void __init_or_module apply_ibt_endbr(s32 start, s32 end) { }
	1061	+static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
	1062	+ s32 start_cfi, s32 end_cfi, bool builtin)
	1063	+{
	1064	+}
628	1065
629		−#endif /* CONFIG_X86_KERNEL_IBT */
	1066	+#endif
	1067	+
	1068	+void apply_fineibt(s32 start_retpoline, s32 end_retpoline,
	1069	+ s32 start_cfi, s32 end_cfi)
	1070	+{
	1071	+ return __apply_fineibt(start_retpoline, end_retpoline,
	1072	+ start_cfi, end_cfi,
	1073	+ /* .builtin = */ false);
	1074	+}
630	1075
631	1076	#ifdef CONFIG_SMP
632	1077	static void alternatives_smp_lock(const s32 start, const s32 end,
@@ -934,6 +1379,9 @@ void __init alternative_instructions(void) · 301 unmodified lines
934	1379	*/
935	1380	apply_paravirt(__parainstructions, __parainstructions_end);
936	1381
	1382	+ __apply_fineibt(__retpoline_sites, __retpoline_sites_end,
	1383	+ __cfi_sites, __cfi_sites_end, true);
	1384	+
937	1385	/*
938	1386	* Rewrite the retpolines, must be done before alternatives since
939	1387	* those can rewrite the retpoline thunks.
@@ -947,6 +1395,12 @@ void __init alternative_instructions(void) · 7 unmodified lines
947	1395	*/
948	1396	apply_alternatives(__alt_instructions, __alt_instructions_end);
949	1397
	1398	+ /*
	1399	+ * Now all calls are established. Apply the call thunks if
	1400	+ * required.
	1401	+ */
	1402	+ callthunks_patch_builtin_calls();
	1403	+
950	1404	apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
951	1405
952	1406	#ifdef CONFIG_SMP
@@ -1236,27 +1690,15 @@ void text_poke_kgdb(void addr, const void *opcode, size_t len) · 283 unmodified lines
1236	1690	return __text_poke(text_poke_memcpy, addr, opcode, len);
1237	1691	}
1238	1692
1239		−/**
1240		− * text_poke_copy - Copy instructions into (an unused part of) RX memory
1241		− * @addr: address to modify
1242		− * @opcode: source of the copy
1243		− * @len: length to copy, could be more than 2x PAGE_SIZE
1244		− *
1245		− * Not safe against concurrent execution; useful for JITs to dump
1246		− * new code blocks into unused regions of RX memory. Can be used in
1247		− * conjunction with synchronize_rcu_tasks() to wait for existing
1248		− * execution to quiesce after having made sure no existing functions
1249		− * pointers are live.
1250		− */
1251		−void text_poke_copy(void addr, const void *opcode, size_t len)
	1693	+void text_poke_copy_locked(void addr, const void *opcode, size_t len,
	1694	+ bool core_ok)
1252	1695	{
1253	1696	unsigned long start = (unsigned long)addr;
1254	1697	size_t patched = 0;
1255	1698
1256		− if (WARN_ON_ONCE(core_kernel_text(start)))
	1699	+ if (WARN_ON_ONCE(!core_ok && core_kernel_text(start)))
1257	1700	return NULL;
1258	1701
1259		− mutex_lock(&text_mutex);
1260	1702	while (patched < len) {
1261	1703	unsigned long ptr = start + patched;
1262	1704	size_t s;
@@ -1266,6 +1708,25 @@ void text_poke_copy(void addr, const void *opcode, size_t len) · 3 unmodified lines
1266	1708	__text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s);
1267	1709	patched += s;
1268	1710	}
	1711	+ return addr;
	1712	+}
	1713	+
	1714	+/**
	1715	+ * text_poke_copy - Copy instructions into (an unused part of) RX memory
	1716	+ * @addr: address to modify
	1717	+ * @opcode: source of the copy
	1718	+ * @len: length to copy, could be more than 2x PAGE_SIZE
	1719	+ *
	1720	+ * Not safe against concurrent execution; useful for JITs to dump
	1721	+ * new code blocks into unused regions of RX memory. Can be used in
	1722	+ * conjunction with synchronize_rcu_tasks() to wait for existing
	1723	+ * execution to quiesce after having made sure no existing functions
	1724	+ * pointers are live.
	1725	+ */
	1726	+void text_poke_copy(void addr, const void *opcode, size_t len)
	1727	+{
	1728	+ mutex_lock(&text_mutex);
	1729	+ addr = text_poke_copy_locked(addr, opcode, len, false);
1269	1730	mutex_unlock(&text_mutex);
1270	1731	return addr;
1271	1732	}

arch/x86/kernel/asm-offsets.c+5−0

@@ -107,4 +107,9 @@ static void __used common(void) · 106 unmodified lines
107	107	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
108	108	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
109	109	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
	110	+ OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack);
	111	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	112	+ OFFSET(X86_call_depth, pcpu_hot, call_depth);
	113	+#endif
	114	+
110	115	}

arch/x86/kernel/asm-offsets_64.c+1−1

@@ -57,7 +57,7 @@ int main(void) · 56 unmodified lines
57	57	BLANK();
58	58
59	59	#ifdef CONFIG_STACKPROTECTOR
60		− DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary));
	60	+ OFFSET(FIXED_stack_canary, fixed_percpu_data, stack_canary);
61	61	BLANK();
62	62	#endif
63	63	return 0;

arch/x86/kernel/callthunks.c+388−0

@@ -0,0 +1,388 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+#define pr_fmt(fmt) "callthunks: " fmt
	4	+
	5	+#include <linux/debugfs.h>
	6	+#include <linux/kallsyms.h>
	7	+#include <linux/memory.h>
	8	+#include <linux/moduleloader.h>
	9	+#include <linux/static_call.h>
	10	+
	11	+#include <asm/alternative.h>
	12	+#include <asm/asm-offsets.h>
	13	+#include <asm/cpu.h>
	14	+#include <asm/ftrace.h>
	15	+#include <asm/insn.h>
	16	+#include <asm/kexec.h>
	17	+#include <asm/nospec-branch.h>
	18	+#include <asm/paravirt.h>
	19	+#include <asm/sections.h>
	20	+#include <asm/switch_to.h>
	21	+#include <asm/sync_core.h>
	22	+#include <asm/text-patching.h>
	23	+#include <asm/xen/hypercall.h>
	24	+
	25	+static int __initdata_or_module debug_callthunks;
	26	+
	27	+#define prdbg(fmt, args...) \
	28	+do { \
	29	+ if (debug_callthunks) \
	30	+ printk(KERN_DEBUG pr_fmt(fmt), ##args); \
	31	+} while(0)
	32	+
	33	+static int __init debug_thunks(char *str)
	34	+{
	35	+ debug_callthunks = 1;
	36	+ return 1;
	37	+}
	38	+__setup("debug-callthunks", debug_thunks);
	39	+
	40	+#ifdef CONFIG_CALL_THUNKS_DEBUG
	41	+DEFINE_PER_CPU(u64, __x86_call_count);
	42	+DEFINE_PER_CPU(u64, __x86_ret_count);
	43	+DEFINE_PER_CPU(u64, __x86_stuffs_count);
	44	+DEFINE_PER_CPU(u64, __x86_ctxsw_count);
	45	+EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
	46	+EXPORT_SYMBOL_GPL(__x86_call_count);
	47	+#endif
	48	+
	49	+extern s32 __call_sites[], __call_sites_end[];
	50	+
	51	+struct thunk_desc {
	52	+ void *template;
	53	+ unsigned int template_size;
	54	+};
	55	+
	56	+struct core_text {
	57	+ unsigned long base;
	58	+ unsigned long end;
	59	+ const char *name;
	60	+};
	61	+
	62	+static bool thunks_initialized __ro_after_init;
	63	+
	64	+static const struct core_text builtin_coretext = {
	65	+ .base = (unsigned long)_text,
	66	+ .end = (unsigned long)_etext,
	67	+ .name = "builtin",
	68	+};
	69	+
	70	+asm (
	71	+ ".pushsection .rodata \n"
	72	+ ".global skl_call_thunk_template \n"
	73	+ "skl_call_thunk_template: \n"
	74	+ __stringify(INCREMENT_CALL_DEPTH)" \n"
	75	+ ".global skl_call_thunk_tail \n"
	76	+ "skl_call_thunk_tail: \n"
	77	+ ".popsection \n"
	78	+);
	79	+
	80	+extern u8 skl_call_thunk_template[];
	81	+extern u8 skl_call_thunk_tail[];
	82	+
	83	+#define SKL_TMPL_SIZE \
	84	+ ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
	85	+
	86	+extern void error_entry(void);
	87	+extern void xen_error_entry(void);
	88	+extern void paranoid_entry(void);
	89	+
	90	+static inline bool within_coretext(const struct core_text ct, void addr)
	91	+{
	92	+ unsigned long p = (unsigned long)addr;
	93	+
	94	+ return ct->base <= p && p < ct->end;
	95	+}
	96	+
	97	+static inline bool within_module_coretext(void *addr)
	98	+{
	99	+ bool ret = false;
	100	+
	101	+#ifdef CONFIG_MODULES
	102	+ struct module *mod;
	103	+
	104	+ preempt_disable();
	105	+ mod = __module_address((unsigned long)addr);
	106	+ if (mod && within_module_core((unsigned long)addr, mod))
	107	+ ret = true;
	108	+ preempt_enable();
	109	+#endif
	110	+ return ret;
	111	+}
	112	+
	113	+static bool is_coretext(const struct core_text ct, void addr)
	114	+{
	115	+ if (ct && within_coretext(ct, addr))
	116	+ return true;
	117	+ if (within_coretext(&builtin_coretext, addr))
	118	+ return true;
	119	+ return within_module_coretext(addr);
	120	+}
	121	+
	122	+static __init_or_module bool skip_addr(void *dest)
	123	+{
	124	+ if (dest == error_entry)
	125	+ return true;
	126	+ if (dest == paranoid_entry)
	127	+ return true;
	128	+ if (dest == xen_error_entry)
	129	+ return true;
	130	+ /* Does FILL_RSB... */
	131	+ if (dest == __switch_to_asm)
	132	+ return true;
	133	+ /* Accounts directly */
	134	+ if (dest == ret_from_fork)
	135	+ return true;
	136	+#ifdef CONFIG_HOTPLUG_CPU
	137	+ if (dest == start_cpu0)
	138	+ return true;
	139	+#endif
	140	+#ifdef CONFIG_FUNCTION_TRACER
	141	+ if (dest == __fentry__)
	142	+ return true;
	143	+#endif
	144	+#ifdef CONFIG_KEXEC_CORE
	145	+ if (dest >= (void *)relocate_kernel &&
	146	+ dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
	147	+ return true;
	148	+#endif
	149	+#ifdef CONFIG_XEN
	150	+ if (dest >= (void *)hypercall_page &&
	151	+ dest < (void*)hypercall_page + PAGE_SIZE)
	152	+ return true;
	153	+#endif
	154	+ return false;
	155	+}
	156	+
	157	+static __init_or_module void call_get_dest(void addr)
	158	+{
	159	+ struct insn insn;
	160	+ void *dest;
	161	+ int ret;
	162	+
	163	+ ret = insn_decode_kernel(&insn, addr);
	164	+ if (ret)
	165	+ return ERR_PTR(ret);
	166	+
	167	+ /* Patched out call? */
	168	+ if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
	169	+ return NULL;
	170	+
	171	+ dest = addr + insn.length + insn.immediate.value;
	172	+ if (skip_addr(dest))
	173	+ return NULL;
	174	+ return dest;
	175	+}
	176	+
	177	+static const u8 nops[] = {
	178	+ 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	179	+ 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	180	+ 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	181	+ 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	182	+};
	183	+
	184	+static __init_or_module void patch_dest(void dest, bool direct)
	185	+{
	186	+ unsigned int tsize = SKL_TMPL_SIZE;
	187	+ u8 *pad = dest - tsize;
	188	+
	189	+ /* Already patched? */
	190	+ if (!bcmp(pad, skl_call_thunk_template, tsize))
	191	+ return pad;
	192	+
	193	+ /* Ensure there are nops */
	194	+ if (bcmp(pad, nops, tsize)) {
	195	+ pr_warn_once("Invalid padding area for %pS\n", dest);
	196	+ return NULL;
	197	+ }
	198	+
	199	+ if (direct)
	200	+ memcpy(pad, skl_call_thunk_template, tsize);
	201	+ else
	202	+ text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
	203	+ return pad;
	204	+}
	205	+
	206	+static __init_or_module void patch_call(void addr, const struct core_text ct)
	207	+{
	208	+ void pad, dest;
	209	+ u8 bytes[8];
	210	+
	211	+ if (!within_coretext(ct, addr))
	212	+ return;
	213	+
	214	+ dest = call_get_dest(addr);
	215	+ if (!dest \|\| WARN_ON_ONCE(IS_ERR(dest)))
	216	+ return;
	217	+
	218	+ if (!is_coretext(ct, dest))
	219	+ return;
	220	+
	221	+ pad = patch_dest(dest, within_coretext(ct, dest));
	222	+ if (!pad)
	223	+ return;
	224	+
	225	+ prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
	226	+ dest, dest, pad);
	227	+ __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
	228	+ text_poke_early(addr, bytes, CALL_INSN_SIZE);
	229	+}
	230	+
	231	+static __init_or_module void
	232	+patch_call_sites(s32 start, s32 end, const struct core_text *ct)
	233	+{
	234	+ s32 *s;
	235	+
	236	+ for (s = start; s < end; s++)
	237	+ patch_call((void )s + s, ct);
	238	+}
	239	+
	240	+static __init_or_module void
	241	+patch_paravirt_call_sites(struct paravirt_patch_site *start,
	242	+ struct paravirt_patch_site *end,
	243	+ const struct core_text *ct)
	244	+{
	245	+ struct paravirt_patch_site *p;
	246	+
	247	+ for (p = start; p < end; p++)
	248	+ patch_call(p->instr, ct);
	249	+}
	250	+
	251	+static __init_or_module void
	252	+callthunks_setup(struct callthunk_sites cs, const struct core_text ct)
	253	+{
	254	+ prdbg("Patching call sites %s\n", ct->name);
	255	+ patch_call_sites(cs->call_start, cs->call_end, ct);
	256	+ patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
	257	+ prdbg("Patching call sites done%s\n", ct->name);
	258	+}
	259	+
	260	+void __init callthunks_patch_builtin_calls(void)
	261	+{
	262	+ struct callthunk_sites cs = {
	263	+ .call_start = __call_sites,
	264	+ .call_end = __call_sites_end,
	265	+ .pv_start = __parainstructions,
	266	+ .pv_end = __parainstructions_end
	267	+ };
	268	+
	269	+ if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
	270	+ return;
	271	+
	272	+ pr_info("Setting up call depth tracking\n");
	273	+ mutex_lock(&text_mutex);
	274	+ callthunks_setup(&cs, &builtin_coretext);
	275	+ static_call_force_reinit();
	276	+ thunks_initialized = true;
	277	+ mutex_unlock(&text_mutex);
	278	+}
	279	+
	280	+void callthunks_translate_call_dest(void dest)
	281	+{
	282	+ void *target;
	283	+
	284	+ lockdep_assert_held(&text_mutex);
	285	+
	286	+ if (!thunks_initialized \|\| skip_addr(dest))
	287	+ return dest;
	288	+
	289	+ if (!is_coretext(NULL, dest))
	290	+ return dest;
	291	+
	292	+ target = patch_dest(dest, false);
	293	+ return target ? : dest;
	294	+}
	295	+
	296	+bool is_callthunk(void *addr)
	297	+{
	298	+ unsigned int tmpl_size = SKL_TMPL_SIZE;
	299	+ void *tmpl = skl_call_thunk_template;
	300	+ unsigned long dest;
	301	+
	302	+ dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
	303	+ if (!thunks_initialized \|\| skip_addr((void *)dest))
	304	+ return false;
	305	+
	306	+ return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
	307	+}
	308	+
	309	+#ifdef CONFIG_BPF_JIT
	310	+int x86_call_depth_emit_accounting(u8 *pprog, void func)
	311	+{
	312	+ unsigned int tmpl_size = SKL_TMPL_SIZE;
	313	+ void *tmpl = skl_call_thunk_template;
	314	+
	315	+ if (!thunks_initialized)
	316	+ return 0;
	317	+
	318	+ /* Is function call target a thunk? */
	319	+ if (func && is_callthunk(func))
	320	+ return 0;
	321	+
	322	+ memcpy(*pprog, tmpl, tmpl_size);
	323	+ *pprog += tmpl_size;
	324	+ return tmpl_size;
	325	+}
	326	+#endif
	327	+
	328	+#ifdef CONFIG_MODULES
	329	+void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
	330	+ struct module *mod)
	331	+{
	332	+ struct core_text ct = {
	333	+ .base = (unsigned long)mod->core_layout.base,
	334	+ .end = (unsigned long)mod->core_layout.base + mod->core_layout.size,
	335	+ .name = mod->name,
	336	+ };
	337	+
	338	+ if (!thunks_initialized)
	339	+ return;
	340	+
	341	+ mutex_lock(&text_mutex);
	342	+ callthunks_setup(cs, &ct);
	343	+ mutex_unlock(&text_mutex);
	344	+}
	345	+#endif /* CONFIG_MODULES */
	346	+
	347	+#if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
	348	+static int callthunks_debug_show(struct seq_file m, void p)
	349	+{
	350	+ unsigned long cpu = (unsigned long)m->private;
	351	+
	352	+ seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
	353	+ per_cpu(__x86_call_count, cpu),
	354	+ per_cpu(__x86_ret_count, cpu),
	355	+ per_cpu(__x86_stuffs_count, cpu),
	356	+ per_cpu(__x86_ctxsw_count, cpu));
	357	+ return 0;
	358	+}
	359	+
	360	+static int callthunks_debug_open(struct inode inode, struct file file)
	361	+{
	362	+ return single_open(file, callthunks_debug_show, inode->i_private);
	363	+}
	364	+
	365	+static const struct file_operations dfs_ops = {
	366	+ .open = callthunks_debug_open,
	367	+ .read = seq_read,
	368	+ .llseek = seq_lseek,
	369	+ .release = single_release,
	370	+};
	371	+
	372	+static int __init callthunks_debugfs_init(void)
	373	+{
	374	+ struct dentry *dir;
	375	+ unsigned long cpu;
	376	+
	377	+ dir = debugfs_create_dir("callthunks", NULL);
	378	+ for_each_possible_cpu(cpu) {
	379	+ void arg = (void )cpu;
	380	+ char name [10];
	381	+
	382	+ sprintf(name, "cpu%lu", cpu);
	383	+ debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
	384	+ }
	385	+ return 0;
	386	+}
	387	+__initcall(callthunks_debugfs_init);
	388	+#endif

arch/x86/kernel/cpu/Makefile+0−3

@@ -17,9 +17,6 @@ KMSAN_SANITIZE_common.o := n · 16 unmodified lines
17	17	# As above, instrumenting secondary CPU boot code causes boot hangs.
18	18	KCSAN_SANITIZE_common.o := n
19	19
20		−# Make sure load_percpu_segment has no stackprotector
21		−CFLAGS_common.o := -fno-stack-protector
22		−
23	20	obj-y := cacheinfo.o scattered.o topology.o
24	21	obj-y += common.o
25	22	obj-y += rdrand.o

arch/x86/kernel/cpu/bugs.c+32−2

@@ -787,6 +787,7 @@ enum retbleed_mitigation { · 786 unmodified lines
787	787	RETBLEED_MITIGATION_IBPB,
788	788	RETBLEED_MITIGATION_IBRS,
789	789	RETBLEED_MITIGATION_EIBRS,
	790	+ RETBLEED_MITIGATION_STUFF,
790	791	};
791	792
792	793	enum retbleed_mitigation_cmd {
@@ -794,6 +795,7 @@ enum retbleed_mitigation_cmd { · 1 unmodified line
794	795	RETBLEED_CMD_AUTO,
795	796	RETBLEED_CMD_UNRET,
796	797	RETBLEED_CMD_IBPB,
	798	+ RETBLEED_CMD_STUFF,
797	799	};
798	800
799	801	static const char * const retbleed_strings[] = {
@@ -802,6 +804,7 @@ static const char * const retbleed_strings[] = { · 2 unmodified lines
802	804	[RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB",
803	805	[RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS",
804	806	[RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS",
	807	+ [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing",
805	808	};
806	809
807	810	static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
@@ -831,8 +834,12 @@ static int __init retbleed_parse_cmdline(char *str) · 23 unmodified lines
831	834	retbleed_cmd = RETBLEED_CMD_UNRET;
832	835	} else if (!strcmp(str, "ibpb")) {
833	836	retbleed_cmd = RETBLEED_CMD_IBPB;
	837	+ } else if (!strcmp(str, "stuff")) {
	838	+ retbleed_cmd = RETBLEED_CMD_STUFF;
834	839	} else if (!strcmp(str, "nosmt")) {
835	840	retbleed_nosmt = true;
	841	+ } else if (!strcmp(str, "force")) {
	842	+ setup_force_cpu_bug(X86_BUG_RETBLEED);
836	843	} else {
837	844	pr_err("Ignoring unknown retbleed option (%s).", str);
838	845	}
@@ -879,6 +886,21 @@ static void __init retbleed_select_mitigation(void) · 40 unmodified lines
879	886	}
880	887	break;
881	888
	889	+ case RETBLEED_CMD_STUFF:
	890	+ if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) &&
	891	+ spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {
	892	+ retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
	893	+
	894	+ } else {
	895	+ if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING))
	896	+ pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
	897	+ else
	898	+ pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n");
	899	+
	900	+ goto do_cmd_auto;
	901	+ }
	902	+ break;
	903	+
882	904	do_cmd_auto:
883	905	case RETBLEED_CMD_AUTO:
884	906	default:
@@ -916,6 +938,12 @@ do_cmd_auto: · 31 unmodified lines
916	938	mitigate_smt = true;
917	939	break;
918	940
	941	+ case RETBLEED_MITIGATION_STUFF:
	942	+ setup_force_cpu_cap(X86_FEATURE_RETHUNK);
	943	+ setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
	944	+ x86_set_skl_return_thunk();
	945	+ break;
	946	+
919	947	default:
920	948	break;
921	949	}
@@ -926,7 +954,7 @@ do_cmd_auto: · 4 unmodified lines
926	954
927	955	/*
928	956	* Let IBRS trump all on Intel without affecting the effects of the
929		− * retbleed= cmdline option.
	957	+ * retbleed= cmdline option except for call depth based stuffing
930	958	*/
931	959	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
932	960	switch (spectre_v2_enabled) {
@@ -939,7 +967,8 @@ do_cmd_auto: · 6 unmodified lines
939	967	retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
940	968	break;
941	969	default:
942		− pr_err(RETBLEED_INTEL_MSG);
	970	+ if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
	971	+ pr_err(RETBLEED_INTEL_MSG);
943	972	}
944	973	}
945	974
@@ -1413,6 +1442,7 @@ static void __init spectre_v2_select_mitigation(void) · 467 unmodified lines
1413	1442	if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) &&
1414	1443	boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1415	1444	retbleed_cmd != RETBLEED_CMD_OFF &&
	1445	+ retbleed_cmd != RETBLEED_CMD_STUFF &&
1416	1446	boot_cpu_has(X86_FEATURE_IBRS) &&
1417	1447	boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1418	1448	mode = SPECTRE_V2_IBRS;

arch/x86/kernel/cpu/common.c+44−53

@@ -610,6 +610,7 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) · 609 unmodified lines
610	610
611	611	if (!ibt_selftest()) {
612	612	pr_err("IBT selftest: Failed!\n");
	613	+ wrmsrl(MSR_IA32_S_CET, 0);
613	614	setup_clear_cpu_cap(X86_FEATURE_IBT);
614	615	return;
615	616	}
@@ -702,16 +703,6 @@ static const char table_lookup_model(struct cpuinfo_x86 c) · 86 unmodified lines
702	703	__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
703	704	__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
704	705
705		−void load_percpu_segment(int cpu)
706		−{
707		−#ifdef CONFIG_X86_32
708		− loadsegment(fs, __KERNEL_PERCPU);
709		−#else
710		− __loadsegment_simple(gs, 0);
711		− wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
712		−#endif
713		−}
714		−
715	706	#ifdef CONFIG_X86_32
716	707	/* The 32-bit entry code needs to find cpu_entry_area. */
717	708	DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
@@ -739,16 +730,45 @@ void load_fixmap_gdt(int cpu) · 21 unmodified lines
739	730	}
740	731	EXPORT_SYMBOL_GPL(load_fixmap_gdt);
741	732
742		−/*
743		− * Current gdt points %fs at the "master" per-cpu area: after this,
744		− * it's on the real one.
	733	+/**
	734	+ * switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
	735	+ * @cpu: The CPU number for which this is invoked
	736	+ *
	737	+ * Invoked during early boot to switch from early GDT and early per CPU to
	738	+ * the direct GDT and the runtime per CPU area. On 32-bit the percpu base
	739	+ * switch is implicit by loading the direct GDT. On 64bit this requires
	740	+ * to update GSBASE.
745	741	*/
746		−void switch_to_new_gdt(int cpu)
	742	+void __init switch_gdt_and_percpu_base(int cpu)
747	743	{
748		− /* Load the original GDT */
749	744	load_direct_gdt(cpu);
750		− /* Reload the per-cpu base */
751		− load_percpu_segment(cpu);
	745	+
	746	+#ifdef CONFIG_X86_64
	747	+ /*
	748	+ * No need to load %gs. It is already correct.
	749	+ *
	750	+ * Writing %gs on 64bit would zero GSBASE which would make any per
	751	+ * CPU operation up to the point of the wrmsrl() fault.
	752	+ *
	753	+ * Set GSBASE to the new offset. Until the wrmsrl() happens the
	754	+ * early mapping is still valid. That means the GSBASE update will
	755	+ * lose any prior per CPU data which was not copied over in
	756	+ * setup_per_cpu_areas().
	757	+ *
	758	+ * This works even with stackprotector enabled because the
	759	+ * per CPU stack canary is 0 in both per CPU areas.
	760	+ */
	761	+ wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
	762	+#else
	763	+ /*
	764	+ * %fs is already set to __KERNEL_PERCPU, but after switching GDT
	765	+ * it is required to load FS again so that the 'hidden' part is
	766	+ * updated from the new GDT. Up to this point the early per CPU
	767	+ * translation is active. Any content of the early per CPU data
	768	+ * which was not copied over in setup_per_cpu_areas() is lost.
	769	+ */
	770	+ loadsegment(fs, __KERNEL_PERCPU);
	771	+#endif
752	772	}
753	773
754	774	static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
@@ -1993,27 +2013,18 @@ static __init int setup_clearcpuid(char *arg) · 1238 unmodified lines
1993	2013	}
1994	2014	__setup("clearcpuid=", setup_clearcpuid);
1995	2015
	2016	+DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
	2017	+ .current_task = &init_task,
	2018	+ .preempt_count = INIT_PREEMPT_COUNT,
	2019	+ .top_of_stack = TOP_OF_INIT_STACK,
	2020	+};
	2021	+EXPORT_PER_CPU_SYMBOL(pcpu_hot);
	2022	+
1996	2023	#ifdef CONFIG_X86_64
1997	2024	DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
1998	2025	fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
1999	2026	EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2000	2027
2001		−/*
2002		− * The following percpu variables are hot. Align current_task to
2003		− * cacheline size such that they fall in the same cacheline.
2004		− */
2005		−DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
2006		− &init_task;
2007		−EXPORT_PER_CPU_SYMBOL(current_task);
2008		−
2009		−DEFINE_PER_CPU(void *, hardirq_stack_ptr);
2010		−DEFINE_PER_CPU(bool, hardirq_stack_inuse);
2011		−
2012		−DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2013		−EXPORT_PER_CPU_SYMBOL(__preempt_count);
2014		−
2015		−DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2016		−
2017	2028	static void wrmsrl_cstar(unsigned long val)
2018	2029	{
2019	2030	/*
@@ -2064,20 +2075,6 @@ void syscall_init(void) · 44 unmodified lines
2064	2075
2065	2076	#else /* CONFIG_X86_64 */
2066	2077
2067		−DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
2068		−EXPORT_PER_CPU_SYMBOL(current_task);
2069		−DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2070		−EXPORT_PER_CPU_SYMBOL(__preempt_count);
2071		−
2072		−/*
2073		− * On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
2074		− * the top of the kernel stack. Use an extra percpu variable to track the
2075		− * top of the kernel stack directly.
2076		− */
2077		−DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
2078		− (unsigned long)&init_thread_union + THREAD_SIZE;
2079		−EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
2080		−
2081	2078	#ifdef CONFIG_STACKPROTECTOR
2082	2079	DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2083	2080	EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
@@ -2248,12 +2245,6 @@ void cpu_init(void) · 164 unmodified lines
2248	2245	boot_cpu_has(X86_FEATURE_TSC) \|\| boot_cpu_has(X86_FEATURE_DE))
2249	2246	cr4_clear_bits(X86_CR4_VME\|X86_CR4_PVI\|X86_CR4_TSD\|X86_CR4_DE);
2250	2247
2251		− /*
2252		− * Initialize the per-CPU GDT with the boot GDT,
2253		− * and set up the GDT descriptor:
2254		− */
2255		− switch_to_new_gdt(cpu);
2256		−
2257	2248	if (IS_ENABLED(CONFIG_X86_64)) {
2258	2249	loadsegment(fs, 0);
2259	2250	memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);

arch/x86/kernel/dumpstack_32.c+2−2

@@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_type type) · 36 unmodified lines
37	37
38	38	static bool in_hardirq_stack(unsigned long stack, struct stack_info info)
39	39	{
40		− unsigned long begin = (unsigned long )this_cpu_read(hardirq_stack_ptr);
	40	+ unsigned long begin = (unsigned long )this_cpu_read(pcpu_hot.hardirq_stack_ptr);
41	41	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));
42	42
43	43	/*
@@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned long stack, struct stack_info info) · 18 unmodified lines
62	62
63	63	static bool in_softirq_stack(unsigned long stack, struct stack_info info)
64	64	{
65		− unsigned long begin = (unsigned long )this_cpu_read(softirq_stack_ptr);
	65	+ unsigned long begin = (unsigned long )this_cpu_read(pcpu_hot.softirq_stack_ptr);
66	66	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));
67	67
68	68	/*

arch/x86/kernel/dumpstack_64.c+1−1

@@ -134,7 +134,7 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac · 133 unmodified lines
134	134
135	135	static __always_inline bool in_irq_stack(unsigned long stack, struct stack_info info)
136	136	{
137		− unsigned long end = (unsigned long )this_cpu_read(hardirq_stack_ptr);
	137	+ unsigned long end = (unsigned long )this_cpu_read(pcpu_hot.hardirq_stack_ptr);
138	138	unsigned long *begin;
139	139
140	140	/*

arch/x86/kernel/ftrace.c+13−7

@@ -69,6 +69,10 @@ static const char *ftrace_nop_replace(void) · 68 unmodified lines
69	69
70	70	static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
71	71	{
	72	+ /*
	73	+ * No need to translate into a callthunk. The trampoline does
	74	+ * the depth accounting itself.
	75	+ */
72	76	return text_gen_insn(CALL_INSN_OPCODE, (void )ip, (void )addr);
73	77	}
74	78
@@ -317,7 +321,7 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 242 unmodified lines
317	321	unsigned long size;
318	322	unsigned long *ptr;
319	323	void *trampoline;
320		− void *ip;
	324	+ void ip, dest;
321	325	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
322	326	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
323	327	unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
@@ -359,7 +363,7 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 35 unmodified lines
359	363
360	364	ip = trampoline + size;
361	365	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
362		− __text_gen_insn(ip, JMP32_INSN_OPCODE, ip, &__x86_return_thunk, JMP32_INSN_SIZE);
	366	+ __text_gen_insn(ip, JMP32_INSN_OPCODE, ip, x86_return_thunk, JMP32_INSN_SIZE);
363	367	else
364	368	memcpy(ip, retq, sizeof(retq));
365	369
@@ -404,17 +408,19 @@ create_trampoline(struct ftrace_ops ops, unsigned int tramp_size) · 38 unmodified lines
404	408	/* put in the call to the function */
405	409	mutex_lock(&text_mutex);
406	410	call_offset -= start_offset;
	411	+ /*
	412	+ * No need to translate into a callthunk. The trampoline does
	413	+ * the depth accounting before the call already.
	414	+ */
	415	+ dest = ftrace_ops_get_func(ops);
407	416	memcpy(trampoline + call_offset,
408		− text_gen_insn(CALL_INSN_OPCODE,
409		− trampoline + call_offset,
410		− ftrace_ops_get_func(ops)), CALL_INSN_SIZE);
	417	+ text_gen_insn(CALL_INSN_OPCODE, trampoline + call_offset, dest),
	418	+ CALL_INSN_SIZE);
411	419	mutex_unlock(&text_mutex);
412	420
413	421	/* ALLOC_TRAMP flags lets us know we created it */
414	422	ops->flags \|= FTRACE_OPS_FL_ALLOC_TRAMP;
415	423
416		− set_vm_flush_reset_perms(trampoline);
417		−
418	424	if (likely(system_state != SYSTEM_BOOTING))
419	425	set_memory_ro((unsigned long)trampoline, npages);
420	426	set_memory_x((unsigned long)trampoline, npages);

arch/x86/kernel/ftrace_64.S+34−3

@@ -3,8 +3,9 @@ · 2 unmodified lines
3	3	* Copyright (C) 2014 Steven Rostedt, Red Hat Inc
4	4	*/
5	5
6		−#include <linux/linkage.h>
7	6	#include <linux/cfi_types.h>
	7	+#include <linux/linkage.h>
	8	+#include <asm/asm-offsets.h>
8	9	#include <asm/ptrace.h>
9	10	#include <asm/ftrace.h>
10	11	#include <asm/export.h>
@@ -131,16 +132,19 @@ · 120 unmodified lines
131	132	.endm
132	133
133	134	SYM_TYPED_FUNC_START(ftrace_stub)
	135	+ CALL_DEPTH_ACCOUNT
134	136	RET
135	137	SYM_FUNC_END(ftrace_stub)
136	138
137	139	SYM_TYPED_FUNC_START(ftrace_stub_graph)
	140	+ CALL_DEPTH_ACCOUNT
138	141	RET
139	142	SYM_FUNC_END(ftrace_stub_graph)
140	143
141	144	#ifdef CONFIG_DYNAMIC_FTRACE
142	145
143	146	SYM_FUNC_START(__fentry__)
	147	+ CALL_DEPTH_ACCOUNT
144	148	RET
145	149	SYM_FUNC_END(__fentry__)
146	150	EXPORT_SYMBOL(__fentry__)
@@ -149,6 +153,8 @@ SYM_FUNC_START(ftrace_caller) · 2 unmodified lines
149	153	/* save_mcount_regs fills in first two parameters */
150	154	save_mcount_regs
151	155
	156	+ CALL_DEPTH_ACCOUNT
	157	+
152	158	/* Stack - skipping return address of ftrace_caller */
153	159	leaq MCOUNT_REG_SIZE+8(%rsp), %rcx
154	160	movq %rcx, RSP(%rsp)
@@ -164,6 +170,9 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) · 9 unmodified lines
164	170	/* Only ops with REGS flag set should have CS register set */
165	171	movq $0, CS(%rsp)
166	172
	173	+ /* Account for the function call below */
	174	+ CALL_DEPTH_ACCOUNT
	175	+
167	176	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
168	177	ANNOTATE_NOENDBR
169	178	call ftrace_stub
@@ -193,6 +202,8 @@ SYM_FUNC_START(ftrace_regs_caller) · 23 unmodified lines
193	202	save_mcount_regs 8
194	203	/* save_mcount_regs fills in first two parameters */
195	204
	205	+ CALL_DEPTH_ACCOUNT
	206	+
196	207	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
197	208	ANNOTATE_NOENDBR
198	209	/* Load the ftrace_ops into the 3rd parameter */
@@ -223,6 +234,9 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) · 24 unmodified lines
223	234	/* regs go into 4th parameter */
224	235	leaq (%rsp), %rcx
225	236
	237	+ /* Account for the function call below */
	238	+ CALL_DEPTH_ACCOUNT
	239	+
226	240	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
227	241	ANNOTATE_NOENDBR
228	242	call ftrace_stub
@@ -275,7 +289,20 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) · 46 unmodified lines
275	289	/* Restore flags */
276	290	popfq
277	291	UNWIND_HINT_FUNC
278		− RET
	292	+
	293	+ /*
	294	+ * The above left an extra return value on the stack; effectively
	295	+ * doing a tail-call without using a register. This PUSH;RET
	296	+ * pattern unbalances the RSB, inject a pointless CALL to rebalance.
	297	+ */
	298	+ ANNOTATE_INTRA_FUNCTION_CALL
	299	+ CALL .Ldo_rebalance
	300	+ int3
	301	+.Ldo_rebalance:
	302	+ add $8, %rsp
	303	+ ALTERNATIVE __stringify(RET), \
	304	+ __stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
	305	+ X86_FEATURE_CALL_DEPTH
279	306
280	307	SYM_FUNC_END(ftrace_regs_caller)
281	308	STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
@@ -284,6 +311,8 @@ STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller) · 2 unmodified lines
284	311	#else /* ! CONFIG_DYNAMIC_FTRACE */
285	312
286	313	SYM_FUNC_START(__fentry__)
	314	+ CALL_DEPTH_ACCOUNT
	315	+
287	316	cmpq $ftrace_stub, ftrace_trace_function
288	317	jnz trace
289	318	RET
@@ -337,6 +366,8 @@ SYM_CODE_START(return_to_handler) · 47 unmodified lines
337	366	int3
338	367	.Ldo_rop:
339	368	mov %rdi, (%rsp)
340		− RET
	369	+ ALTERNATIVE __stringify(RET), \
	370	+ __stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
	371	+ X86_FEATURE_CALL_DEPTH
341	372	SYM_CODE_END(return_to_handler)
342	373	#endif

arch/x86/kernel/head_64.S+1−0

@@ -370,6 +370,7 @@ SYM_CODE_END(secondary_startup_64) · 369 unmodified lines
370	370	* start_secondary() via .Ljump_to_C_code.
371	371	*/
372	372	SYM_CODE_START(start_cpu0)
	373	+ ANNOTATE_NOENDBR
373	374	UNWIND_HINT_EMPTY
374	375	movq initial_stack(%rip), %rsp
375	376	jmp .Ljump_to_C_code

arch/x86/kernel/irq_32.c+5−8

@@ -52,9 +52,6 @@ static inline int check_stack_overflow(void) { return 0; } · 51 unmodified lines
52	52	static inline void print_stack_overflow(void) { }
53	53	#endif
54	54
55		−DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
56		−DEFINE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
57		−
58	55	static void call_on_stack(void func, void stack)
59	56	{
60	57	asm volatile("xchgl %%ebx,%%esp \n"
@@ -77,7 +74,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc) · 16 unmodified lines
77	74	u32 isp, prev_esp, arg1;
78	75
79	76	curstk = (struct irq_stack *) current_stack();
80		− irqstk = __this_cpu_read(hardirq_stack_ptr);
	77	+ irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr);
81	78
82	79	/*
83	80	* this is where we switch to the IRQ stack. However, if we are
@@ -115,7 +112,7 @@ int irq_init_percpu_irqstack(unsigned int cpu) · 31 unmodified lines
115	112	int node = cpu_to_node(cpu);
116	113	struct page ph, ps;
117	114
118		− if (per_cpu(hardirq_stack_ptr, cpu))
	115	+ if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
119	116	return 0;
120	117
121	118	ph = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
@@ -127,8 +124,8 @@ int irq_init_percpu_irqstack(unsigned int cpu) · 5 unmodified lines
127	124	return -ENOMEM;
128	125	}
129	126
130		− per_cpu(hardirq_stack_ptr, cpu) = page_address(ph);
131		− per_cpu(softirq_stack_ptr, cpu) = page_address(ps);
	127	+ per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph);
	128	+ per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps);
132	129	return 0;
133	130	}
134	131
@@ -138,7 +135,7 @@ void do_softirq_own_stack(void) · 3 unmodified lines
138	135	struct irq_stack *irqstk;
139	136	u32 isp, prev_esp;
140	137
141		− irqstk = __this_cpu_read(softirq_stack_ptr);
	138	+ irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr);
142	139
143	140	/* build the stack frame on the softirq stack */
144	141	isp = (u32 ) ((char )irqstk + sizeof(*irqstk));

arch/x86/kernel/irq_64.c+3−3

@@ -50,7 +50,7 @@ static int map_irq_stack(unsigned int cpu) · 49 unmodified lines
50	50	return -ENOMEM;
51	51
52	52	/* Store actual TOS to avoid adjustment in the hotpath */
53		− per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
	53	+ per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
54	54	return 0;
55	55	}
56	56	#else
@@ -63,14 +63,14 @@ static int map_irq_stack(unsigned int cpu) · 6 unmodified lines
63	63	void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);
64	64
65	65	/* Store actual TOS to avoid adjustment in the hotpath */
66		− per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
	66	+ per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
67	67	return 0;
68	68	}
69	69	#endif
70	70
71	71	int irq_init_percpu_irqstack(unsigned int cpu)
72	72	{
73		− if (per_cpu(hardirq_stack_ptr, cpu))
	73	+ if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
74	74	return 0;
75	75	return map_irq_stack(cpu);
76	76	}

arch/x86/kernel/kprobes/core.c+0−1

@@ -414,7 +414,6 @@ void *alloc_insn_page(void) · 413 unmodified lines
414	414	if (!page)
415	415	return NULL;
416	416
417		− set_vm_flush_reset_perms(page);
418	417	/*
419	418	* First make the page read-only, and only then make it executable to
420	419	* prevent it from being W+X in between.

arch/x86/kernel/kvm.c+6−12

@@ -798,19 +798,13 @@ extern bool __raw_callee_save___kvm_vcpu_is_preempted(long); · 797 unmodified lines
798	798	* Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
799	799	* restoring to/from the stack.
800	800	*/
801		−asm(
802		−".pushsection .text;"
803		−".global __raw_callee_save___kvm_vcpu_is_preempted;"
804		−".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
805		−"__raw_callee_save___kvm_vcpu_is_preempted:"
806		−ASM_ENDBR
807		−"movq __per_cpu_offset(,%rdi,8), %rax;"
808		−"cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
809		−"setne %al;"
810		−ASM_RET
811		−".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
812		−".popsection");
	801	+#define PV_VCPU_PREEMPTED_ASM \
	802	+ "movq __per_cpu_offset(,%rdi,8), %rax\n\t" \
	803	+ "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax)\n\t" \
	804	+ "setne %al\n\t"
813	805
	806	+DEFINE_PARAVIRT_ASM(__raw_callee_save___kvm_vcpu_is_preempted,
	807	+ PV_VCPU_PREEMPTED_ASM, .text);
814	808	#endif
815	809
816	810	static void __init kvm_guest_init(void)

arch/x86/kernel/module.c+42−5

@@ -74,10 +74,11 @@ void *module_alloc(unsigned long size) · 73 unmodified lines
74	74	return NULL;
75	75
76	76	p = __vmalloc_node_range(size, MODULE_ALIGN,
77		− MODULES_VADDR + get_module_load_offset(),
78		− MODULES_END, gfp_mask,
79		− PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,
80		− __builtin_return_address(0));
	77	+ MODULES_VADDR + get_module_load_offset(),
	78	+ MODULES_END, gfp_mask, PAGE_KERNEL,
	79	+ VM_FLUSH_RESET_PERMS \| VM_DEFER_KMEMLEAK,
	80	+ NUMA_NO_NODE, __builtin_return_address(0));
	81	+
81	82	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
82	83	vfree(p);
83	84	return NULL;
@@ -253,7 +254,8 @@ int module_finalize(const Elf_Ehdr *hdr, · 169 unmodified lines
253	254	{
254	255	const Elf_Shdr s, alt = NULL, *locks = NULL,
255	256	para = NULL, orc = NULL, *orc_ip = NULL,
256		− retpolines = NULL, returns = NULL, *ibt_endbr = NULL;
	257	+ retpolines = NULL, returns = NULL, *ibt_endbr = NULL,
	258	+ calls = NULL, cfi = NULL;
257	259	char secstrings = (void )hdr + sechdrs[hdr->e_shstrndx].sh_offset;
258	260
259	261	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -271,6 +273,10 @@ int module_finalize(const Elf_Ehdr *hdr, · 11 unmodified lines
271	273	retpolines = s;
272	274	if (!strcmp(".return_sites", secstrings + s->sh_name))
273	275	returns = s;
	276	+ if (!strcmp(".call_sites", secstrings + s->sh_name))
	277	+ calls = s;
	278	+ if (!strcmp(".cfi_sites", secstrings + s->sh_name))
	279	+ cfi = s;
274	280	if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
275	281	ibt_endbr = s;
276	282	}
@@ -283,6 +289,22 @@ int module_finalize(const Elf_Ehdr *hdr, · 6 unmodified lines
283	289	void pseg = (void )para->sh_addr;
284	290	apply_paravirt(pseg, pseg + para->sh_size);
285	291	}
	292	+ if (retpolines \|\| cfi) {
	293	+ void rseg = NULL, cseg = NULL;
	294	+ unsigned int rsize = 0, csize = 0;
	295	+
	296	+ if (retpolines) {
	297	+ rseg = (void *)retpolines->sh_addr;
	298	+ rsize = retpolines->sh_size;
	299	+ }
	300	+
	301	+ if (cfi) {
	302	+ cseg = (void *)cfi->sh_addr;
	303	+ csize = cfi->sh_size;
	304	+ }
	305	+
	306	+ apply_fineibt(rseg, rseg + rsize, cseg, cseg + csize);
	307	+ }
286	308	if (retpolines) {
287	309	void rseg = (void )retpolines->sh_addr;
288	310	apply_retpolines(rseg, rseg + retpolines->sh_size);
@@ -296,6 +318,21 @@ int module_finalize(const Elf_Ehdr *hdr, · 7 unmodified lines
296	318	void aseg = (void )alt->sh_addr;
297	319	apply_alternatives(aseg, aseg + alt->sh_size);
298	320	}
	321	+ if (calls \|\| para) {
	322	+ struct callthunk_sites cs = {};
	323	+
	324	+ if (calls) {
	325	+ cs.call_start = (void *)calls->sh_addr;
	326	+ cs.call_end = (void *)calls->sh_addr + calls->sh_size;
	327	+ }
	328	+
	329	+ if (para) {
	330	+ cs.pv_start = (void *)para->sh_addr;
	331	+ cs.pv_end = (void *)para->sh_addr + para->sh_size;
	332	+ }
	333	+
	334	+ callthunks_patch_module_calls(&cs, me);
	335	+ }
299	336	if (ibt_endbr) {
300	337	void iseg = (void )ibt_endbr->sh_addr;
301	338	apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);

arch/x86/kernel/paravirt.c+2−19

@@ -37,27 +37,10 @@ · 36 unmodified lines
37	37	* nop stub, which must not clobber anything including the stack to
38	38	* avoid confusing the entry prologues.
39	39	*/
40		−extern void _paravirt_nop(void);
41		−asm (".pushsection .entry.text, \"ax\"\n"
42		− ".global _paravirt_nop\n"
43		− "_paravirt_nop:\n\t"
44		− ASM_ENDBR
45		− ASM_RET
46		− ".size _paravirt_nop, . - _paravirt_nop\n\t"
47		− ".type _paravirt_nop, @function\n\t"
48		− ".popsection");
	40	+DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text);
49	41
50	42	/* stub always returning 0. */
51		−asm (".pushsection .entry.text, \"ax\"\n"
52		− ".global paravirt_ret0\n"
53		− "paravirt_ret0:\n\t"
54		− ASM_ENDBR
55		− "xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
56		− ASM_RET
57		− ".size paravirt_ret0, . - paravirt_ret0\n\t"
58		− ".type paravirt_ret0, @function\n\t"
59		− ".popsection");
60		−
	43	+DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text);
61	44
62	45	void __init default_banner(void)
63	46	{

arch/x86/kernel/process_32.c+3−3

@@ -191,13 +191,13 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 190 unmodified lines
191	191	arch_end_context_switch(next_p);
192	192
193	193	/*
194		− * Reload esp0 and cpu_current_top_of_stack. This changes
	194	+ * Reload esp0 and pcpu_hot.top_of_stack. This changes
195	195	* current_thread_info(). Refresh the SYSENTER configuration in
196	196	* case prev or next is vm86.
197	197	*/
198	198	update_task_stack(next_p);
199	199	refresh_sysenter_cs(next);
200		− this_cpu_write(cpu_current_top_of_stack,
	200	+ this_cpu_write(pcpu_hot.top_of_stack,
201	201	(unsigned long)task_stack_page(next_p) +
202	202	THREAD_SIZE);
203	203
@@ -207,7 +207,7 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 3 unmodified lines
207	207	if (prev->gs \| next->gs)
208	208	loadsegment(gs, next->gs);
209	209
210		− this_cpu_write(current_task, next_p);
	210	+ raw_cpu_write(pcpu_hot.current_task, next_p);
211	211
212	212	switch_fpu_finish();
213	213

arch/x86/kernel/process_64.c+3−3

@@ -563,7 +563,7 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 562 unmodified lines
563	563	int cpu = smp_processor_id();
564	564
565	565	WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
566		− this_cpu_read(hardirq_stack_inuse));
	566	+ this_cpu_read(pcpu_hot.hardirq_stack_inuse));
567	567
568	568	if (!test_thread_flag(TIF_NEED_FPU_LOAD))
569	569	switch_fpu_prepare(prev_fpu, cpu);
@@ -617,8 +617,8 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 47 unmodified lines
617	617	/*
618	618	* Switch the PDA and FPU contexts.
619	619	*/
620		− this_cpu_write(current_task, next_p);
621		− this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
	620	+ raw_cpu_write(pcpu_hot.current_task, next_p);
	621	+ raw_cpu_write(pcpu_hot.top_of_stack, task_top_of_stack(next_p));
622	622
623	623	switch_fpu_finish();
624	624

arch/x86/kernel/relocate_kernel_64.S+3−2

@@ -41,6 +41,7 @@ · 40 unmodified lines
41	41	.text
42	42	.align PAGE_SIZE
43	43	.code64
	44	+SYM_CODE_START_NOALIGN(relocate_range)
44	45	SYM_CODE_START_NOALIGN(relocate_kernel)
45	46	UNWIND_HINT_EMPTY
46	47	ANNOTATE_NOENDBR
@@ -312,5 +313,5 @@ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) · 265 unmodified lines
312	313	int3
313	314	SYM_CODE_END(swap_pages)
314	315
315		− .globl kexec_control_code_size
316		−.set kexec_control_code_size, . - relocate_kernel
	316	+ .skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc
	317	+SYM_CODE_END(relocate_range);

arch/x86/kernel/setup_percpu.c+2−5

@@ -23,9 +23,6 @@ · 22 unmodified lines
23	23	#include <asm/cpumask.h>
24	24	#include <asm/cpu.h>
25	25
26		−DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
27		−EXPORT_PER_CPU_SYMBOL(cpu_number);
28		−
29	26	#ifdef CONFIG_X86_64
30	27	#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
31	28	#else
@@ -172,7 +169,7 @@ void __init setup_per_cpu_areas(void) · 140 unmodified lines
172	169	for_each_possible_cpu(cpu) {
173	170	per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu];
174	171	per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
175		− per_cpu(cpu_number, cpu) = cpu;
	172	+ per_cpu(pcpu_hot.cpu_number, cpu) = cpu;
176	173	setup_percpu_segment(cpu);
177	174	/*
178	175	* Copy data used in early init routines from the
@@ -211,7 +208,7 @@ void __init setup_per_cpu_areas(void) · 32 unmodified lines
211	208	* area. Reload any changed state for the boot CPU.
212	209	*/
213	210	if (!cpu)
214		− switch_to_new_gdt(cpu);
	211	+ switch_gdt_and_percpu_base(cpu);
215	212	}
216	213
217	214	/* indicate the early static arrays will soon be gone */

arch/x86/kernel/smpboot.c+7−3

@@ -1048,7 +1048,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) · 1047 unmodified lines
1048	1048	/* Just in case we booted with a single CPU. */
1049	1049	alternatives_enable_smp();
1050	1050
1051		− per_cpu(current_task, cpu) = idle;
	1051	+ per_cpu(pcpu_hot.current_task, cpu) = idle;
1052	1052	cpu_init_stack_canary(cpu, idle);
1053	1053
1054	1054	/* Initialize the interrupt stack(s) */
@@ -1058,7 +1058,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) · 3 unmodified lines
1058	1058
1059	1059	#ifdef CONFIG_X86_32
1060	1060	/* Stack for startup_32 can be just as for start_secondary onwards */
1061		− per_cpu(cpu_current_top_of_stack, cpu) = task_top_of_stack(idle);
	1061	+ per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle);
1062	1062	#else
1063	1063	initial_gs = per_cpu_offset(cpu);
1064	1064	#endif
@@ -1453,7 +1453,11 @@ void arch_thaw_secondary_cpus_end(void) · 388 unmodified lines
1453	1453	void __init native_smp_prepare_boot_cpu(void)
1454	1454	{
1455	1455	int me = smp_processor_id();
1456		− switch_to_new_gdt(me);
	1456	+
	1457	+ /* SMP handles this from setup_per_cpu_areas() */
	1458	+ if (!IS_ENABLED(CONFIG_SMP))
	1459	+ switch_gdt_and_percpu_base(me);
	1460	+
1457	1461	/* already set me in cpu_online_mask in boot_cpu_init() */
1458	1462	cpumask_set_cpu(me, cpu_callout_mask);
1459	1463	cpu_set_state_online(me);

arch/x86/kernel/static_call.c+2−1

@@ -34,6 +34,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, · 33 unmodified lines
34	34
35	35	switch (type) {
36	36	case CALL:
	37	+ func = callthunks_translate_call_dest(func);
37	38	code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
38	39	if (func == &__static_call_return0) {
39	40	emulate = code;
@@ -52,7 +53,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, · 12 unmodified lines
52	53
53	54	case RET:
54	55	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
55		− code = text_gen_insn(JMP32_INSN_OPCODE, insn, &__x86_return_thunk);
	56	+ code = text_gen_insn(JMP32_INSN_OPCODE, insn, x86_return_thunk);
56	57	else
57	58	code = &retinsn;
58	59	break;

arch/x86/kernel/traps.c+2−2

@@ -858,7 +858,7 @@ DEFINE_IDTENTRY_RAW(exc_int3) · 857 unmodified lines
858	858	*/
859	859	asmlinkage __visible noinstr struct pt_regs sync_regs(struct pt_regs eregs)
860	860	{
861		− struct pt_regs regs = (struct pt_regs )this_cpu_read(cpu_current_top_of_stack) - 1;
	861	+ struct pt_regs regs = (struct pt_regs )this_cpu_read(pcpu_hot.top_of_stack) - 1;
862	862	if (regs != eregs)
863	863	regs = eregs;
864	864	return regs;
@@ -876,7 +876,7 @@ asmlinkage __visible noinstr struct pt_regs vc_switch_off_ist(struct pt_regs r · 11 unmodified lines
876	876	* trust it and switch to the current kernel stack
877	877	*/
878	878	if (ip_within_syscall_gap(regs)) {
879		− sp = this_cpu_read(cpu_current_top_of_stack);
	879	+ sp = this_cpu_read(pcpu_hot.top_of_stack);
880	880	goto sync;
881	881	}
882	882

arch/x86/kernel/unwind_orc.c+20−1

@@ -136,6 +136,21 @@ static struct orc_entry null_orc_entry = { · 135 unmodified lines
136	136	.type = UNWIND_HINT_TYPE_CALL
137	137	};
138	138
	139	+#ifdef CONFIG_CALL_THUNKS
	140	+static struct orc_entry *orc_callthunk_find(unsigned long ip)
	141	+{
	142	+ if (!is_callthunk((void *)ip))
	143	+ return NULL;
	144	+
	145	+ return &null_orc_entry;
	146	+}
	147	+#else
	148	+static struct orc_entry *orc_callthunk_find(unsigned long ip)
	149	+{
	150	+ return NULL;
	151	+}
	152	+#endif
	153	+
139	154	/* Fake frame pointer entry -- used as a fallback for generated code */
140	155	static struct orc_entry orc_fp_entry = {
141	156	.type = UNWIND_HINT_TYPE_CALL,
@@ -189,7 +204,11 @@ static struct orc_entry *orc_find(unsigned long ip) · 47 unmodified lines
189	204	if (orc)
190	205	return orc;
191	206
192		− return orc_ftrace_find(ip);
	207	+ orc = orc_ftrace_find(ip);
	208	+ if (orc)
	209	+ return orc;
	210	+
	211	+ return orc_callthunk_find(ip);
193	212	}
194	213
195	214	#ifdef CONFIG_MODULES

arch/x86/kernel/vmlinux.lds.S+23−14

@@ -132,18 +132,19 @@ SECTIONS · 131 unmodified lines
132	132	CPUIDLE_TEXT
133	133	LOCK_TEXT
134	134	KPROBES_TEXT
135		− ALIGN_ENTRY_TEXT_BEGIN
136		− ENTRY_TEXT
137		− ALIGN_ENTRY_TEXT_END
138	135	SOFTIRQENTRY_TEXT
139		− STATIC_CALL_TEXT
140		− *(.gnu.warning)
141		−
142	136	#ifdef CONFIG_RETPOLINE
143	137	__indirect_thunk_start = .;
144	138	(.text.__x86.)
145	139	__indirect_thunk_end = .;
146	140	#endif
	141	+ STATIC_CALL_TEXT
	142	+
	143	+ ALIGN_ENTRY_TEXT_BEGIN
	144	+ ENTRY_TEXT
	145	+ ALIGN_ENTRY_TEXT_END
	146	+ *(.gnu.warning)
	147	+
147	148	} :text =0xcccc
148	149
149	150	/* End of text section, which should occupy whole number of pages */
@@ -290,6 +291,13 @@ SECTIONS · 140 unmodified lines
290	291	*(.return_sites)
291	292	__return_sites_end = .;
292	293	}
	294	+
	295	+ . = ALIGN(8);
	296	+ .call_sites : AT(ADDR(.call_sites) - LOAD_OFFSET) {
	297	+ __call_sites = .;
	298	+ *(.call_sites)
	299	+ __call_sites_end = .;
	300	+ }
293	301	#endif
294	302
295	303	#ifdef CONFIG_X86_KERNEL_IBT
@@ -301,6 +309,15 @@ SECTIONS · 5 unmodified lines
301	309	}
302	310	#endif
303	311
	312	+#ifdef CONFIG_FINEIBT
	313	+ . = ALIGN(8);
	314	+ .cfi_sites : AT(ADDR(.cfi_sites) - LOAD_OFFSET) {
	315	+ __cfi_sites = .;
	316	+ *(.cfi_sites)
	317	+ __cfi_sites_end = .;
	318	+ }
	319	+#endif
	320	+
304	321	/*
305	322	* struct alt_inst entries. From the header (alternative.h):
306	323	* "Alternative instructions for different CPU types or capabilities"
@@ -493,11 +510,3 @@ INIT_PER_CPU(irq_stack_backing_store); · 186 unmodified lines
493	510	#endif
494	511
495	512	#endif /* CONFIG_X86_64 */
496		−
497		−#ifdef CONFIG_KEXEC_CORE
498		−#include <asm/kexec.h>
499		−
500		−. = ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
501		− "kexec control code size is too big");
502		−#endif
503		−

arch/x86/kvm/svm/vmenter.S+1−0

@@ -1,6 +1,7 @@
1	1	/* SPDX-License-Identifier: GPL-2.0 */
2	2	#include <linux/linkage.h>
3	3	#include <asm/asm.h>
	4	+#include <asm/asm-offsets.h>
4	5	#include <asm/bitsperlong.h>
5	6	#include <asm/kvm_vcpu_regs.h>
6	7	#include <asm/nospec-branch.h>

arch/x86/lib/error-inject.c+1−0

@@ -11,6 +11,7 @@ asm( · 10 unmodified lines
11	11	".text\n"
12	12	".type just_return_func, @function\n"
13	13	".globl just_return_func\n"
	14	+ ASM_FUNC_ALIGN
14	15	"just_return_func:\n"
15	16	ANNOTATE_NOENDBR
16	17	ASM_RET

arch/x86/lib/putuser.S+49−13

@@ -47,8 +47,6 @@ SYM_FUNC_START(__put_user_1) · 46 unmodified lines
47	47	LOAD_TASK_SIZE_MINUS_N(0)
48	48	cmp %_ASM_BX,%_ASM_CX
49	49	jae .Lbad_put_user
50		−SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
51		− ENDBR
52	50	ASM_STAC
53	51	1: movb %al,(%_ASM_CX)
54	52	xor %ecx,%ecx
@@ -56,54 +54,87 @@ SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL) · 1 unmodified line
56	54	RET
57	55	SYM_FUNC_END(__put_user_1)
58	56	EXPORT_SYMBOL(__put_user_1)
	57	+
	58	+SYM_FUNC_START(__put_user_nocheck_1)
	59	+ ENDBR
	60	+ ASM_STAC
	61	+2: movb %al,(%_ASM_CX)
	62	+ xor %ecx,%ecx
	63	+ ASM_CLAC
	64	+ RET
	65	+SYM_FUNC_END(__put_user_nocheck_1)
59	66	EXPORT_SYMBOL(__put_user_nocheck_1)
60	67
61	68	SYM_FUNC_START(__put_user_2)
62	69	LOAD_TASK_SIZE_MINUS_N(1)
63	70	cmp %_ASM_BX,%_ASM_CX
64	71	jae .Lbad_put_user
65		−SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
66		− ENDBR
67	72	ASM_STAC
68		−2: movw %ax,(%_ASM_CX)
	73	+3: movw %ax,(%_ASM_CX)
69	74	xor %ecx,%ecx
70	75	ASM_CLAC
71	76	RET
72	77	SYM_FUNC_END(__put_user_2)
73	78	EXPORT_SYMBOL(__put_user_2)
	79	+
	80	+SYM_FUNC_START(__put_user_nocheck_2)
	81	+ ENDBR
	82	+ ASM_STAC
	83	+4: movw %ax,(%_ASM_CX)
	84	+ xor %ecx,%ecx
	85	+ ASM_CLAC
	86	+ RET
	87	+SYM_FUNC_END(__put_user_nocheck_2)
74	88	EXPORT_SYMBOL(__put_user_nocheck_2)
75	89
76	90	SYM_FUNC_START(__put_user_4)
77	91	LOAD_TASK_SIZE_MINUS_N(3)
78	92	cmp %_ASM_BX,%_ASM_CX
79	93	jae .Lbad_put_user
80		−SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
81		− ENDBR
82	94	ASM_STAC
83		−3: movl %eax,(%_ASM_CX)
	95	+5: movl %eax,(%_ASM_CX)
84	96	xor %ecx,%ecx
85	97	ASM_CLAC
86	98	RET
87	99	SYM_FUNC_END(__put_user_4)
88	100	EXPORT_SYMBOL(__put_user_4)
	101	+
	102	+SYM_FUNC_START(__put_user_nocheck_4)
	103	+ ENDBR
	104	+ ASM_STAC
	105	+6: movl %eax,(%_ASM_CX)
	106	+ xor %ecx,%ecx
	107	+ ASM_CLAC
	108	+ RET
	109	+SYM_FUNC_END(__put_user_nocheck_4)
89	110	EXPORT_SYMBOL(__put_user_nocheck_4)
90	111
91	112	SYM_FUNC_START(__put_user_8)
92	113	LOAD_TASK_SIZE_MINUS_N(7)
93	114	cmp %_ASM_BX,%_ASM_CX
94	115	jae .Lbad_put_user
95		−SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
96		− ENDBR
97	116	ASM_STAC
98		−4: mov %_ASM_AX,(%_ASM_CX)
	117	+7: mov %_ASM_AX,(%_ASM_CX)
99	118	#ifdef CONFIG_X86_32
100		−5: movl %edx,4(%_ASM_CX)
	119	+8: movl %edx,4(%_ASM_CX)
101	120	#endif
102	121	xor %ecx,%ecx
103	122	ASM_CLAC
104	123	RET
105	124	SYM_FUNC_END(__put_user_8)
106	125	EXPORT_SYMBOL(__put_user_8)
	126	+
	127	+SYM_FUNC_START(__put_user_nocheck_8)
	128	+ ENDBR
	129	+ ASM_STAC
	130	+9: mov %_ASM_AX,(%_ASM_CX)
	131	+#ifdef CONFIG_X86_32
	132	+10: movl %edx,4(%_ASM_CX)
	133	+#endif
	134	+ xor %ecx,%ecx
	135	+ ASM_CLAC
	136	+ RET
	137	+SYM_FUNC_END(__put_user_nocheck_8)
107	138	EXPORT_SYMBOL(__put_user_nocheck_8)
108	139
109	140	SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
@@ -117,6 +148,11 @@ SYM_CODE_END(.Lbad_put_user_clac) · 7 unmodified lines
117	148	_ASM_EXTABLE_UA(2b, .Lbad_put_user_clac)
118	149	_ASM_EXTABLE_UA(3b, .Lbad_put_user_clac)
119	150	_ASM_EXTABLE_UA(4b, .Lbad_put_user_clac)
120		−#ifdef CONFIG_X86_32
121	151	_ASM_EXTABLE_UA(5b, .Lbad_put_user_clac)
	152	+ _ASM_EXTABLE_UA(6b, .Lbad_put_user_clac)
	153	+ _ASM_EXTABLE_UA(7b, .Lbad_put_user_clac)
	154	+ _ASM_EXTABLE_UA(9b, .Lbad_put_user_clac)
	155	+#ifdef CONFIG_X86_32
	156	+ _ASM_EXTABLE_UA(8b, .Lbad_put_user_clac)
	157	+ _ASM_EXTABLE_UA(10b, .Lbad_put_user_clac)
122	158	#endif

arch/x86/lib/retpoline.S+99−8

@@ -5,24 +5,27 @@ · 4 unmodified lines
5	5	#include <asm/dwarf2.h>
6	6	#include <asm/cpufeatures.h>
7	7	#include <asm/alternative.h>
	8	+#include <asm/asm-offsets.h>
8	9	#include <asm/export.h>
9	10	#include <asm/nospec-branch.h>
10	11	#include <asm/unwind_hints.h>
	12	+#include <asm/percpu.h>
11	13	#include <asm/frame.h>
12	14
13	15	.section .text.__x86.indirect_thunk
14	16
15		−.macro RETPOLINE reg
	17	+
	18	+.macro POLINE reg
16	19	ANNOTATE_INTRA_FUNCTION_CALL
17	20	call .Ldo_rop_\@
18		−.Lspec_trap_\@:
19		− UNWIND_HINT_EMPTY
20		− pause
21		− lfence
22		− jmp .Lspec_trap_\@
	21	+ int3
23	22	.Ldo_rop_\@:
24	23	mov %\reg, (%_ASM_SP)
25	24	UNWIND_HINT_FUNC
	25	+.endm
	26	+
	27	+.macro RETPOLINE reg
	28	+ POLINE \reg
26	29	RET
27	30	.endm
28	31
@@ -52,7 +55,6 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) · 23 unmodified lines
52	55	*/
53	56
54	57	#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
55		−#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
56	58
57	59	.align RETPOLINE_THUNK_SIZE
58	60	SYM_CODE_START(__x86_indirect_thunk_array)
@@ -64,10 +66,65 @@ SYM_CODE_START(__x86_indirect_thunk_array) · 5 unmodified lines
64	66	.align RETPOLINE_THUNK_SIZE
65	67	SYM_CODE_END(__x86_indirect_thunk_array)
66	68
67		−#define GEN(reg) EXPORT_THUNK(reg)
	69	+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
	70	+#include <asm/GEN-for-each-reg.h>
	71	+#undef GEN
	72	+
	73	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	74	+.macro CALL_THUNK reg
	75	+ .align RETPOLINE_THUNK_SIZE
	76	+
	77	+SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL)
	78	+ UNWIND_HINT_EMPTY
	79	+ ANNOTATE_NOENDBR
	80	+
	81	+ CALL_DEPTH_ACCOUNT
	82	+ POLINE \reg
	83	+ ANNOTATE_UNRET_SAFE
	84	+ ret
	85	+ int3
	86	+.endm
	87	+
	88	+ .align RETPOLINE_THUNK_SIZE
	89	+SYM_CODE_START(__x86_indirect_call_thunk_array)
	90	+
	91	+#define GEN(reg) CALL_THUNK reg
68	92	#include <asm/GEN-for-each-reg.h>
69	93	#undef GEN
70	94
	95	+ .align RETPOLINE_THUNK_SIZE
	96	+SYM_CODE_END(__x86_indirect_call_thunk_array)
	97	+
	98	+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_call_thunk_ ## reg)
	99	+#include <asm/GEN-for-each-reg.h>
	100	+#undef GEN
	101	+
	102	+.macro JUMP_THUNK reg
	103	+ .align RETPOLINE_THUNK_SIZE
	104	+
	105	+SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL)
	106	+ UNWIND_HINT_EMPTY
	107	+ ANNOTATE_NOENDBR
	108	+ POLINE \reg
	109	+ ANNOTATE_UNRET_SAFE
	110	+ ret
	111	+ int3
	112	+.endm
	113	+
	114	+ .align RETPOLINE_THUNK_SIZE
	115	+SYM_CODE_START(__x86_indirect_jump_thunk_array)
	116	+
	117	+#define GEN(reg) JUMP_THUNK reg
	118	+#include <asm/GEN-for-each-reg.h>
	119	+#undef GEN
	120	+
	121	+ .align RETPOLINE_THUNK_SIZE
	122	+SYM_CODE_END(__x86_indirect_jump_thunk_array)
	123	+
	124	+#define GEN(reg) __EXPORT_THUNK(__x86_indirect_jump_thunk_ ## reg)
	125	+#include <asm/GEN-for-each-reg.h>
	126	+#undef GEN
	127	+#endif
71	128	/*
72	129	* This function name is magical and is used by -mfunction-return=thunk-extern
73	130	* for the compiler to generate JMPs to it.
@@ -140,3 +197,37 @@ __EXPORT_THUNK(zen_untrain_ret) · 66 unmodified lines
140	197	EXPORT_SYMBOL(__x86_return_thunk)
141	198
142	199	#endif /* CONFIG_RETHUNK */
	200	+
	201	+#ifdef CONFIG_CALL_DEPTH_TRACKING
	202	+
	203	+ .align 64
	204	+SYM_FUNC_START(__x86_return_skl)
	205	+ ANNOTATE_NOENDBR
	206	+ /*
	207	+ * Keep the hotpath in a 16byte I-fetch for the non-debug
	208	+ * case.
	209	+ */
	210	+ CALL_THUNKS_DEBUG_INC_RETS
	211	+ shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth)
	212	+ jz 1f
	213	+ ANNOTATE_UNRET_SAFE
	214	+ ret
	215	+ int3
	216	+1:
	217	+ CALL_THUNKS_DEBUG_INC_STUFFS
	218	+ .rept 16
	219	+ ANNOTATE_INTRA_FUNCTION_CALL
	220	+ call 2f
	221	+ int3
	222	+2:
	223	+ .endr
	224	+ add $(8*16), %rsp
	225	+
	226	+ CREDIT_CALL_DEPTH
	227	+
	228	+ ANNOTATE_UNRET_SAFE
	229	+ ret
	230	+ int3
	231	+SYM_FUNC_END(__x86_return_skl)
	232	+
	233	+#endif /* CONFIG_CALL_DEPTH_TRACKING */

arch/x86/net/bpf_jit_comp.c+34−11

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/memory.h>
13	13	#include <linux/sort.h>
14	14	#include <asm/extable.h>
	15	+#include <asm/ftrace.h>
15	16	#include <asm/set_memory.h>
16	17	#include <asm/nospec-branch.h>
17	18	#include <asm/text-patching.h>
@@ -340,6 +341,13 @@ static int emit_call(u8 *pprog, void func, void *ip) · 322 unmodified lines
340	341	return emit_patch(pprog, func, ip, 0xE8);
341	342	}
342	343
	344	+static int emit_rsb_call(u8 *pprog, void func, void *ip)
	345	+{
	346	+ OPTIMIZER_HIDE_VAR(func);
	347	+ x86_call_depth_emit_accounting(pprog, func);
	348	+ return emit_patch(pprog, func, ip, 0xE8);
	349	+}
	350	+
343	351	static int emit_jump(u8 *pprog, void func, void *ip)
344	352	{
345	353	return emit_patch(pprog, func, ip, 0xE9);
@@ -417,7 +425,10 @@ static void emit_indirect_jump(u8 *pprog, int reg, u8 ip) · 71 unmodified lines
417	425	EMIT2(0xFF, 0xE0 + reg);
418	426	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
419	427	OPTIMIZER_HIDE_VAR(reg);
420		− emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
	428	+ if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
	429	+ emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);
	430	+ else
	431	+ emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
421	432	} else {
422	433	EMIT2(0xFF, 0xE0 + reg); /* jmp %\reg /
423	434	if (IS_ENABLED(CONFIG_RETPOLINE) \|\| IS_ENABLED(CONFIG_SLS))
@@ -432,7 +443,7 @@ static void emit_return(u8 *pprog, u8 ip) · 8 unmodified lines
432	443	u8 prog = pprog;
433	444
434	445	if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
435		− emit_jump(&prog, &__x86_return_thunk, ip);
	446	+ emit_jump(&prog, x86_return_thunk, ip);
436	447	} else {
437	448	EMIT1(0xC3); /* ret */
438	449	if (IS_ENABLED(CONFIG_SLS))
@@ -1514,19 +1525,26 @@ st: if (is_imm8(insn->off)) · 1075 unmodified lines
1514	1525	break;
1515	1526
1516	1527	/* call */
1517		− case BPF_JMP \| BPF_CALL:
	1528	+ case BPF_JMP \| BPF_CALL: {
	1529	+ int offs;
	1530	+
1518	1531	func = (u8 *) __bpf_call_base + imm32;
1519	1532	if (tail_call_reachable) {
1520	1533	/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
1521	1534	EMIT3_off32(0x48, 0x8B, 0x85,
1522	1535	-round_up(bpf_prog->aux->stack_depth, 8) - 8);
1523		− if (!imm32 \|\| emit_call(&prog, func, image + addrs[i - 1] + 7))
	1536	+ if (!imm32)
1524	1537	return -EINVAL;
	1538	+ offs = 7 + x86_call_depth_emit_accounting(&prog, func);
1525	1539	} else {
1526		− if (!imm32 \|\| emit_call(&prog, func, image + addrs[i - 1]))
	1540	+ if (!imm32)
1527	1541	return -EINVAL;
	1542	+ offs = x86_call_depth_emit_accounting(&prog, func);
1528	1543	}
	1544	+ if (emit_call(&prog, func, image + addrs[i - 1] + offs))
	1545	+ return -EINVAL;
1529	1546	break;
	1547	+ }
1530	1548
1531	1549	case BPF_JMP \| BPF_TAIL_CALL:
1532	1550	if (imm32)
@@ -1917,7 +1935,7 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 384 unmodified lines
1917	1935	/* arg2: lea rsi, [rbp - ctx_cookie_off] */
1918	1936	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
1919	1937
1920		− if (emit_call(&prog, bpf_trampoline_enter(p), prog))
	1938	+ if (emit_rsb_call(&prog, bpf_trampoline_enter(p), prog))
1921	1939	return -EINVAL;
1922	1940	/* remember prog start time returned by __bpf_prog_enter */
1923	1941	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
@@ -1938,7 +1956,7 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 14 unmodified lines
1938	1956	(long) p->insnsi >> 32,
1939	1957	(u32) (long) p->insnsi);
1940	1958	/* call JITed bpf program or interpreter */
1941		− if (emit_call(&prog, p->bpf_func, prog))
	1959	+ if (emit_rsb_call(&prog, p->bpf_func, prog))
1942	1960	return -EINVAL;
1943	1961
1944	1962	/*
@@ -1962,7 +1980,7 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 17 unmodified lines
1962	1980	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1963	1981	/* arg3: lea rdx, [rbp - run_ctx_off] */
1964	1982	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
1965		− if (emit_call(&prog, bpf_trampoline_exit(p), prog))
	1983	+ if (emit_rsb_call(&prog, bpf_trampoline_exit(p), prog))
1966	1984	return -EINVAL;
1967	1985
1968	1986	*pprog = prog;
@@ -2184,6 +2202,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 215 unmodified lines
2184	2202	prog = image;
2185	2203
2186	2204	EMIT_ENDBR();
	2205	+ /*
	2206	+ * This is the direct-call trampoline, as such it needs accounting
	2207	+ * for the __fentry__ call.
	2208	+ */
	2209	+ x86_call_depth_emit_accounting(&prog, NULL);
2187	2210	EMIT1(0x55); /* push rbp */
2188	2211	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
2189	2212	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
@@ -2210,7 +2233,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 20 unmodified lines
2210	2233	if (flags & BPF_TRAMP_F_CALL_ORIG) {
2211	2234	/* arg1: mov rdi, im */
2212	2235	emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
2213		− if (emit_call(&prog, __bpf_tramp_enter, prog)) {
	2236	+ if (emit_rsb_call(&prog, __bpf_tramp_enter, prog)) {
2214	2237	ret = -EINVAL;
2215	2238	goto cleanup;
2216	2239	}
@@ -2242,7 +2265,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 25 unmodified lines
2242	2265	EMIT2(0xff, 0xd0); /* call rax /
2243	2266	} else {
2244	2267	/* call original function */
2245		− if (emit_call(&prog, orig_call, prog)) {
	2268	+ if (emit_rsb_call(&prog, orig_call, prog)) {
2246	2269	ret = -EINVAL;
2247	2270	goto cleanup;
2248	2271	}
@@ -2286,7 +2309,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, void *i · 37 unmodified lines
2286	2309	im->ip_epilogue = prog;
2287	2310	/* arg1: mov rdi, im */
2288	2311	emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
2289		− if (emit_call(&prog, __bpf_tramp_exit, prog)) {
	2312	+ if (emit_rsb_call(&prog, __bpf_tramp_exit, prog)) {
2290	2313	ret = -EINVAL;
2291	2314	goto cleanup;
2292	2315	}

arch/x86/power/hibernate.c+1−1

@@ -159,7 +159,7 @@ int relocate_restore_code(void) · 158 unmodified lines
159	159	if (!relocated_restore_code)
160	160	return -ENOMEM;
161	161
162		− memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);
	162	+ __memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);
163	163
164	164	/* Make the page containing the relocated code executable */
165	165	pgd = (pgd_t *)__va(read_cr3_pa()) +

arch/x86/xen/enlighten_pv.c+1−1

@@ -1210,7 +1210,7 @@ static void __init xen_setup_gdt(int cpu) · 1209 unmodified lines
1210	1210	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot;
1211	1211	pv_ops.cpu.load_gdt = xen_load_gdt_boot;
1212	1212
1213		− switch_to_new_gdt(cpu);
	1213	+ switch_gdt_and_percpu_base(cpu);
1214	1214
1215	1215	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry;
1216	1216	pv_ops.cpu.load_gdt = xen_load_gdt;

include/asm-generic/vmlinux.lds.h+2−2

@@ -81,8 +81,8 @@ · 80 unmodified lines
81	81	#define RO_EXCEPTION_TABLE
82	82	#endif
83	83
84		−/* Align . to a 8 byte boundary equals to maximum function alignment. */
85		−#define ALIGN_FUNCTION() . = ALIGN(8)
	84	+/* Align . function alignment. */
	85	+#define ALIGN_FUNCTION() . = ALIGN(CONFIG_FUNCTION_ALIGNMENT)
86	86
87	87	/*
88	88	* LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which

include/linux/linkage.h+2−2

@@ -69,8 +69,8 @@ · 68 unmodified lines
69	69	#endif
70	70
71	71	#ifndef __ALIGN
72		−#define __ALIGN .align 4,0x90
73		−#define __ALIGN_STR ".align 4,0x90"
	72	+#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT
	73	+#define __ALIGN_STR __stringify(__ALIGN)
74	74	#endif
75	75
76	76	#ifdef __ASSEMBLY__

include/linux/static_call.h+2−0

@@ -162,6 +162,8 @@ extern void arch_static_call_transform(void site, void tramp, void *func, bool · 161 unmodified lines
162	162
163	163	extern int __init static_call_init(void);
164	164
	165	+extern void static_call_force_reinit(void);
	166	+
165	167	struct static_call_mod {
166	168	struct static_call_mod *next;
167	169	struct module mod; / for vmlinux, mod == NULL */

kernel/static_call_inline.c+18−5

@@ -15,7 +15,18 @@ extern struct static_call_site __start_static_call_sites[], · 14 unmodified lines
15	15	extern struct static_call_tramp_key __start_static_call_tramp_key[],
16	16	__stop_static_call_tramp_key[];
17	17
18		−static bool static_call_initialized;
	18	+static int static_call_initialized;
	19	+
	20	+/*
	21	+ * Must be called before early_initcall() to be effective.
	22	+ */
	23	+void static_call_force_reinit(void)
	24	+{
	25	+ if (WARN_ON_ONCE(!static_call_initialized))
	26	+ return;
	27	+
	28	+ static_call_initialized++;
	29	+}
19	30
20	31	/* mutex to protect key modules/sites */
21	32	static DEFINE_MUTEX(static_call_mutex);
@@ -475,7 +486,8 @@ int __init static_call_init(void) · 453 unmodified lines
475	486	{
476	487	int ret;
477	488
478		− if (static_call_initialized)
	489	+ /* See static_call_force_reinit(). */
	490	+ if (static_call_initialized == 1)
479	491	return 0;
480	492
481	493	cpus_read_lock();
@@ -490,11 +502,12 @@ int __init static_call_init(void) · 8 unmodified lines
490	502	BUG();
491	503	}
492	504
493		− static_call_initialized = true;
494		−
495	505	#ifdef CONFIG_MODULES
496		− register_module_notifier(&static_call_module_nb);
	506	+ if (!static_call_initialized)
	507	+ register_module_notifier(&static_call_module_nb);
497	508	#endif
	509	+
	510	+ static_call_initialized = 1;
498	511	return 0;
499	512	}
500	513	early_initcall(static_call_init);

kernel/trace/trace_selftest.c+8−1

@@ -785,7 +785,14 @@ static struct fgraph_ops fgraph_ops __initdata = { · 784 unmodified lines
785	785	};
786	786
787	787	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
788		−noinline __noclone static void trace_direct_tramp(void) { }
	788	+#ifndef CALL_DEPTH_ACCOUNT
	789	+#define CALL_DEPTH_ACCOUNT ""
	790	+#endif
	791	+
	792	+noinline __noclone static void trace_direct_tramp(void)
	793	+{
	794	+ asm(CALL_DEPTH_ACCOUNT);
	795	+}
789	796	#endif
790	797
791	798	/*

lib/Kconfig.debug+1−0

@@ -469,6 +469,7 @@ config SECTION_MISMATCH_WARN_ONLY · 468 unmodified lines
469	469	config DEBUG_FORCE_FUNCTION_ALIGN_64B
470	470	bool "Force all function address 64B aligned"
471	471	depends on EXPERT && (X86_64 \|\| ARM64 \|\| PPC32 \|\| PPC64 \|\| ARC)
	472	+ select FUNCTION_ALIGNMENT_64B
472	473	help
473	474	There are cases that a commit from one domain changes the function
474	475	address alignment of other domains, and cause magic performance

samples/ftrace/ftrace-direct-modify.c+3−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/kthread.h>
4	4	#include <linux/ftrace.h>
5	5	#include <asm/asm-offsets.h>
	6	+#include <asm/nospec-branch.h>
6	7
7	8	extern void my_direct_func1(void);
8	9	extern void my_direct_func2(void);
@@ -34,6 +35,7 @@ asm ( · 25 unmodified lines
34	35	ASM_ENDBR
35	36	" pushq %rbp\n"
36	37	" movq %rsp, %rbp\n"
	38	+ CALL_DEPTH_ACCOUNT
37	39	" call my_direct_func1\n"
38	40	" leave\n"
39	41	" .size my_tramp1, .-my_tramp1\n"
@@ -45,6 +47,7 @@ asm ( · 5 unmodified lines
45	47	ASM_ENDBR
46	48	" pushq %rbp\n"
47	49	" movq %rsp, %rbp\n"
	50	+ CALL_DEPTH_ACCOUNT
48	51	" call my_direct_func2\n"
49	52	" leave\n"
50	53	ASM_RET

samples/ftrace/ftrace-direct-multi-modify.c+3−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/kthread.h>
4	4	#include <linux/ftrace.h>
5	5	#include <asm/asm-offsets.h>
	6	+#include <asm/nospec-branch.h>
6	7
7	8	extern void my_direct_func1(unsigned long ip);
8	9	extern void my_direct_func2(unsigned long ip);
@@ -32,6 +33,7 @@ asm ( · 23 unmodified lines
32	33	ASM_ENDBR
33	34	" pushq %rbp\n"
34	35	" movq %rsp, %rbp\n"
	36	+ CALL_DEPTH_ACCOUNT
35	37	" pushq %rdi\n"
36	38	" movq 8(%rbp), %rdi\n"
37	39	" call my_direct_func1\n"
@@ -46,6 +48,7 @@ asm ( · 8 unmodified lines
46	48	ASM_ENDBR
47	49	" pushq %rbp\n"
48	50	" movq %rsp, %rbp\n"
	51	+ CALL_DEPTH_ACCOUNT
49	52	" pushq %rdi\n"
50	53	" movq 8(%rbp), %rdi\n"
51	54	" call my_direct_func2\n"

samples/ftrace/ftrace-direct-multi.c+2−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <linux/ftrace.h>
6	6	#include <linux/sched/stat.h>
7	7	#include <asm/asm-offsets.h>
	8	+#include <asm/nospec-branch.h>
8	9
9	10	extern void my_direct_func(unsigned long ip);
10	11
@@ -27,6 +28,7 @@ asm ( · 16 unmodified lines
27	28	ASM_ENDBR
28	29	" pushq %rbp\n"
29	30	" movq %rsp, %rbp\n"
	31	+ CALL_DEPTH_ACCOUNT
30	32	" pushq %rdi\n"
31	33	" movq 8(%rbp), %rdi\n"
32	34	" call my_direct_func\n"

samples/ftrace/ftrace-direct-too.c+2−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	#include <linux/mm.h> /* for handle_mm_fault() */
5	5	#include <linux/ftrace.h>
6	6	#include <asm/asm-offsets.h>
	7	+#include <asm/nospec-branch.h>
7	8
8	9	extern void my_direct_func(struct vm_area_struct *vma,
9	10	unsigned long address, unsigned int flags);
@@ -29,6 +30,7 @@ asm ( · 19 unmodified lines
29	30	ASM_ENDBR
30	31	" pushq %rbp\n"
31	32	" movq %rsp, %rbp\n"
	33	+ CALL_DEPTH_ACCOUNT
32	34	" pushq %rdi\n"
33	35	" pushq %rsi\n"
34	36	" pushq %rdx\n"

samples/ftrace/ftrace-direct.c+2−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4	#include <linux/sched.h> /* for wake_up_process() */
5	5	#include <linux/ftrace.h>
6	6	#include <asm/asm-offsets.h>
	7	+#include <asm/nospec-branch.h>
7	8
8	9	extern void my_direct_func(struct task_struct *p);
9	10
@@ -26,6 +27,7 @@ asm ( · 16 unmodified lines
26	27	ASM_ENDBR
27	28	" pushq %rbp\n"
28	29	" movq %rsp, %rbp\n"
	30	+ CALL_DEPTH_ACCOUNT
29	31	" pushq %rdi\n"
30	32	" call my_direct_func\n"
31	33	" popq %rdi\n"

scripts/Makefile.lib+3−0

@@ -254,7 +254,9 @@ objtool := $(objtree)/tools/objtool/objtool · 253 unmodified lines
254	254
255	255	objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK) += --hacks=jump_label
256	256	objtool-args-$(CONFIG_HAVE_NOINSTR_HACK) += --hacks=noinstr
	257	+objtool-args-$(CONFIG_CALL_DEPTH_TRACKING) += --hacks=skylake
257	258	objtool-args-$(CONFIG_X86_KERNEL_IBT) += --ibt
	259	+objtool-args-$(CONFIG_FINEIBT) += --cfi
258	260	objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL) += --mcount
259	261	objtool-args-$(CONFIG_UNWINDER_ORC) += --orc
260	262	objtool-args-$(CONFIG_RETPOLINE) += --retpoline
@@ -264,6 +266,7 @@ objtool-args-$(CONFIG_STACK_VALIDATION) += --stackval · 3 unmodified lines
264	266	objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE) += --static-call
265	267	objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess
266	268	objtool-args-$(CONFIG_GCOV_KERNEL) += --no-unreachable
	269	+objtool-args-$(CONFIG_PREFIX_SYMBOLS) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
267	270
268	271	objtool-args = $(objtool-args-y) \
269	272	$(if $(delay-objtool), --link) \

tools/include/linux/interval_tree_generic.h+187−0

@@ -0,0 +1,187 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-or-later */
	2	+/*
	3	+ Interval Trees
	4	+ (C) 2012 Michel Lespinasse <walken@google.com>
	5	+
	6	+
	7	+ include/linux/interval_tree_generic.h
	8	+*/
	9	+
	10	+#include <linux/rbtree_augmented.h>
	11	+
	12	+/*
	13	+ * Template for implementing interval trees
	14	+ *
	15	+ * ITSTRUCT: struct type of the interval tree nodes
	16	+ * ITRB: name of struct rb_node field within ITSTRUCT
	17	+ * ITTYPE: type of the interval endpoints
	18	+ * ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree
	19	+ * ITSTART(n): start endpoint of ITSTRUCT node n
	20	+ * ITLAST(n): last endpoint of ITSTRUCT node n
	21	+ * ITSTATIC: 'static' or empty
	22	+ * ITPREFIX: prefix to use for the inline tree definitions
	23	+ *
	24	+ * Note - before using this, please consider if generic version
	25	+ * (interval_tree.h) would work for you...
	26	+ */
	27	+
	28	+#define INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \
	29	+ ITSTART, ITLAST, ITSTATIC, ITPREFIX) \
	30	+ \
	31	+/* Callbacks for augmented rbtree insert and remove */ \
	32	+ \
	33	+RB_DECLARE_CALLBACKS_MAX(static, ITPREFIX ## _augment, \
	34	+ ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, ITLAST) \
	35	+ \
	36	+/* Insert / remove interval nodes from the tree */ \
	37	+ \
	38	+ITSTATIC void ITPREFIX ## _insert(ITSTRUCT *node, \
	39	+ struct rb_root_cached *root) \
	40	+{ \
	41	+ struct rb_node *link = &root->rb_root.rb_node, rb_parent = NULL; \
	42	+ ITTYPE start = ITSTART(node), last = ITLAST(node); \
	43	+ ITSTRUCT *parent; \
	44	+ bool leftmost = true; \
	45	+ \
	46	+ while (*link) { \
	47	+ rb_parent = *link; \
	48	+ parent = rb_entry(rb_parent, ITSTRUCT, ITRB); \
	49	+ if (parent->ITSUBTREE < last) \
	50	+ parent->ITSUBTREE = last; \
	51	+ if (start < ITSTART(parent)) \
	52	+ link = &parent->ITRB.rb_left; \
	53	+ else { \
	54	+ link = &parent->ITRB.rb_right; \
	55	+ leftmost = false; \
	56	+ } \
	57	+ } \
	58	+ \
	59	+ node->ITSUBTREE = last; \
	60	+ rb_link_node(&node->ITRB, rb_parent, link); \
	61	+ rb_insert_augmented_cached(&node->ITRB, root, \
	62	+ leftmost, &ITPREFIX ## _augment); \
	63	+} \
	64	+ \
	65	+ITSTATIC void ITPREFIX ## _remove(ITSTRUCT *node, \
	66	+ struct rb_root_cached *root) \
	67	+{ \
	68	+ rb_erase_augmented_cached(&node->ITRB, root, &ITPREFIX ## _augment); \
	69	+} \
	70	+ \
	71	+/* \
	72	+ * Iterate over intervals intersecting [start;last] \
	73	+ * \
	74	+ * Note that a node's interval intersects [start;last] iff: \
	75	+ * Cond1: ITSTART(node) <= last \
	76	+ * and \
	77	+ * Cond2: start <= ITLAST(node) \
	78	+ */ \
	79	+ \
	80	+static ITSTRUCT * \
	81	+ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
	82	+{ \
	83	+ while (true) { \
	84	+ /* \
	85	+ * Loop invariant: start <= node->ITSUBTREE \
	86	+ * (Cond2 is satisfied by one of the subtree nodes) \
	87	+ */ \
	88	+ if (node->ITRB.rb_left) { \
	89	+ ITSTRUCT *left = rb_entry(node->ITRB.rb_left, \
	90	+ ITSTRUCT, ITRB); \
	91	+ if (start <= left->ITSUBTREE) { \
	92	+ /* \
	93	+ * Some nodes in left subtree satisfy Cond2. \
	94	+ * Iterate to find the leftmost such node N. \
	95	+ * If it also satisfies Cond1, that's the \
	96	+ * match we are looking for. Otherwise, there \
	97	+ * is no matching interval as nodes to the \
	98	+ * right of N can't satisfy Cond1 either. \
	99	+ */ \
	100	+ node = left; \
	101	+ continue; \
	102	+ } \
	103	+ } \
	104	+ if (ITSTART(node) <= last) { /* Cond1 */ \
	105	+ if (start <= ITLAST(node)) /* Cond2 */ \
	106	+ return node; /* node is leftmost match */ \
	107	+ if (node->ITRB.rb_right) { \
	108	+ node = rb_entry(node->ITRB.rb_right, \
	109	+ ITSTRUCT, ITRB); \
	110	+ if (start <= node->ITSUBTREE) \
	111	+ continue; \
	112	+ } \
	113	+ } \
	114	+ return NULL; /* No match */ \
	115	+ } \
	116	+} \
	117	+ \
	118	+ITSTATIC ITSTRUCT * \
	119	+ITPREFIX ## _iter_first(struct rb_root_cached *root, \
	120	+ ITTYPE start, ITTYPE last) \
	121	+{ \
	122	+ ITSTRUCT node, leftmost; \
	123	+ \
	124	+ if (!root->rb_root.rb_node) \
	125	+ return NULL; \
	126	+ \
	127	+ /* \
	128	+ * Fastpath range intersection/overlap between A: [a0, a1] and \
	129	+ * B: [b0, b1] is given by: \
	130	+ * \
	131	+ * a0 <= b1 && b0 <= a1 \
	132	+ * \
	133	+ * ... where A holds the lock range and B holds the smallest \
	134	+ * 'start' and largest 'last' in the tree. For the later, we \
	135	+ * rely on the root node, which by augmented interval tree \
	136	+ * property, holds the largest value in its last-in-subtree. \
	137	+ * This allows mitigating some of the tree walk overhead for \
	138	+ * for non-intersecting ranges, maintained and consulted in O(1). \
	139	+ */ \
	140	+ node = rb_entry(root->rb_root.rb_node, ITSTRUCT, ITRB); \
	141	+ if (node->ITSUBTREE < start) \
	142	+ return NULL; \
	143	+ \
	144	+ leftmost = rb_entry(root->rb_leftmost, ITSTRUCT, ITRB); \
	145	+ if (ITSTART(leftmost) > last) \
	146	+ return NULL; \
	147	+ \
	148	+ return ITPREFIX ## _subtree_search(node, start, last); \
	149	+} \
	150	+ \
	151	+ITSTATIC ITSTRUCT * \
	152	+ITPREFIX ## _iter_next(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
	153	+{ \
	154	+ struct rb_node rb = node->ITRB.rb_right, prev; \
	155	+ \
	156	+ while (true) { \
	157	+ /* \
	158	+ * Loop invariants: \
	159	+ * Cond1: ITSTART(node) <= last \
	160	+ * rb == node->ITRB.rb_right \
	161	+ * \
	162	+ * First, search right subtree if suitable \
	163	+ */ \
	164	+ if (rb) { \
	165	+ ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB); \
	166	+ if (start <= right->ITSUBTREE) \
	167	+ return ITPREFIX ## _subtree_search(right, \
	168	+ start, last); \
	169	+ } \
	170	+ \
	171	+ /* Move up the tree until we come from a node's left child */ \
	172	+ do { \
	173	+ rb = rb_parent(&node->ITRB); \
	174	+ if (!rb) \
	175	+ return NULL; \
	176	+ prev = &node->ITRB; \
	177	+ node = rb_entry(rb, ITSTRUCT, ITRB); \
	178	+ rb = node->ITRB.rb_right; \
	179	+ } while (prev == rb); \
	180	+ \
	181	+ /* Check if the node intersects [start;last] */ \
	182	+ if (last < ITSTART(node)) /* !Cond1 */ \
	183	+ return NULL; \
	184	+ else if (start <= ITLAST(node)) /* Cond2 */ \
	185	+ return node; \
	186	+ } \
	187	+}

tools/objtool/arch/x86/decode.c+24−0

@@ -73,6 +73,30 @@ unsigned long arch_jump_destination(struct instruction *insn) · 72 unmodified lines
73	73	return insn->offset + insn->len + insn->immediate;
74	74	}
75	75
	76	+bool arch_pc_relative_reloc(struct reloc *reloc)
	77	+{
	78	+ /*
	79	+ * All relocation types where P (the address of the target)
	80	+ * is included in the computation.
	81	+ */
	82	+ switch (reloc->type) {
	83	+ case R_X86_64_PC8:
	84	+ case R_X86_64_PC16:
	85	+ case R_X86_64_PC32:
	86	+ case R_X86_64_PC64:
	87	+
	88	+ case R_X86_64_PLT32:
	89	+ case R_X86_64_GOTPC32:
	90	+ case R_X86_64_GOTPCREL:
	91	+ return true;
	92	+
	93	+ default:
	94	+ break;
	95	+ }
	96	+
	97	+ return false;
	98	+}
	99	+
76	100	#define ADD_OP(op) \
77	101	if (!(op = calloc(1, sizeof(*op)))) \
78	102	return -1; \

tools/objtool/builtin-check.c+8−1

@@ -57,12 +57,17 @@ static int parse_hacks(const struct option opt, const char str, int unset) · 56 unmodified lines
57	57	found = true;
58	58	}
59	59
	60	+ if (!str \|\| strstr(str, "skylake")) {
	61	+ opts.hack_skylake = true;
	62	+ found = true;
	63	+ }
	64	+
60	65	return found ? 0 : -1;
61	66	}
62	67
63	68	const struct option check_options[] = {
64	69	OPT_GROUP("Actions:"),
65		− OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr", "patch toolchain bugs/limitations", parse_hacks),
	70	+ OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks),
66	71	OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"),
67	72	OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"),
68	73	OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"),
@@ -70,10 +75,12 @@ const struct option check_options[] = { · 1 unmodified line
70	75	OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"),
71	76	OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"),
72	77	OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"),
	78	+ OPT_INTEGER(0, "prefix", &opts.prefix, "generate prefix symbols"),
73	79	OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"),
74	80	OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"),
75	81	OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"),
76	82	OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"),
	83	+ OPT_BOOLEAN(0 , "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"),
77	84	OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump),
78	85
79	86	OPT_GROUP("Options:"),

tools/objtool/check.c+310−81

@@ -62,12 +62,12 @@ static struct instruction next_insn_same_func(struct objtool_file file, · 61 unmodified lines
62	62	struct instruction *insn)
63	63	{
64	64	struct instruction *next = list_next_entry(insn, list);
65		− struct symbol *func = insn->func;
	65	+ struct symbol *func = insn_func(insn);
66	66
67	67	if (!func)
68	68	return NULL;
69	69
70		− if (&next->list != &file->insn_list && next->func == func)
	70	+ if (&next->list != &file->insn_list && insn_func(next) == func)
71	71	return next;
72	72
73	73	/* Check if we're already in the subfunction: */
@@ -83,7 +83,7 @@ static struct instruction prev_insn_same_sym(struct objtool_file file, · 9 unmodified lines
83	83	{
84	84	struct instruction *prev = list_prev_entry(insn, list);
85	85
86		− if (&prev->list != &file->insn_list && prev->func == insn->func)
	86	+ if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
87	87	return prev;
88	88
89	89	return NULL;
@@ -129,16 +129,13 @@ static bool is_jump_table_jump(struct instruction *insn) · 39 unmodified lines
129	129	static bool is_sibling_call(struct instruction *insn)
130	130	{
131	131	/*
132		− * Assume only ELF functions can make sibling calls. This ensures
133		− * sibling call detection consistency between vmlinux.o and individual
134		− * objects.
	132	+ * Assume only STT_FUNC calls have jump-tables.
135	133	*/
136		− if (!insn->func)
137		− return false;
138		−
139		− /* An indirect jump is either a sibling call or a jump to a table. */
140		− if (insn->type == INSN_JUMP_DYNAMIC)
141		− return !is_jump_table_jump(insn);
	134	+ if (insn_func(insn)) {
	135	+ /* An indirect jump is either a sibling call or a jump to a table. */
	136	+ if (insn->type == INSN_JUMP_DYNAMIC)
	137	+ return !is_jump_table_jump(insn);
	138	+ }
142	139
143	140	/* add_jump_destinations() sets insn->call_dest for sibling calls. */
144	141	return (is_static_jump(insn) && insn->call_dest);
@@ -207,7 +204,7 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 62 unmodified lines
207	204	return false;
208	205
209	206	insn = find_insn(file, func->sec, func->offset);
210		− if (!insn->func)
	207	+ if (!insn_func(insn))
211	208	return false;
212	209
213	210	func_for_each_insn(file, func, insn) {
@@ -243,7 +240,7 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 29 unmodified lines
243	240	return false;
244	241	}
245	242
246		− return __dead_end_function(file, dest->func, recursion+1);
	243	+ return __dead_end_function(file, insn_func(dest), recursion+1);
247	244	}
248	245	}
249	246
@@ -382,6 +379,15 @@ static int decode_instructions(struct objtool_file *file) · 132 unmodified lines
382	379	!strncmp(sec->name, ".text.__x86.", 12))
383	380	sec->noinstr = true;
384	381
	382	+ /*
	383	+ * .init.text code is ran before userspace and thus doesn't
	384	+ * strictly need retpolines, except for modules which are
	385	+ * loaded late, they very much do need retpoline in their
	386	+ * .init.text
	387	+ */
	388	+ if (!strcmp(sec->name, ".init.text") && !opts.module)
	389	+ sec->init = true;
	390	+
385	391	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
386	392	insn = malloc(sizeof(*insn));
387	393	if (!insn) {
@@ -418,7 +424,10 @@ static int decode_instructions(struct objtool_file *file) · 30 unmodified lines
418	424	}
419	425
420	426	list_for_each_entry(func, &sec->symbol_list, list) {
421		− if (func->type != STT_FUNC \|\| func->alias != func)
	427	+ if (func->type != STT_NOTYPE && func->type != STT_FUNC)
	428	+ continue;
	429	+
	430	+ if (func->return_thunk \|\| func->alias != func)
422	431	continue;
423	432
424	433	if (!find_insn(file, sec, func->offset)) {
@@ -428,9 +437,11 @@ static int decode_instructions(struct objtool_file *file) · 3 unmodified lines
428	437	}
429	438
430	439	sym_for_each_insn(file, func, insn) {
431		− insn->func = func;
432		− if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
433		− if (insn->offset == insn->func->offset) {
	440	+ insn->sym = func;
	441	+ if (func->type == STT_FUNC &&
	442	+ insn->type == INSN_ENDBR &&
	443	+ list_empty(&insn->call_node)) {
	444	+ if (insn->offset == func->offset) {
434	445	list_add_tail(&insn->call_node, &file->endbr_list);
435	446	file->nr_endbr++;
436	447	} else {
@@ -850,6 +861,68 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) · 413 unmodified lines
850	861	return 0;
851	862	}
852	863
	864	+static int create_cfi_sections(struct objtool_file *file)
	865	+{
	866	+ struct section sec, s;
	867	+ struct symbol *sym;
	868	+ unsigned int *loc;
	869	+ int idx;
	870	+
	871	+ sec = find_section_by_name(file->elf, ".cfi_sites");
	872	+ if (sec) {
	873	+ INIT_LIST_HEAD(&file->call_list);
	874	+ WARN("file already has .cfi_sites section, skipping");
	875	+ return 0;
	876	+ }
	877	+
	878	+ idx = 0;
	879	+ for_each_sec(file, s) {
	880	+ if (!s->text)
	881	+ continue;
	882	+
	883	+ list_for_each_entry(sym, &s->symbol_list, list) {
	884	+ if (sym->type != STT_FUNC)
	885	+ continue;
	886	+
	887	+ if (strncmp(sym->name, "__cfi_", 6))
	888	+ continue;
	889	+
	890	+ idx++;
	891	+ }
	892	+ }
	893	+
	894	+ sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
	895	+ if (!sec)
	896	+ return -1;
	897	+
	898	+ idx = 0;
	899	+ for_each_sec(file, s) {
	900	+ if (!s->text)
	901	+ continue;
	902	+
	903	+ list_for_each_entry(sym, &s->symbol_list, list) {
	904	+ if (sym->type != STT_FUNC)
	905	+ continue;
	906	+
	907	+ if (strncmp(sym->name, "__cfi_", 6))
	908	+ continue;
	909	+
	910	+ loc = (unsigned int *)sec->data->d_buf + idx;
	911	+ memset(loc, 0, sizeof(unsigned int));
	912	+
	913	+ if (elf_add_reloc_to_insn(file->elf, sec,
	914	+ idx * sizeof(unsigned int),
	915	+ R_X86_64_PC32,
	916	+ s, sym->offset))
	917	+ return -1;
	918	+
	919	+ idx++;
	920	+ }
	921	+ }
	922	+
	923	+ return 0;
	924	+}
	925	+
853	926	static int create_mcount_loc_sections(struct objtool_file *file)
854	927	{
855	928	struct section *sec;
@@ -893,6 +966,49 @@ static int create_mcount_loc_sections(struct objtool_file *file) · 37 unmodified lines
893	966	return 0;
894	967	}
895	968
	969	+static int create_direct_call_sections(struct objtool_file *file)
	970	+{
	971	+ struct instruction *insn;
	972	+ struct section *sec;
	973	+ unsigned int *loc;
	974	+ int idx;
	975	+
	976	+ sec = find_section_by_name(file->elf, ".call_sites");
	977	+ if (sec) {
	978	+ INIT_LIST_HEAD(&file->call_list);
	979	+ WARN("file already has .call_sites section, skipping");
	980	+ return 0;
	981	+ }
	982	+
	983	+ if (list_empty(&file->call_list))
	984	+ return 0;
	985	+
	986	+ idx = 0;
	987	+ list_for_each_entry(insn, &file->call_list, call_node)
	988	+ idx++;
	989	+
	990	+ sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
	991	+ if (!sec)
	992	+ return -1;
	993	+
	994	+ idx = 0;
	995	+ list_for_each_entry(insn, &file->call_list, call_node) {
	996	+
	997	+ loc = (unsigned int *)sec->data->d_buf + idx;
	998	+ memset(loc, 0, sizeof(unsigned int));
	999	+
	1000	+ if (elf_add_reloc_to_insn(file->elf, sec,
	1001	+ idx * sizeof(unsigned int),
	1002	+ R_X86_64_PC32,
	1003	+ insn->sec, insn->offset))
	1004	+ return -1;
	1005	+
	1006	+ idx++;
	1007	+ }
	1008	+
	1009	+ return 0;
	1010	+}
	1011	+
896	1012	/*
897	1013	* Warnings shouldn't be reported for ignored functions.
898	1014	*/
@@ -1280,6 +1396,9 @@ static void annotate_call_site(struct objtool_file *file, · 381 unmodified lines
1280	1396	return;
1281	1397	}
1282	1398
	1399	+ if (insn->type == INSN_CALL && !insn->sec->init)
	1400	+ list_add_tail(&insn->call_node, &file->call_list);
	1401	+
1283	1402	if (!sibling && dead_end_function(file, sym))
1284	1403	insn->dead_end = true;
1285	1404	}
@@ -1350,27 +1469,50 @@ static void add_return_call(struct objtool_file file, struct instruction insn, · 64 unmodified lines
1350	1469	list_add_tail(&insn->call_node, &file->return_thunk_list);
1351	1470	}
1352	1471
1353		−static bool same_function(struct instruction insn1, struct instruction insn2)
	1472	+static bool is_first_func_insn(struct objtool_file *file,
	1473	+ struct instruction insn, struct symbol sym)
1354	1474	{
1355		− return insn1->func->pfunc == insn2->func->pfunc;
1356		−}
1357		−
1358		−static bool is_first_func_insn(struct objtool_file file, struct instruction insn)
1359		−{
1360		− if (insn->offset == insn->func->offset)
	1475	+ if (insn->offset == sym->offset)
1361	1476	return true;
1362	1477
	1478	+ /* Allow direct CALL/JMP past ENDBR */
1363	1479	if (opts.ibt) {
1364	1480	struct instruction *prev = prev_insn_same_sym(file, insn);
1365	1481
1366	1482	if (prev && prev->type == INSN_ENDBR &&
1367		− insn->offset == insn->func->offset + prev->len)
	1483	+ insn->offset == sym->offset + prev->len)
1368	1484	return true;
1369	1485	}
1370	1486
1371	1487	return false;
1372	1488	}
1373	1489
	1490	+/*
	1491	+ * A sibling call is a tail-call to another symbol -- to differentiate from a
	1492	+ * recursive tail-call which is to the same symbol.
	1493	+ */
	1494	+static bool jump_is_sibling_call(struct objtool_file *file,
	1495	+ struct instruction from, struct instruction to)
	1496	+{
	1497	+ struct symbol *fs = from->sym;
	1498	+ struct symbol *ts = to->sym;
	1499	+
	1500	+ /* Not a sibling call if from/to a symbol hole */
	1501	+ if (!fs \|\| !ts)
	1502	+ return false;
	1503	+
	1504	+ /* Not a sibling call if not targeting the start of a symbol. */
	1505	+ if (!is_first_func_insn(file, to, ts))
	1506	+ return false;
	1507	+
	1508	+ /* Disallow sibling calls into STT_NOTYPE */
	1509	+ if (ts->type == STT_NOTYPE)
	1510	+ return false;
	1511	+
	1512	+ /* Must not be self to be a sibling */
	1513	+ return fs->pfunc != ts->pfunc;
	1514	+}
	1515	+
1374	1516	/*
1375	1517	* Find the destination instructions for all jumps.
1376	1518	*/
@@ -1405,7 +1547,7 @@ static int add_jump_destinations(struct objtool_file *file) · 28 unmodified lines
1405	1547	} else if (reloc->sym->return_thunk) {
1406	1548	add_return_call(file, insn, true);
1407	1549	continue;
1408		− } else if (insn->func) {
	1550	+ } else if (insn_func(insn)) {
1409	1551	/*
1410	1552	* External sibling call or internal sibling call with
1411	1553	* STT_FUNC reloc.
@@ -1447,8 +1589,8 @@ static int add_jump_destinations(struct objtool_file *file) · 35 unmodified lines
1447	1589	/*
1448	1590	* Cross-function jump.
1449	1591	*/
1450		− if (insn->func && jump_dest->func &&
1451		− insn->func != jump_dest->func) {
	1592	+ if (insn_func(insn) && insn_func(jump_dest) &&
	1593	+ insn_func(insn) != insn_func(jump_dest)) {
1452	1594
1453	1595	/*
1454	1596	* For GCC 8+, create parent/child links for any cold
@@ -1465,22 +1607,22 @@ static int add_jump_destinations(struct objtool_file *file) · 10 unmodified lines
1465	1607	* case where the parent function's only reference to a
1466	1608	* subfunction is through a jump table.
1467	1609	*/
1468		− if (!strstr(insn->func->name, ".cold") &&
1469		− strstr(jump_dest->func->name, ".cold")) {
1470		− insn->func->cfunc = jump_dest->func;
1471		− jump_dest->func->pfunc = insn->func;
1472		−
1473		− } else if (!same_function(insn, jump_dest) &&
1474		− is_first_func_insn(file, jump_dest)) {
1475		− /*
1476		− * Internal sibling call without reloc or with
1477		− * STT_SECTION reloc.
1478		− */
1479		− add_call_dest(file, insn, jump_dest->func, true);
1480		− continue;
	1610	+ if (!strstr(insn_func(insn)->name, ".cold") &&
	1611	+ strstr(insn_func(jump_dest)->name, ".cold")) {
	1612	+ insn_func(insn)->cfunc = insn_func(jump_dest);
	1613	+ insn_func(jump_dest)->pfunc = insn_func(insn);
1481	1614	}
1482	1615	}
1483	1616
	1617	+ if (jump_is_sibling_call(file, insn, jump_dest)) {
	1618	+ /*
	1619	+ * Internal sibling call without reloc or with
	1620	+ * STT_SECTION reloc.
	1621	+ */
	1622	+ add_call_dest(file, insn, insn_func(jump_dest), true);
	1623	+ continue;
	1624	+ }
	1625	+
1484	1626	insn->jump_dest = jump_dest;
1485	1627	}
1486	1628
@@ -1527,7 +1669,7 @@ static int add_call_destinations(struct objtool_file *file) · 40 unmodified lines
1527	1669	return -1;
1528	1670	}
1529	1671
1530		− if (insn->func && insn->call_dest->type != STT_FUNC) {
	1672	+ if (insn_func(insn) && insn->call_dest->type != STT_FUNC) {
1531	1673	WARN_FUNC("unsupported call to non-function",
1532	1674	insn->sec, insn->offset);
1533	1675	return -1;
@@ -1623,7 +1765,7 @@ static int handle_group_alt(struct objtool_file *file, · 89 unmodified lines
1623	1765	nop->offset = special_alt->new_off + special_alt->new_len;
1624	1766	nop->len = special_alt->orig_len - special_alt->new_len;
1625	1767	nop->type = INSN_NOP;
1626		− nop->func = orig_insn->func;
	1768	+ nop->sym = orig_insn->sym;
1627	1769	nop->alt_group = new_alt_group;
1628	1770	nop->ignore = orig_insn->ignore_alts;
1629	1771	}
@@ -1643,7 +1785,7 @@ static int handle_group_alt(struct objtool_file *file, · 13 unmodified lines
1643	1785	last_new_insn = insn;
1644	1786
1645	1787	insn->ignore = orig_insn->ignore_alts;
1646		− insn->func = orig_insn->func;
	1788	+ insn->sym = orig_insn->sym;
1647	1789	insn->alt_group = new_alt_group;
1648	1790
1649	1791	/*
@@ -1655,7 +1797,7 @@ static int handle_group_alt(struct objtool_file *file, · 5 unmodified lines
1655	1797	* accordingly.
1656	1798	*/
1657	1799	alt_reloc = insn_reloc(file, insn);
1658		− if (alt_reloc &&
	1800	+ if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1659	1801	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1660	1802
1661	1803	WARN_FUNC("unsupported relocation in alternatives section",
@@ -1837,7 +1979,7 @@ static int add_jump_table(struct objtool_file file, struct instruction insn, · 175 unmodified lines
1837	1979	struct reloc *reloc = table;
1838	1980	struct instruction *dest_insn;
1839	1981	struct alternative *alt;
1840		− struct symbol *pfunc = insn->func->pfunc;
	1982	+ struct symbol *pfunc = insn_func(insn)->pfunc;
1841	1983	unsigned int prev_offset = 0;
1842	1984
1843	1985	/*
@@ -1864,7 +2006,7 @@ static int add_jump_table(struct objtool_file file, struct instruction insn, · 20 unmodified lines
1864	2006	break;
1865	2007
1866	2008	/* Make sure the destination is in the same function: */
1867		− if (!dest_insn->func \|\| dest_insn->func->pfunc != pfunc)
	2009	+ if (!insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != pfunc)
1868	2010	break;
1869	2011
1870	2012	alt = malloc(sizeof(*alt));
@@ -1904,7 +2046,7 @@ static struct reloc find_jump_table(struct objtool_file file, · 33 unmodified lines
1904	2046	* it.
1905	2047	*/
1906	2048	for (;
1907		− insn && insn->func && insn->func->pfunc == func;
	2049	+ insn && insn_func(insn) && insn_func(insn)->pfunc == func;
1908	2050	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1909	2051
1910	2052	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
@@ -1921,7 +2063,7 @@ static struct reloc find_jump_table(struct objtool_file file, · 10 unmodified lines
1921	2063	if (!table_reloc)
1922	2064	continue;
1923	2065	dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1924		− if (!dest_insn \|\| !dest_insn->func \|\| dest_insn->func->pfunc != func)
	2066	+ if (!dest_insn \|\| !insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != func)
1925	2067	continue;
1926	2068
1927	2069	return table_reloc;
@@ -2370,6 +2512,13 @@ static int decode_sections(struct objtool_file *file) · 442 unmodified lines
2370	2512	if (ret)
2371	2513	return ret;
2372	2514
	2515	+ /*
	2516	+ * Must be before add_{jump_call}_destination.
	2517	+ */
	2518	+ ret = classify_symbols(file);
	2519	+ if (ret)
	2520	+ return ret;
	2521	+
2373	2522	ret = decode_instructions(file);
2374	2523	if (ret)
2375	2524	return ret;
@@ -2388,13 +2537,6 @@ static int decode_sections(struct objtool_file *file) · 12 unmodified lines
2388	2537	if (ret)
2389	2538	return ret;
2390	2539
2391		− /*
2392		− * Must be before add_{jump_call}_destination.
2393		− */
2394		− ret = classify_symbols(file);
2395		− if (ret)
2396		− return ret;
2397		−
2398	2540	/*
2399	2541	* Must be before add_jump_destinations(), which depends on 'func'
2400	2542	* being set for alternatives, to enable proper sibling call detection.
@@ -2603,7 +2745,7 @@ static int update_cfi_state(struct instruction *insn, · 202 unmodified lines
2603	2745
2604	2746	/* stack operations don't make sense with an undefined CFA */
2605	2747	if (cfa->base == CFI_UNDEFINED) {
2606		− if (insn->func) {
	2748	+ if (insn_func(insn)) {
2607	2749	WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2608	2750	return -1;
2609	2751	}
@@ -2949,7 +3091,7 @@ static int update_cfi_state(struct instruction *insn, · 339 unmodified lines
2949	3091	}
2950	3092
2951	3093	/* detect when asm code uses rbp as a scratch register */
2952		− if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
	3094	+ if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
2953	3095	cfa->base != CFI_BP)
2954	3096	cfi->bp_scratch = true;
2955	3097	break;
@@ -3259,7 +3401,7 @@ static int validate_sibling_call(struct objtool_file *file, · 303 unmodified lines
3259	3401	struct instruction *insn,
3260	3402	struct insn_state *state)
3261	3403	{
3262		− if (has_modified_stack_frame(insn, state)) {
	3404	+ if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3263	3405	WARN_FUNC("sibling call from callable instruction with modified stack frame",
3264	3406	insn->sec, insn->offset);
3265	3407	return 1;
@@ -3345,13 +3487,14 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 79 unmodified lines
3345	3487	while (1) {
3346	3488	next_insn = next_insn_to_validate(file, insn);
3347	3489
3348		− if (func && insn->func && func != insn->func->pfunc) {
	3490	+ if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3349	3491	/* Ignore KCFI type preambles, which always fall through */
3350		− if (!strncmp(func->name, "__cfi_", 6))
	3492	+ if (!strncmp(func->name, "__cfi_", 6) \|\|
	3493	+ !strncmp(func->name, "__pfx_", 6))
3351	3494	return 0;
3352	3495
3353	3496	WARN("%s() falls through to next function %s()",
3354		− func->name, insn->func->name);
	3497	+ func->name, insn_func(insn)->name);
3355	3498	return 1;
3356	3499	}
3357	3500
@@ -3593,7 +3736,7 @@ static int validate_unwind_hints(struct objtool_file file, struct section sec) · 235 unmodified lines
3593	3736
3594	3737	while (&insn->list != &file->insn_list && (!sec \|\| insn->sec == sec)) {
3595	3738	if (insn->hint && !insn->visited && !insn->ignore) {
3596		− ret = validate_branch(file, insn->func, insn, state);
	3739	+ ret = validate_branch(file, insn_func(insn), insn, state);
3597	3740	if (ret && opts.backtrace)
3598	3741	BT_FUNC("<=== (hint)", insn);
3599	3742	warnings += ret;
@@ -3758,13 +3901,7 @@ static int validate_retpoline(struct objtool_file *file) · 158 unmodified lines
3758	3901	if (insn->retpoline_safe)
3759	3902	continue;
3760	3903
3761		− /*
3762		− * .init.text code is ran before userspace and thus doesn't
3763		− * strictly need retpolines, except for modules which are
3764		− * loaded late, they very much do need retpoline in their
3765		− * .init.text
3766		− */
3767		− if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
	3904	+ if (insn->sec->init)
3768	3905	continue;
3769	3906
3770	3907	if (insn->type == INSN_RETURN) {
@@ -3822,7 +3959,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 51 unmodified lines
3822	3959	* In this case we'll find a piece of code (whole function) that is not
3823	3960	* covered by a !section symbol. Ignore them.
3824	3961	*/
3825		− if (opts.link && !insn->func) {
	3962	+ if (opts.link && !insn_func(insn)) {
3826	3963	int size = find_symbol_hole_containing(insn->sec, insn->offset);
3827	3964	unsigned long end = insn->offset + size;
3828	3965
@@ -3846,10 +3983,10 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 17 unmodified lines
3846	3983	/*
3847	3984	* If this hole jumps to a .cold function, mark it ignore too.
3848	3985	*/
3849		− if (insn->jump_dest && insn->jump_dest->func &&
3850		− strstr(insn->jump_dest->func->name, ".cold")) {
	3986	+ if (insn->jump_dest && insn_func(insn->jump_dest) &&
	3987	+ strstr(insn_func(insn->jump_dest)->name, ".cold")) {
3851	3988	struct instruction *dest = insn->jump_dest;
3852		− func_for_each_insn(file, dest->func, dest)
	3989	+ func_for_each_insn(file, insn_func(dest), dest)
3853	3990	dest->ignore = true;
3854	3991	}
3855	3992	}
@@ -3857,10 +3994,10 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 1 unmodified line
3857	3994	return false;
3858	3995	}
3859	3996
3860		− if (!insn->func)
	3997	+ if (!insn_func(insn))
3861	3998	return false;
3862	3999
3863		− if (insn->func->static_call_tramp)
	4000	+ if (insn_func(insn)->static_call_tramp)
3864	4001	return true;
3865	4002
3866	4003	/*
@@ -3891,7 +4028,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 24 unmodified lines
3891	4028
3892	4029	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3893	4030	if (insn->jump_dest &&
3894		− insn->jump_dest->func == insn->func) {
	4031	+ insn_func(insn->jump_dest) == insn_func(insn)) {
3895	4032	insn = insn->jump_dest;
3896	4033	continue;
3897	4034	}
@@ -3899,7 +4036,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 1 unmodified line
3899	4036	break;
3900	4037	}
3901	4038
3902		− if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
	4039	+ if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
3903	4040	break;
3904	4041
3905	4042	insn = list_next_entry(insn, list);
@@ -3908,6 +4045,54 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 2 unmodified lines
3908	4045	return false;
3909	4046	}
3910	4047
	4048	+static int add_prefix_symbol(struct objtool_file file, struct symbol func,
	4049	+ struct instruction *insn)
	4050	+{
	4051	+ if (!opts.prefix)
	4052	+ return 0;
	4053	+
	4054	+ for (;;) {
	4055	+ struct instruction *prev = list_prev_entry(insn, list);
	4056	+ u64 offset;
	4057	+
	4058	+ if (&prev->list == &file->insn_list)
	4059	+ break;
	4060	+
	4061	+ if (prev->type != INSN_NOP)
	4062	+ break;
	4063	+
	4064	+ offset = func->offset - prev->offset;
	4065	+ if (offset >= opts.prefix) {
	4066	+ if (offset == opts.prefix) {
	4067	+ /*
	4068	+ * Since the sec->symbol_list is ordered by
	4069	+ * offset (see elf_add_symbol()) the added
	4070	+ * symbol will not be seen by the iteration in
	4071	+ * validate_section().
	4072	+ *
	4073	+ * Hence the lack of list_for_each_entry_safe()
	4074	+ * there.
	4075	+ *
	4076	+ * The direct concequence is that prefix symbols
	4077	+ * don't get visited (because pointless), except
	4078	+ * for the logic in ignore_unreachable_insn()
	4079	+ * that needs the terminating insn to be visited
	4080	+ * otherwise it will report the hole.
	4081	+ *
	4082	+ * Hence mark the first instruction of the
	4083	+ * prefix symbol as visisted.
	4084	+ */
	4085	+ prev->visited \|= VISITED_BRANCH;
	4086	+ elf_create_prefix_symbol(file->elf, func, opts.prefix);
	4087	+ }
	4088	+ break;
	4089	+ }
	4090	+ insn = prev;
	4091	+ }
	4092	+
	4093	+ return 0;
	4094	+}
	4095	+
3911	4096	static int validate_symbol(struct objtool_file file, struct section sec,
3912	4097	struct symbol sym, struct insn_state state)
3913	4098	{
@@ -3926,9 +4111,11 @@ static int validate_symbol(struct objtool_file file, struct section sec, · 12 unmodified lines
3926	4111	if (!insn \|\| insn->ignore \|\| insn->visited)
3927	4112	return 0;
3928	4113
	4114	+ add_prefix_symbol(file, sym, insn);
	4115	+
3929	4116	state->uaccess = sym->uaccess_safe;
3930	4117
3931		− ret = validate_branch(file, insn->func, insn, *state);
	4118	+ ret = validate_branch(file, insn_func(insn), insn, *state);
3932	4119	if (ret && opts.backtrace)
3933	4120	BT_FUNC("<=== (sym)", insn);
3934	4121	return ret;
@@ -3994,6 +4181,24 @@ static void mark_endbr_used(struct instruction *insn) · 59 unmodified lines
3994	4181	list_del_init(&insn->call_node);
3995	4182	}
3996	4183
	4184	+static bool noendbr_range(struct objtool_file file, struct instruction insn)
	4185	+{
	4186	+ struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
	4187	+ struct instruction *first;
	4188	+
	4189	+ if (!sym)
	4190	+ return false;
	4191	+
	4192	+ first = find_insn(file, sym->sec, sym->offset);
	4193	+ if (!first)
	4194	+ return false;
	4195	+
	4196	+ if (first->type != INSN_ENDBR && !first->noendbr)
	4197	+ return false;
	4198	+
	4199	+ return insn->offset == sym->offset + sym->len;
	4200	+}
	4201	+
3997	4202	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)
3998	4203	{
3999	4204	struct instruction *dest;
@@ -4047,7 +4252,7 @@ static int validate_ibt_insn(struct objtool_file file, struct instruction insn · 47 unmodified lines
4047	4252	continue;
4048	4253	}
4049	4254
4050		− if (dest->func && dest->func == insn->func) {
	4255	+ if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
4051	4256	/*
4052	4257	* Anything from->to self is either _THIS_IP_ or
4053	4258	* IRET-to-self.
@@ -4066,9 +4271,19 @@ static int validate_ibt_insn(struct objtool_file file, struct instruction insn · 12 unmodified lines
4066	4271	continue;
4067	4272	}
4068	4273
	4274	+ /*
	4275	+ * Accept anything ANNOTATE_NOENDBR.
	4276	+ */
4069	4277	if (dest->noendbr)
4070	4278	continue;
4071	4279
	4280	+ /*
	4281	+ * Accept if this is the instruction after a symbol
	4282	+ * that is (no)endbr -- typical code-range usage.
	4283	+ */
	4284	+ if (noendbr_range(file, dest))
	4285	+ continue;
	4286	+
4072	4287	WARN_FUNC("relocation to !ENDBR: %s",
4073	4288	insn->sec, insn->offset,
4074	4289	offstr(dest->sec, dest->offset));
@@ -4307,11 +4522,25 @@ int check(struct objtool_file *file) · 232 unmodified lines
4307	4522	warnings += ret;
4308	4523	}
4309	4524
	4525	+ if (opts.cfi) {
	4526	+ ret = create_cfi_sections(file);
	4527	+ if (ret < 0)
	4528	+ goto out;
	4529	+ warnings += ret;
	4530	+ }
	4531	+
4310	4532	if (opts.rethunk) {
4311	4533	ret = create_return_sites_sections(file);
4312	4534	if (ret < 0)
4313	4535	goto out;
4314	4536	warnings += ret;
	4537	+
	4538	+ if (opts.hack_skylake) {
	4539	+ ret = create_direct_call_sections(file);
	4540	+ if (ret < 0)
	4541	+ goto out;
	4542	+ warnings += ret;
	4543	+ }
4315	4544	}
4316	4545
4317	4546	if (opts.mcount) {

tools/objtool/elf.c+199−97

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <string.h>
17	17	#include <unistd.h>
18	18	#include <errno.h>
	19	+#include <linux/interval_tree_generic.h>
19	20	#include <objtool/builtin.h>
20	21
21	22	#include <objtool/elf.h>
@@ -50,38 +51,22 @@ static inline u32 str_hash(const char *str) · 28 unmodified lines
50	51	__elf_table(name); \
51	52	})
52	53
53		−static bool symbol_to_offset(struct rb_node a, const struct rb_node b)
	54	+static inline unsigned long __sym_start(struct symbol *s)
54	55	{
55		− struct symbol *sa = rb_entry(a, struct symbol, node);
56		− struct symbol *sb = rb_entry(b, struct symbol, node);
57		−
58		− if (sa->offset < sb->offset)
59		− return true;
60		− if (sa->offset > sb->offset)
61		− return false;
62		−
63		− if (sa->len < sb->len)
64		− return true;
65		− if (sa->len > sb->len)
66		− return false;
67		−
68		− sa->alias = sb;
69		−
70		− return false;
	56	+ return s->offset;
71	57	}
72	58
73		−static int symbol_by_offset(const void key, const struct rb_node node)
	59	+static inline unsigned long __sym_last(struct symbol *s)
74	60	{
75		− const struct symbol *s = rb_entry(node, struct symbol, node);
76		− const unsigned long *o = key;
	61	+ return s->offset + s->len - 1;
	62	+}
77	63
78		− if (*o < s->offset)
79		− return -1;
80		− if (*o >= s->offset + s->len)
81		− return 1;
	64	+INTERVAL_TREE_DEFINE(struct symbol, node, unsigned long, __subtree_last,
	65	+ __sym_start, __sym_last, static, __sym)
82	66
83		− return 0;
84		−}
	67	+#define __sym_for_each(_iter, _tree, _start, _end) \
	68	+ for (_iter = __sym_iter_first((_tree), (_start), (_end)); \
	69	+ _iter; _iter = __sym_iter_next(_iter, (_start), (_end)))
85	70
86	71	struct symbol_hole {
87	72	unsigned long key;
@@ -147,13 +132,12 @@ static struct symbol find_symbol_by_index(struct elf elf, unsigned int idx) · 59 unmodified lines
147	132
148	133	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset)
149	134	{
150		− struct rb_node *node;
151		−
152		− rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
153		− struct symbol *s = rb_entry(node, struct symbol, node);
	135	+ struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
	136	+ struct symbol *iter;
154	137
155		− if (s->offset == offset && s->type != STT_SECTION)
156		− return s;
	138	+ __sym_for_each(iter, tree, offset, offset) {
	139	+ if (iter->offset == offset && iter->type != STT_SECTION)
	140	+ return iter;
157	141	}
158	142
159	143	return NULL;
@@ -161,13 +145,12 @@ struct symbol find_symbol_by_offset(struct section sec, unsigned long offset) · 1 unmodified line
161	145
162	146	struct symbol find_func_by_offset(struct section sec, unsigned long offset)
163	147	{
164		− struct rb_node *node;
	148	+ struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
	149	+ struct symbol *iter;
165	150
166		− rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
167		− struct symbol *s = rb_entry(node, struct symbol, node);
168		−
169		− if (s->offset == offset && s->type == STT_FUNC)
170		− return s;
	151	+ __sym_for_each(iter, tree, offset, offset) {
	152	+ if (iter->offset == offset && iter->type == STT_FUNC)
	153	+ return iter;
171	154	}
172	155
173	156	return NULL;
@@ -175,13 +158,12 @@ struct symbol find_func_by_offset(struct section sec, unsigned long offset) · 1 unmodified line
175	158
176	159	struct symbol find_symbol_containing(const struct section sec, unsigned long offset)
177	160	{
178		− struct rb_node *node;
179		−
180		− rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
181		− struct symbol *s = rb_entry(node, struct symbol, node);
	161	+ struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
	162	+ struct symbol *iter;
182	163
183		− if (s->type != STT_SECTION)
184		− return s;
	164	+ __sym_for_each(iter, tree, offset, offset) {
	165	+ if (iter->type != STT_SECTION)
	166	+ return iter;
185	167	}
186	168
187	169	return NULL;
@@ -202,7 +184,7 @@ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) · 14 unmodified lines
202	184	/*
203	185	* Find the rightmost symbol for which @offset is after it.
204	186	*/
205		− n = rb_find(&hole, &sec->symbol_tree, symbol_hole_by_offset);
	187	+ n = rb_find(&hole, &sec->symbol_tree.rb_root, symbol_hole_by_offset);
206	188
207	189	/* found a symbol that contains @offset */
208	190	if (n)
@@ -224,13 +206,12 @@ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) · 15 unmodified lines
224	206
225	207	struct symbol find_func_containing(struct section sec, unsigned long offset)
226	208	{
227		− struct rb_node *node;
228		−
229		− rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
230		− struct symbol *s = rb_entry(node, struct symbol, node);
	209	+ struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
	210	+ struct symbol *iter;
231	211
232		− if (s->type == STT_FUNC)
233		− return s;
	212	+ __sym_for_each(iter, tree, offset, offset) {
	213	+ if (iter->type == STT_FUNC)
	214	+ return iter;
234	215	}
235	216
236	217	return NULL;
@@ -373,7 +354,9 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 136 unmodified lines
373	354	{
374	355	struct list_head *entry;
375	356	struct rb_node *pnode;
	357	+ struct symbol *iter;
376	358
	359	+ INIT_LIST_HEAD(&sym->reloc_list);
377	360	INIT_LIST_HEAD(&sym->pv_target);
378	361	sym->alias = sym;
379	362
@@ -386,7 +369,12 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 6 unmodified lines
386	369	sym->offset = sym->sym.st_value;
387	370	sym->len = sym->sym.st_size;
388	371
389		− rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset);
	372	+ __sym_for_each(iter, &sym->sec->symbol_tree, sym->offset, sym->offset) {
	373	+ if (iter->offset == sym->offset && iter->type == sym->type)
	374	+ iter->alias = sym;
	375	+ }
	376	+
	377	+ __sym_insert(sym, &sym->sec->symbol_tree);
390	378	pnode = rb_prev(&sym->node);
391	379	if (pnode)
392	380	entry = &rb_entry(pnode, struct symbol, node)->list;
@@ -401,7 +389,7 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 8 unmodified lines
401	389	* can exist within a function, confusing the sorting.
402	390	*/
403	391	if (!sym->len)
404		− rb_erase(&sym->node, &sym->sec->symbol_tree);
	392	+ __sym_remove(sym, &sym->sec->symbol_tree);
405	393	}
406	394
407	395	static int read_symbols(struct elf *elf)
@@ -570,6 +558,7 @@ int elf_add_reloc(struct elf elf, struct section sec, unsigned long offset, · 162 unmodified lines
570	558	reloc->sym = sym;
571	559	reloc->addend = addend;
572	560
	561	+ list_add_tail(&reloc->sym_reloc_entry, &sym->reloc_list);
573	562	list_add_tail(&reloc->list, &sec->reloc->reloc_list);
574	563	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
575	564
@@ -586,21 +575,10 @@ int elf_add_reloc(struct elf elf, struct section sec, unsigned long offset, · 10 unmodified lines
586	575	*/
587	576	static void elf_dirty_reloc_sym(struct elf elf, struct symbol sym)
588	577	{
589		− struct section *sec;
590		−
591		− list_for_each_entry(sec, &elf->sections, list) {
592		− struct reloc *reloc;
593		−
594		− if (sec->changed)
595		− continue;
	578	+ struct reloc *reloc;
596	579
597		− list_for_each_entry(reloc, &sec->reloc_list, list) {
598		− if (reloc->sym == sym) {
599		− sec->changed = true;
600		− break;
601		− }
602		− }
603		− }
	580	+ list_for_each_entry(reloc, &sym->reloc_list, sym_reloc_entry)
	581	+ reloc->sec->changed = true;
604	582	}
605	583
606	584	/*
@@ -647,6 +625,12 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 40 unmodified lines
647	625
648	626	/* end-of-list */
649	627	if (!symtab_data) {
	628	+ /*
	629	+ * Over-allocate to avoid O(n^2) symbol creation
	630	+ * behaviour. The down side is that libelf doesn't
	631	+ * like this; see elf_truncate_section() for the fixup.
	632	+ */
	633	+ int num = max(1U, sym->idx/3);
650	634	void *buf;
651	635
652	636	if (idx) {
@@ -660,28 +644,34 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 7 unmodified lines
660	644	if (t)
661	645	shndx_data = elf_newdata(t);
662	646
663		− buf = calloc(1, entsize);
	647	+ buf = calloc(num, entsize);
664	648	if (!buf) {
665	649	WARN("malloc");
666	650	return -1;
667	651	}
668	652
669	653	symtab_data->d_buf = buf;
670		− symtab_data->d_size = entsize;
	654	+ symtab_data->d_size = num * entsize;
671	655	symtab_data->d_align = 1;
672	656	symtab_data->d_type = ELF_T_SYM;
673	657
674		− symtab->sh.sh_size += entsize;
675	658	symtab->changed = true;
	659	+ symtab->truncate = true;
676	660
677	661	if (t) {
678		− shndx_data->d_buf = &sym->sec->idx;
679		− shndx_data->d_size = sizeof(Elf32_Word);
	662	+ buf = calloc(num, sizeof(Elf32_Word));
	663	+ if (!buf) {
	664	+ WARN("malloc");
	665	+ return -1;
	666	+ }
	667	+
	668	+ shndx_data->d_buf = buf;
	669	+ shndx_data->d_size = num * sizeof(Elf32_Word);
680	670	shndx_data->d_align = sizeof(Elf32_Word);
681	671	shndx_data->d_type = ELF_T_WORD;
682	672
683		− symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
684	673	symtab_shndx->changed = true;
	674	+ symtab_shndx->truncate = true;
685	675	}
686	676
687	677	break;
@@ -730,11 +720,11 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 42 unmodified lines
730	720	}
731	721
732	722	static struct symbol *
733		−elf_create_section_symbol(struct elf elf, struct section sec)
	723	+__elf_create_symbol(struct elf elf, struct symbol sym)
734	724	{
735	725	struct section symtab, symtab_shndx;
736	726	Elf32_Word first_non_local, new_idx;
737		− struct symbol sym, old;
	727	+ struct symbol *old;
738	728
739	729	symtab = find_section_by_name(elf, ".symtab");
740	730	if (symtab) {
@@ -744,27 +734,16 @@ elf_create_section_symbol(struct elf elf, struct section sec) · 3 unmodified lines
744	734	return NULL;
745	735	}
746	736
747		− sym = calloc(1, sizeof(*sym));
748		− if (!sym) {
749		− perror("malloc");
750		− return NULL;
751		− }
752		−
753		− sym->name = sec->name;
754		− sym->sec = sec;
	737	+ new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
755	738
756		− // st_name 0
757		− sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
758		− // st_other 0
759		− // st_value 0
760		− // st_size 0
	739	+ if (GELF_ST_BIND(sym->sym.st_info) != STB_LOCAL)
	740	+ goto non_local;
761	741
762	742	/*
763	743	* Move the first global symbol, as per sh_info, into a new, higher
764	744	* symbol index. This fees up a spot for a new local symbol.
765	745	*/
766	746	first_non_local = symtab->sh.sh_info;
767		− new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
768	747	old = find_symbol_by_index(elf, first_non_local);
769	748	if (old) {
770	749	old->idx = new_idx;
@@ -782,18 +761,82 @@ elf_create_section_symbol(struct elf elf, struct section sec) · 11 unmodified lines
782	761	new_idx = first_non_local;
783	762	}
784	763
	764	+ /*
	765	+ * Either way, we will add a LOCAL symbol.
	766	+ */
	767	+ symtab->sh.sh_info += 1;
	768	+
	769	+non_local:
785	770	sym->idx = new_idx;
786	771	if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) {
787	772	WARN("elf_update_symbol");
788	773	return NULL;
789	774	}
790	775
791		− /*
792		− * Either way, we added a LOCAL symbol.
793		− */
794		− symtab->sh.sh_info += 1;
	776	+ symtab->sh.sh_size += symtab->sh.sh_entsize;
	777	+ symtab->changed = true;
795	778
796		− elf_add_symbol(elf, sym);
	779	+ if (symtab_shndx) {
	780	+ symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
	781	+ symtab_shndx->changed = true;
	782	+ }
	783	+
	784	+ return sym;
	785	+}
	786	+
	787	+static struct symbol *
	788	+elf_create_section_symbol(struct elf elf, struct section sec)
	789	+{
	790	+ struct symbol sym = calloc(1, sizeof(sym));
	791	+
	792	+ if (!sym) {
	793	+ perror("malloc");
	794	+ return NULL;
	795	+ }
	796	+
	797	+ sym->name = sec->name;
	798	+ sym->sec = sec;
	799	+
	800	+ // st_name 0
	801	+ sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
	802	+ // st_other 0
	803	+ // st_value 0
	804	+ // st_size 0
	805	+
	806	+ sym = __elf_create_symbol(elf, sym);
	807	+ if (sym)
	808	+ elf_add_symbol(elf, sym);
	809	+
	810	+ return sym;
	811	+}
	812	+
	813	+static int elf_add_string(struct elf elf, struct section strtab, char *str);
	814	+
	815	+struct symbol *
	816	+elf_create_prefix_symbol(struct elf elf, struct symbol orig, long size)
	817	+{
	818	+ struct symbol sym = calloc(1, sizeof(sym));
	819	+ size_t namelen = strlen(orig->name) + sizeof("__pfx_");
	820	+ char *name = malloc(namelen);
	821	+
	822	+ if (!sym \|\| !name) {
	823	+ perror("malloc");
	824	+ return NULL;
	825	+ }
	826	+
	827	+ snprintf(name, namelen, "__pfx_%s", orig->name);
	828	+
	829	+ sym->name = name;
	830	+ sym->sec = orig->sec;
	831	+
	832	+ sym->sym.st_name = elf_add_string(elf, NULL, name);
	833	+ sym->sym.st_info = orig->sym.st_info;
	834	+ sym->sym.st_value = orig->sym.st_value - size;
	835	+ sym->sym.st_size = size;
	836	+
	837	+ sym = __elf_create_symbol(elf, sym);
	838	+ if (sym)
	839	+ elf_add_symbol(elf, sym);
797	840
798	841	return sym;
799	842	}
@@ -850,11 +893,12 @@ static int read_rela_reloc(struct section sec, int i, struct reloc reloc, unsi · 50 unmodified lines
850	893
851	894	static int read_relocs(struct elf *elf)
852	895	{
	896	+ unsigned long nr_reloc, max_reloc = 0, tot_reloc = 0;
853	897	struct section *sec;
854	898	struct reloc *reloc;
855		− int i;
856	899	unsigned int symndx;
857		− unsigned long nr_reloc, max_reloc = 0, tot_reloc = 0;
	900	+ struct symbol *sym;
	901	+ int i;
858	902
859	903	if (!elf_alloc_hash(reloc, elf->text_size / 16))
860	904	return -1;
@@ -895,13 +939,14 @@ static int read_relocs(struct elf *elf) · 34 unmodified lines
895	939
896	940	reloc->sec = sec;
897	941	reloc->idx = i;
898		− reloc->sym = find_symbol_by_index(elf, symndx);
	942	+ reloc->sym = sym = find_symbol_by_index(elf, symndx);
899	943	if (!reloc->sym) {
900	944	WARN("can't find reloc entry symbol %d for %s",
901	945	symndx, sec->name);
902	946	return -1;
903	947	}
904	948
	949	+ list_add_tail(&reloc->sym_reloc_entry, &sym->reloc_list);
905	950	list_add_tail(&reloc->list, &sec->reloc_list);
906	951	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
907	952
@@ -1285,6 +1330,60 @@ int elf_write_reloc(struct elf elf, struct reloc reloc) · 377 unmodified lines
1285	1330	return 0;
1286	1331	}
1287	1332
	1333	+/*
	1334	+ * When Elf_Scn::sh_size is smaller than the combined Elf_Data::d_size
	1335	+ * do you:
	1336	+ *
	1337	+ * A) adhere to the section header and truncate the data, or
	1338	+ * B) ignore the section header and write out all the data you've got?
	1339	+ *
	1340	+ * Yes, libelf sucks and we need to manually truncate if we over-allocate data.
	1341	+ */
	1342	+static int elf_truncate_section(struct elf elf, struct section sec)
	1343	+{
	1344	+ u64 size = sec->sh.sh_size;
	1345	+ bool truncated = false;
	1346	+ Elf_Data *data = NULL;
	1347	+ Elf_Scn *s;
	1348	+
	1349	+ s = elf_getscn(elf->elf, sec->idx);
	1350	+ if (!s) {
	1351	+ WARN_ELF("elf_getscn");
	1352	+ return -1;
	1353	+ }
	1354	+
	1355	+ for (;;) {
	1356	+ /* get next data descriptor for the relevant section */
	1357	+ data = elf_getdata(s, data);
	1358	+
	1359	+ if (!data) {
	1360	+ if (size) {
	1361	+ WARN("end of section data but non-zero size left\n");
	1362	+ return -1;
	1363	+ }
	1364	+ return 0;
	1365	+ }
	1366	+
	1367	+ if (truncated) {
	1368	+ /* when we remove symbols */
	1369	+ WARN("truncated; but more data\n");
	1370	+ return -1;
	1371	+ }
	1372	+
	1373	+ if (!data->d_size) {
	1374	+ WARN("zero size data");
	1375	+ return -1;
	1376	+ }
	1377	+
	1378	+ if (data->d_size > size) {
	1379	+ truncated = true;
	1380	+ data->d_size = size;
	1381	+ }
	1382	+
	1383	+ size -= data->d_size;
	1384	+ }
	1385	+}
	1386	+
1288	1387	int elf_write(struct elf *elf)
1289	1388	{
1290	1389	struct section *sec;
@@ -1295,6 +1394,9 @@ int elf_write(struct elf *elf) · 4 unmodified lines
1295	1394
1296	1395	/* Update changed relocation sections and section headers: */
1297	1396	list_for_each_entry(sec, &elf->sections, list) {
	1397	+ if (sec->truncate)
	1398	+ elf_truncate_section(elf, sec);
	1399	+
1298	1400	if (sec->changed) {
1299	1401	s = elf_getscn(elf->elf, sec->idx);
1300	1402	if (!s) {

tools/objtool/include/objtool/arch.h+2−0

@@ -93,4 +93,6 @@ bool arch_is_rethunk(struct symbol *sym); · 92 unmodified lines
93	93
94	94	int arch_rewrite_retpolines(struct objtool_file *file);
95	95
	96	+bool arch_pc_relative_reloc(struct reloc *reloc);
	97	+
96	98	#endif /* _ARCH_H */

tools/objtool/include/objtool/builtin.h+3−0

@@ -14,6 +14,7 @@ struct opts { · 13 unmodified lines
14	14	bool dump_orc;
15	15	bool hack_jump_label;
16	16	bool hack_noinstr;
	17	+ bool hack_skylake;
17	18	bool ibt;
18	19	bool mcount;
19	20	bool noinstr;
@@ -25,6 +26,8 @@ struct opts { · 5 unmodified lines
25	26	bool stackval;
26	27	bool static_call;
27	28	bool uaccess;
	29	+ int prefix;
	30	+ bool cfi;
28	31
29	32	/* options: */
30	33	bool backtrace;

tools/objtool/include/objtool/check.h+11−1

@@ -67,11 +67,21 @@ struct instruction { · 66 unmodified lines
67	67	struct reloc *jump_table;
68	68	struct reloc *reloc;
69	69	struct list_head alts;
70		− struct symbol *func;
	70	+ struct symbol *sym;
71	71	struct list_head stack_ops;
72	72	struct cfi_state *cfi;
73	73	};
74	74
	75	+static inline struct symbol insn_func(struct instruction insn)
	76	+{
	77	+ struct symbol *sym = insn->sym;
	78	+
	79	+ if (sym && sym->type != STT_FUNC)
	80	+ sym = NULL;
	81	+
	82	+ return sym;
	83	+}
	84	+
75	85	#define VISITED_BRANCH 0x01
76	86	#define VISITED_BRANCH_UACCESS 0x02
77	87	#define VISITED_BRANCH_MASK 0x03

tools/objtool/include/objtool/elf.h+7−2

@@ -30,7 +30,7 @@ struct section { · 29 unmodified lines
30	30	struct hlist_node hash;
31	31	struct hlist_node name_hash;
32	32	GElf_Shdr sh;
33		− struct rb_root symbol_tree;
	33	+ struct rb_root_cached symbol_tree;
34	34	struct list_head symbol_list;
35	35	struct list_head reloc_list;
36	36	struct section base, reloc;
@@ -38,7 +38,7 @@ struct section { · 1 unmodified line
38	38	Elf_Data *data;
39	39	char *name;
40	40	int idx;
41		− bool changed, text, rodata, noinstr;
	41	+ bool changed, text, rodata, noinstr, init, truncate;
42	42	};
43	43
44	44	struct symbol {
@@ -53,6 +53,7 @@ struct symbol { · 8 unmodified lines
53	53	unsigned char bind, type;
54	54	unsigned long offset;
55	55	unsigned int len;
	56	+ unsigned long __subtree_last;
56	57	struct symbol pfunc, cfunc, *alias;
57	58	u8 uaccess_safe : 1;
58	59	u8 static_call_tramp : 1;
@@ -61,6 +62,7 @@ struct symbol { · 2 unmodified lines
61	62	u8 fentry : 1;
62	63	u8 profiling_func : 1;
63	64	struct list_head pv_target;
	65	+ struct list_head reloc_list;
64	66	};
65	67
66	68	struct reloc {
@@ -72,6 +74,7 @@ struct reloc { · 5 unmodified lines
72	74	};
73	75	struct section *sec;
74	76	struct symbol *sym;
	77	+ struct list_head sym_reloc_entry;
75	78	unsigned long offset;
76	79	unsigned int type;
77	80	s64 addend;
@@ -145,6 +148,8 @@ static inline bool has_multiple_files(struct elf *elf) · 67 unmodified lines
145	148	struct elf elf_open_read(const char name, int flags);
146	149	struct section elf_create_section(struct elf elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
147	150
	151	+struct symbol elf_create_prefix_symbol(struct elf elf, struct symbol *orig, long size);
	152	+
148	153	int elf_add_reloc(struct elf elf, struct section sec, unsigned long offset,
149	154	unsigned int type, struct symbol *sym, s64 addend);
150	155	int elf_add_reloc_to_insn(struct elf elf, struct section sec,

tools/objtool/include/objtool/objtool.h+1−0

@@ -28,6 +28,7 @@ struct objtool_file { · 27 unmodified lines
28	28	struct list_head static_call_list;
29	29	struct list_head mcount_loc_list;
30	30	struct list_head endbr_list;
	31	+ struct list_head call_list;
31	32	bool ignore_unreachables, hints, rodata;
32	33
33	34	unsigned int nr_endbr;

tools/objtool/objtool.c+1−0

@@ -106,6 +106,7 @@ struct objtool_file objtool_open_read(const char _objname) · 105 unmodified lines
106	106	INIT_LIST_HEAD(&file.static_call_list);
107	107	INIT_LIST_HEAD(&file.mcount_loc_list);
108	108	INIT_LIST_HEAD(&file.endbr_list);
	109	+ INIT_LIST_HEAD(&file.call_list);
109	110	file.ignore_unreachables = opts.no_unreachable;
110	111	file.hints = false;
111	112