lael-labs/linux

Makefile+2−2

@@ -1006,8 +1006,8 @@ KBUILD_CFLAGS += $(CC_FLAGS_CFI) · 1005 unmodified lines
1006	export CC_FLAGS_CFI	1006	export CC_FLAGS_CFI
1007	endif	1007	endif
1008		1008
1009	ifdef CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B	1009	ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0)
1010	KBUILD_CFLAGS += -falign-functions=64	1010	KBUILD_CFLAGS += -falign-functions=$(CONFIG_FUNCTION_ALIGNMENT)
1011	endif	1011	endif
1012		1012
1013	# arch Makefile may override CC so keep this after arch Makefile is included	1013	# arch Makefile may override CC so keep this after arch Makefile is included

arch/Kconfig+24−0

@@ -1438,4 +1438,28 @@ source "kernel/gcov/Kconfig" · 1437 unmodified lines
1438		1438
1439	source "scripts/gcc-plugins/Kconfig"	1439	source "scripts/gcc-plugins/Kconfig"
1440		1440
		1441	config FUNCTION_ALIGNMENT_4B
		1442	bool
		1443
		1444	config FUNCTION_ALIGNMENT_8B
		1445	bool
		1446
		1447	config FUNCTION_ALIGNMENT_16B
		1448	bool
		1449
		1450	config FUNCTION_ALIGNMENT_32B
		1451	bool
		1452
		1453	config FUNCTION_ALIGNMENT_64B
		1454	bool
		1455
		1456	config FUNCTION_ALIGNMENT
		1457	int
		1458	default 64 if FUNCTION_ALIGNMENT_64B
		1459	default 32 if FUNCTION_ALIGNMENT_32B
		1460	default 16 if FUNCTION_ALIGNMENT_16B
		1461	default 8 if FUNCTION_ALIGNMENT_8B
		1462	default 4 if FUNCTION_ALIGNMENT_4B
		1463	default 0
		1464
1441	endmenu	1465	endmenu

arch/ia64/Kconfig+1−0

@@ -63,6 +63,7 @@ config IA64 · 62 unmodified lines
63	select NUMA if !FLATMEM	63	select NUMA if !FLATMEM
64	select PCI_MSI_ARCH_FALLBACKS if PCI_MSI	64	select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
65	select ZONE_DMA32	65	select ZONE_DMA32
		66	select FUNCTION_ALIGNMENT_32B
66	default y	67	default y
67	help	68	help
68	The Itanium Processor Family is Intel's 64-bit successor to	69	The Itanium Processor Family is Intel's 64-bit successor to

arch/ia64/Makefile+1−1

@@ -23,7 +23,7 @@ KBUILD_AFLAGS_KERNEL := -mconstant-gp · 22 unmodified lines
23	EXTRA :=	23	EXTRA :=
24		24
25	cflags-y := -pipe $(EXTRA) -ffixed-r13 -mfixed-range=f12-f15,f32-f127 \	25	cflags-y := -pipe $(EXTRA) -ffixed-r13 -mfixed-range=f12-f15,f32-f127 \
26	-falign-functions=32 -frename-registers -fno-optimize-sibling-calls	26	-frename-registers -fno-optimize-sibling-calls
27	KBUILD_CFLAGS_KERNEL := -mconstant-gp	27	KBUILD_CFLAGS_KERNEL := -mconstant-gp
28		28
29	GAS_STATUS = $(shell $(srctree)/arch/ia64/scripts/check-gas "$(CC)" "$(OBJDUMP)")	29	GAS_STATUS = $(shell $(srctree)/arch/ia64/scripts/check-gas "$(CC)" "$(OBJDUMP)")

arch/um/kernel/um_arch.c+5−0

 {
 }
+void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
+		   s32 *start_cfi, s32 *end_cfi)
+{
+}
 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 {
 }

arch/x86/Kconfig+74−1

@@ -292,6 +292,8 @@ config X86 · 291 unmodified lines
292	select X86_FEATURE_NAMES if PROC_FS	292	select X86_FEATURE_NAMES if PROC_FS
293	select PROC_PID_ARCH_STATUS if PROC_FS	293	select PROC_PID_ARCH_STATUS if PROC_FS
294	select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX	294	select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX
		295	select FUNCTION_ALIGNMENT_16B if X86_64 \|\| X86_ALIGNMENT_16
		296	select FUNCTION_ALIGNMENT_4B
295	imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI	297	imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI
296	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE	298	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
297		299
@@ -1855,7 +1857,7 @@ config CC_HAS_IBT · 1557 unmodified lines
1855		1857
1856	config X86_KERNEL_IBT	1858	config X86_KERNEL_IBT
1857	prompt "Indirect Branch Tracking"	1859	prompt "Indirect Branch Tracking"
1858	bool	1860	def_bool y
1859	depends on X86_64 && CC_HAS_IBT && HAVE_OBJTOOL	1861	depends on X86_64 && CC_HAS_IBT && HAVE_OBJTOOL
1860	# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f	1862	# https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
1861	depends on !LD_IS_LLD \|\| LLD_VERSION >= 140000	1863	depends on !LD_IS_LLD \|\| LLD_VERSION >= 140000
@@ -2492,6 +2494,46 @@ config CC_HAS_SLS · 630 unmodified lines
2492	config CC_HAS_RETURN_THUNK	2494	config CC_HAS_RETURN_THUNK
2493	def_bool $(cc-option,-mfunction-return=thunk-extern)	2495	def_bool $(cc-option,-mfunction-return=thunk-extern)
2494		2496
		2497	config CC_HAS_ENTRY_PADDING
		2498	def_bool $(cc-option,-fpatchable-function-entry=16,16)
		2499
		2500	config FUNCTION_PADDING_CFI
		2501	int
		2502	default 59 if FUNCTION_ALIGNMENT_64B
		2503	default 27 if FUNCTION_ALIGNMENT_32B
		2504	default 11 if FUNCTION_ALIGNMENT_16B
		2505	default 3 if FUNCTION_ALIGNMENT_8B
		2506	default 0
		2507
		2508	# Basically: FUNCTION_ALIGNMENT - 5*CFI_CLANG
		2509	# except Kconfig can't do arithmetic :/
		2510	config FUNCTION_PADDING_BYTES
		2511	int
		2512	default FUNCTION_PADDING_CFI if CFI_CLANG
		2513	default FUNCTION_ALIGNMENT
		2514
		2515	config CALL_PADDING
		2516	def_bool n
		2517	depends on CC_HAS_ENTRY_PADDING && OBJTOOL
		2518	select FUNCTION_ALIGNMENT_16B
		2519
		2520	config FINEIBT
		2521	def_bool y
		2522	depends on X86_KERNEL_IBT && CFI_CLANG && RETPOLINE
		2523	select CALL_PADDING
		2524
		2525	config HAVE_CALL_THUNKS
		2526	def_bool y
		2527	depends on CC_HAS_ENTRY_PADDING && RETHUNK && OBJTOOL
		2528
		2529	config CALL_THUNKS
		2530	def_bool n
		2531	select CALL_PADDING
		2532
		2533	config PREFIX_SYMBOLS
		2534	def_bool y
		2535	depends on CALL_PADDING && !CFI_CLANG
		2536
2495	menuconfig SPECULATION_MITIGATIONS	2537	menuconfig SPECULATION_MITIGATIONS
2496	bool "Mitigations for speculative execution vulnerabilities"	2538	bool "Mitigations for speculative execution vulnerabilities"
2497	default y	2539	default y
@@ -2543,6 +2585,37 @@ config CPU_UNRET_ENTRY · 45 unmodified lines
2543	help	2585	help
2544	Compile the kernel with support for the retbleed=unret mitigation.	2586	Compile the kernel with support for the retbleed=unret mitigation.
2545		2587
		2588	config CALL_DEPTH_TRACKING
		2589	bool "Mitigate RSB underflow with call depth tracking"
		2590	depends on CPU_SUP_INTEL && HAVE_CALL_THUNKS
		2591	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
		2592	select CALL_THUNKS
		2593	default y
		2594	help
		2595	Compile the kernel with call depth tracking to mitigate the Intel
		2596	SKL Return-Speculation-Buffer (RSB) underflow issue. The
		2597	mitigation is off by default and needs to be enabled on the
		2598	kernel command line via the retbleed=stuff option. For
		2599	non-affected systems the overhead of this option is marginal as
		2600	the call depth tracking is using run-time generated call thunks
		2601	in a compiler generated padding area and call patching. This
		2602	increases text size by ~5%. For non affected systems this space
		2603	is unused. On affected SKL systems this results in a significant
		2604	performance gain over the IBRS mitigation.
		2605
		2606	config CALL_THUNKS_DEBUG
		2607	bool "Enable call thunks and call depth tracking debugging"
		2608	depends on CALL_DEPTH_TRACKING
		2609	select FUNCTION_ALIGNMENT_32B
		2610	default n
		2611	help
		2612	Enable call/ret counters for imbalance detection and build in
		2613	a noisy dmesg about callthunks generation and call patching for
		2614	trouble shooting. The debug prints need to be enabled on the
		2615	kernel command line with 'debug-callthunks'.
		2616	Only enable this, when you are debugging call thunks as this
		2617	creates a noticable runtime overhead. If unsure say N.
		2618
2546	config CPU_IBPB_ENTRY	2619	config CPU_IBPB_ENTRY
2547	bool "Enable IBPB on kernel entry"	2620	bool "Enable IBPB on kernel entry"
2548	depends on CPU_SUP_AMD && X86_64	2621	depends on CPU_SUP_AMD && X86_64

arch/x86/Makefile+6−0

   KBUILD_CFLAGS += -mharden-sls=all
 endif
+ifdef CONFIG_CALL_PADDING
+PADDING_CFLAGS := -fpatchable-function-entry=$(CONFIG_FUNCTION_PADDING_BYTES),$(CONFIG_FUNCTION_PADDING_BYTES)
+KBUILD_CFLAGS += $(PADDING_CFLAGS)
+export PADDING_CFLAGS
+endif
 KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)
 ifdef CONFIG_LTO_CLANG

arch/x86/boot/compressed/head_64.S+8−0

@@ -37,6 +37,14 @@ · 36 unmodified lines
37	#include <asm/trapnr.h>	37	#include <asm/trapnr.h>
38	#include "pgtable.h"	38	#include "pgtable.h"
39		39
		40	/*
		41	* Fix alignment at 16 bytes. Following CONFIG_FUNCTION_ALIGNMENT will result
		42	* in assembly errors due to trying to move .org backward due to the excessive
		43	* alignment.
		44	*/
		45	#undef __ALIGN
		46	#define __ALIGN .balign 16, 0x90
		47
40	/*	48	/*
41	* Locally defined symbols should be marked hidden:	49	* Locally defined symbols should be marked hidden:
42	*/	50	*/

arch/x86/crypto/camellia-aesni-avx-asm_64.S+0−2

 .text
-.align 8
 SYM_FUNC_START_LOCAL(__camellia_enc_blk16)
 	/* input:
 	 *	%rdi: ctx, CTX
 	jmp .Lenc_done;
 SYM_FUNC_END(__camellia_enc_blk16)
-.align 8
 SYM_FUNC_START_LOCAL(__camellia_dec_blk16)
 	/* input:
 	 *	%rdi: ctx, CTX

arch/x86/crypto/camellia-aesni-avx2-asm_64.S+0−4

  * Size optimization... with inlined roundsm32 binary would be over 5 times
  * larger and would only marginally faster.
  */
-.align 8
 SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
 	roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
 		  %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
 	RET;
 SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
-.align 8
 SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
 	roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
 		  %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
 .text
-.align 8
 SYM_FUNC_START_LOCAL(__camellia_enc_blk32)
 	/* input:
 	 *	%rdi: ctx, CTX
 	jmp .Lenc_done;
 SYM_FUNC_END(__camellia_enc_blk32)
-.align 8
 SYM_FUNC_START_LOCAL(__camellia_dec_blk32)
 	/* input:
 	 *	%rdi: ctx, CTX

arch/x86/crypto/cast5-avx-x86_64-asm_64.S+0−2

 .text
-.align 16
 SYM_FUNC_START_LOCAL(__cast5_enc_blk16)
 	/* input:
 	 *	%rdi: ctx
 	RET;
 SYM_FUNC_END(__cast5_enc_blk16)
-.align 16
 SYM_FUNC_START_LOCAL(__cast5_dec_blk16)
 	/* input:
 	 *	%rdi: ctx

arch/x86/crypto/crct10dif-pcl-asm_64.S+0−1

@@ -94,7 +94,6 @@ · 93 unmodified lines
94	#	94	#
95	# Assumes len >= 16.	95	# Assumes len >= 16.
96	#	96	#
97	.align 16
98	SYM_FUNC_START(crc_t10dif_pcl)	97	SYM_FUNC_START(crc_t10dif_pcl)
99		98
100	movdqa .Lbswap_mask(%rip), BSWAP_MASK	99	movdqa .Lbswap_mask(%rip), BSWAP_MASK

arch/x86/crypto/poly1305-x86_64-cryptogams.pl+0−1

@@ -108,7 +108,6 @@ if (!$kernel) { · 107 unmodified lines
108	sub declare_function() {	108	sub declare_function() {
109	my ($name, $align, $nargs) = @_;	109	my ($name, $align, $nargs) = @_;
110	if($kernel) {	110	if($kernel) {
111	$code .= ".align $align\n";
112	$code .= "SYM_FUNC_START($name)\n";	111	$code .= "SYM_FUNC_START($name)\n";
113	$code .= ".L$name:\n";	112	$code .= ".L$name:\n";
114	} else {	113	} else {

arch/x86/crypto/serpent-avx-x86_64-asm_64.S+0−2

 #define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
 	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-.align 8
 SYM_FUNC_START_LOCAL(__serpent_enc_blk8_avx)
 	/* input:
 	 *	%rdi: ctx, CTX
 	RET;
 SYM_FUNC_END(__serpent_enc_blk8_avx)
-.align 8
 SYM_FUNC_START_LOCAL(__serpent_dec_blk8_avx)
 	/* input:
 	 *	%rdi: ctx, CTX

arch/x86/crypto/serpent-avx2-asm_64.S+0−2

 #define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
 	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
-.align 8
 SYM_FUNC_START_LOCAL(__serpent_enc_blk16)
 	/* input:
 	 *	%rdi: ctx, CTX
 	RET;
 SYM_FUNC_END(__serpent_enc_blk16)
-.align 8
 SYM_FUNC_START_LOCAL(__serpent_dec_blk16)
 	/* input:
 	 *	%rdi: ctx, CTX

arch/x86/crypto/sha1_ni_asm.S+0−1

@@ -93,7 +93,6 @@ · 92 unmodified lines
93	* numBlocks: Number of blocks to process	93	* numBlocks: Number of blocks to process
94	*/	94	*/
95	.text	95	.text
96	.align 32
97	SYM_TYPED_FUNC_START(sha1_ni_transform)	96	SYM_TYPED_FUNC_START(sha1_ni_transform)
98	push %rbp	97	push %rbp
99	mov %rsp, %rbp	98	mov %rsp, %rbp

arch/x86/crypto/sha256-avx-asm.S+0−1

@@ -348,7 +348,6 @@ a = TMP_ · 347 unmodified lines
348	########################################################################	348	########################################################################
349	.text	349	.text
350	SYM_TYPED_FUNC_START(sha256_transform_avx)	350	SYM_TYPED_FUNC_START(sha256_transform_avx)
351	.align 32
352	pushq %rbx	351	pushq %rbx
353	pushq %r12	352	pushq %r12
354	pushq %r13	353	pushq %r13

arch/x86/crypto/sha256-avx2-asm.S+0−1

@@ -525,7 +525,6 @@ STACK_SIZE = _CTX + _CTX_SIZE · 524 unmodified lines
525	########################################################################	525	########################################################################
526	.text	526	.text
527	SYM_TYPED_FUNC_START(sha256_transform_rorx)	527	SYM_TYPED_FUNC_START(sha256_transform_rorx)
528	.align 32
529	pushq %rbx	528	pushq %rbx
530	pushq %r12	529	pushq %r12
531	pushq %r13	530	pushq %r13

arch/x86/crypto/sha256-ssse3-asm.S+0−1

@@ -357,7 +357,6 @@ a = TMP_ · 356 unmodified lines
357	########################################################################	357	########################################################################
358	.text	358	.text
359	SYM_TYPED_FUNC_START(sha256_transform_ssse3)	359	SYM_TYPED_FUNC_START(sha256_transform_ssse3)
360	.align 32
361	pushq %rbx	360	pushq %rbx
362	pushq %r12	361	pushq %r12
363	pushq %r13	362	pushq %r13

arch/x86/crypto/sha256_ni_asm.S+0−1

@@ -97,7 +97,6 @@ · 96 unmodified lines
97	*/	97	*/
98		98
99	.text	99	.text
100	.align 32
101	SYM_TYPED_FUNC_START(sha256_ni_transform)	100	SYM_TYPED_FUNC_START(sha256_ni_transform)
102		101
103	shl $6, NUM_BLKS /* convert to bytes */	102	shl $6, NUM_BLKS /* convert to bytes */

arch/x86/crypto/sm3-avx-asm_64.S+0−1

@@ -328,7 +328,6 @@ · 327 unmodified lines
328	* void sm3_transform_avx(struct sm3_state *state,	328	* void sm3_transform_avx(struct sm3_state *state,
329	* const u8 *data, int nblocks);	329	* const u8 *data, int nblocks);
330	*/	330	*/
331	.align 16
332	SYM_TYPED_FUNC_START(sm3_transform_avx)	331	SYM_TYPED_FUNC_START(sm3_transform_avx)
333	/* input:	332	/* input:
334	* %rdi: ctx, CTX	333	* %rdi: ctx, CTX

arch/x86/crypto/sm4-aesni-avx-asm_64.S+0−7

 .text
-.align 16
 /*
  * void sm4_aesni_avx_crypt4(const u32 *rk, u8 *dst,
  *                           const u8 *src, int nblocks)
  */
-.align 8
 SYM_FUNC_START(sm4_aesni_avx_crypt4)
 	/* input:
 	 *	%rdi: round key array, CTX
 	RET;
 SYM_FUNC_END(sm4_aesni_avx_crypt4)
-.align 8
 SYM_FUNC_START_LOCAL(__sm4_crypt_blk8)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx_crypt8(const u32 *rk, u8 *dst,
  *                           const u8 *src, int nblocks)
  */
-.align 8
 SYM_FUNC_START(sm4_aesni_avx_crypt8)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx_ctr_enc_blk8(const u32 *rk, u8 *dst,
  *                                 const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx_ctr_enc_blk8)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx_cbc_dec_blk8(const u32 *rk, u8 *dst,
  *                                 const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx_cbc_dec_blk8)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx_cfb_dec_blk8(const u32 *rk, u8 *dst,
  *                                 const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx_cfb_dec_blk8)
 	/* input:
 	 *	%rdi: round key array, CTX

arch/x86/crypto/sm4-aesni-avx2-asm_64.S+0−6

 	.long 0xdeadbeef, 0xdeadbeef, 0xdeadbeef
 .text
-.align 16
-.align 8
 SYM_FUNC_START_LOCAL(__sm4_crypt_blk16)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
  *                                   const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx2_ctr_enc_blk16)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
  *                                   const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16)
 	/* input:
 	 *	%rdi: round key array, CTX
  * void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
  *                                   const u8 *src, u8 *iv)
  */
-.align 8
 SYM_TYPED_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16)
 	/* input:
 	 *	%rdi: round key array, CTX

arch/x86/crypto/twofish-avx-x86_64-asm_64.S+0−2

 	vpxor		x2, wkey, x2; \
 	vpxor		x3, wkey, x3;
-.align 8
 SYM_FUNC_START_LOCAL(__twofish_enc_blk8)
 	/* input:
 	 *	%rdi: ctx, CTX
 	RET;
 SYM_FUNC_END(__twofish_enc_blk8)
-.align 8
 SYM_FUNC_START_LOCAL(__twofish_dec_blk8)
 	/* input:
 	 *	%rdi: ctx, CTX

arch/x86/entry/entry_32.S+2−2

@@ -1181,7 +1181,7 @@ SYM_CODE_START(asm_exc_nmi) · 1180 unmodified lines
1181	* is using the thread stack right now, so it's safe for us to use it.	1181	* is using the thread stack right now, so it's safe for us to use it.
1182	*/	1182	*/
1183	movl %esp, %ebx	1183	movl %esp, %ebx
1184	movl PER_CPU_VAR(cpu_current_top_of_stack), %esp	1184	movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp
1185	call exc_nmi	1185	call exc_nmi
1186	movl %ebx, %esp	1186	movl %ebx, %esp
1187		1187
@@ -1243,7 +1243,7 @@ SYM_CODE_START(rewind_stack_and_make_dead) · 55 unmodified lines
1243	/* Prevent any naive code from trying to unwind to our caller. */	1243	/* Prevent any naive code from trying to unwind to our caller. */
1244	xorl %ebp, %ebp	1244	xorl %ebp, %ebp
1245		1245
1246	movl PER_CPU_VAR(cpu_current_top_of_stack), %esi	1246	movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
1247	leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp	1247	leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1248		1248
1249	call make_task_dead	1249	call make_task_dead

arch/x86/entry/entry_64.S+28−22

 	/* tss.sp2 is scratch space. */
 	movq	%rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
 #ifdef CONFIG_STACKPROTECTOR
 	movq	TASK_stack_canary(%rsi), %rbx
-	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
+	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + FIXED_stack_canary
 #endif
 	/*
  * r12: kernel thread arg
  */
 .pushsection .text, "ax"
-SYM_CODE_START(ret_from_fork)
+	__FUNC_ALIGN
+SYM_CODE_START_NOALIGN(ret_from_fork)
 	UNWIND_HINT_EMPTY
 	ANNOTATE_NOENDBR // copy_thread
+	CALL_DEPTH_ACCOUNT
 	movq	%rax, %rdi
 	call	schedule_tail			/* rdi: 'prev' task parameter */
 #endif
 .endm
-SYM_CODE_START_LOCAL(xen_error_entry)
+SYM_CODE_START(xen_error_entry)
+	ANNOTATE_NOENDBR
 	UNWIND_HINT_FUNC
 	PUSH_AND_CLEAR_REGS save_ret=1
 	ENCODE_FRAME_POINTER 8
-	UNTRAIN_RET
+	UNTRAIN_RET_FROM_CALL
 	RET
 SYM_CODE_END(xen_error_entry)
  * shared between 32 and 64 bit and emit the __irqentry_text_* markers
  * so the stacktrace boundary checks work.
  */
-	.align 16
+	__ALIGN
 	.globl __irqentry_text_start
 __irqentry_text_start:
 #include <asm/idtentry.h>
-	.align 16
+	__ALIGN
 	.globl __irqentry_text_end
 __irqentry_text_end:
 	ANNOTATE_NOENDBR
  *
  * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
  */
-SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
+	__FUNC_ALIGN
+SYM_CODE_START_LOCAL_NOALIGN(exc_xen_hypervisor_callback)
 /*
  * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
  * We distinguish between categories by comparing each saved segment register
  * with its current contents: any discrepancy means we in category 1.
  */
-SYM_CODE_START(xen_failsafe_callback)
+	__FUNC_ALIGN
+SYM_CODE_START_NOALIGN(xen_failsafe_callback)
 	UNWIND_HINT_EMPTY
 	ENDBR
 	movl	%ds, %ecx
  * R14 - old CR3
  * R15 - old SPEC_CTRL
  */
-SYM_CODE_START_LOCAL(paranoid_entry)
+SYM_CODE_START(paranoid_entry)
+	ANNOTATE_NOENDBR
 	UNWIND_HINT_FUNC
 	PUSH_AND_CLEAR_REGS save_ret=1
 	ENCODE_FRAME_POINTER 8
 	 * CR3 above, keep the old value in a callee saved register.
 	 */
 	IBRS_ENTER save_reg=%r15
-	UNTRAIN_RET
+	UNTRAIN_RET_FROM_CALL
 	RET
 SYM_CODE_END(paranoid_entry)
 /*
  * Switch GS and CR3 if needed.
  */
-SYM_CODE_START_LOCAL(error_entry)
+SYM_CODE_START(error_entry)
+	ANNOTATE_NOENDBR
 	UNWIND_HINT_FUNC
 	PUSH_AND_CLEAR_REGS save_ret=1
 	/* We have user CR3.  Change to kernel CR3. */
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 	IBRS_ENTER
-	UNTRAIN_RET
+	UNTRAIN_RET_FROM_CALL
 	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */
-.Lerror_entry_from_usermode_after_swapgs:
 	/* Put us onto the real thread stack. */
-	call	sync_regs
+	jmp	sync_regs
-	RET
 	/*
 	 * There are two places in the kernel that can potentially fault with
 	 */
 .Lerror_entry_done_lfence:
 	FENCE_SWAPGS_KERNEL_ENTRY
+	CALL_DEPTH_ACCOUNT
 	leaq	8(%rsp), %rax			/* return pt_regs pointer */
 	ANNOTATE_UNRET_END
 	RET
 	FENCE_SWAPGS_USER_ENTRY
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 	IBRS_ENTER
-	UNTRAIN_RET
+	UNTRAIN_RET_FROM_CALL
 	/*
 	 * Pretend that the exception came from user mode: set up pt_regs
 	leaq	8(%rsp), %rdi			/* arg0 = pt_regs pointer */
 	call	fixup_bad_iret
 	mov	%rax, %rdi
-	jmp	.Lerror_entry_from_usermode_after_swapgs
+	jmp	sync_regs
 SYM_CODE_END(error_entry)
 SYM_CODE_START_LOCAL(error_return)
 	FENCE_SWAPGS_USER_ENTRY
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
 	movq	%rsp, %rdx
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
 	UNWIND_HINT_IRET_REGS base=%rdx offset=8
 	pushq	5*8(%rdx)	/* pt_regs->ss */
 	pushq	4*8(%rdx)	/* pt_regs->rsp */
 #endif
 .pushsection .text, "ax"
-SYM_CODE_START(rewind_stack_and_make_dead)
+	__FUNC_ALIGN
+SYM_CODE_START_NOALIGN(rewind_stack_and_make_dead)
 	UNWIND_HINT_FUNC
 	/* Prevent any naive code from trying to unwind to our caller. */
 	xorl	%ebp, %ebp
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rax
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
 	leaq	-PTREGS_SIZE(%rax), %rsp
 	UNWIND_HINT_REGS

arch/x86/entry/entry_64_compat.S+3−4

 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
 	popq	%rax
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
 	/* Construct struct pt_regs on stack */
 	pushq	$__USER_DS		/* pt_regs->ss */
 	popfq
 	jmp	.Lsysenter_flags_fixed
 SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR // is_sysenter_singlestep
 SYM_CODE_END(entry_SYSENTER_compat)
 /*
 	SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
 	/* Switch to the kernel stack */
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
 SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
 	ALTERNATIVE "", "jmp .Lint80_keep_stack", X86_FEATURE_XENPV
 	movq	%rsp, %rax
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+	movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
 	pushq	5*8(%rax)		/* regs->ss */
 	pushq	4*8(%rax)		/* regs->rsp */

arch/x86/entry/thunk_64.S+2−2

@@ -11,7 +11,7 @@ · 10 unmodified lines
11		11
12	/* rdi: arg1 ... normal C conventions. rax is saved/restored. */	12	/* rdi: arg1 ... normal C conventions. rax is saved/restored. */
13	.macro THUNK name, func	13	.macro THUNK name, func
14	SYM_FUNC_START_NOALIGN(\name)	14	SYM_FUNC_START(\name)
15	pushq %rbp	15	pushq %rbp
16	movq %rsp, %rbp	16	movq %rsp, %rbp
17		17
@@ -36,7 +36,7 @@ SYM_FUNC_END(\name) · 18 unmodified lines
36	EXPORT_SYMBOL(preempt_schedule_thunk)	36	EXPORT_SYMBOL(preempt_schedule_thunk)
37	EXPORT_SYMBOL(preempt_schedule_notrace_thunk)	37	EXPORT_SYMBOL(preempt_schedule_notrace_thunk)
38		38
39	SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore)	39	SYM_CODE_START_LOCAL(__thunk_restore)
40	popq %r11	40	popq %r11
41	popq %r10	41	popq %r10
42	popq %r9	42	popq %r9

arch/x86/entry/vdso/Makefile+8−6

 vobjs-$(CONFIG_X86_SGX)	+= vsgx.o
 # files to link into kernel
-obj-y				+= vma.o extable.o
+obj-y					+= vma.o extable.o
-KASAN_SANITIZE_vma.o		:= y
+KASAN_SANITIZE_vma.o			:= y
-UBSAN_SANITIZE_vma.o		:= y
+UBSAN_SANITIZE_vma.o			:= y
-KCSAN_SANITIZE_vma.o		:= y
+KCSAN_SANITIZE_vma.o			:= y
-OBJECT_FILES_NON_STANDARD_vma.o	:= n
+OBJECT_FILES_NON_STANDARD_vma.o		:= n
+OBJECT_FILES_NON_STANDARD_extable.o	:= n
 # vDSO images to build
 vdso_img-$(VDSO64-y)		+= 64
 endif
 endif
-$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
+$(vobjs): KBUILD_CFLAGS := $(filter-out $(PADDING_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
 $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
 #
 KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))
+KBUILD_CFLAGS_32 := $(filter-out $(PADDING_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic
 KBUILD_CFLAGS_32 += -fno-stack-protector
 KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)

arch/x86/include/asm/alternative.h+65−3

@@ -78,8 +78,43 @@ extern void apply_alternatives(struct alt_instr start, struct alt_instr end); · 77 unmodified lines
78	extern void apply_retpolines(s32 start, s32 end);	78	extern void apply_retpolines(s32 start, s32 end);
79	extern void apply_returns(s32 start, s32 end);	79	extern void apply_returns(s32 start, s32 end);
80	extern void apply_ibt_endbr(s32 start, s32 end);	80	extern void apply_ibt_endbr(s32 start, s32 end);
		81	extern void apply_fineibt(s32 start_retpoline, s32 end_retpoine,
		82	s32 start_cfi, s32 end_cfi);
81		83
82	struct module;	84	struct module;
		85	struct paravirt_patch_site;
		86
		87	struct callthunk_sites {
		88	s32 call_start, call_end;
		89	struct paravirt_patch_site pv_start, pv_end;
		90	};
		91
		92	#ifdef CONFIG_CALL_THUNKS
		93	extern void callthunks_patch_builtin_calls(void);
		94	extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
		95	struct module *mod);
		96	extern void callthunks_translate_call_dest(void dest);
		97	extern bool is_callthunk(void *addr);
		98	extern int x86_call_depth_emit_accounting(u8 *pprog, void func);
		99	#else
		100	static __always_inline void callthunks_patch_builtin_calls(void) {}
		101	static __always_inline void
		102	callthunks_patch_module_calls(struct callthunk_sites *sites,
		103	struct module *mod) {}
		104	static __always_inline void callthunks_translate_call_dest(void dest)
		105	{
		106	return dest;
		107	}
		108	static __always_inline bool is_callthunk(void *addr)
		109	{
		110	return false;
		111	}
		112	static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
		113	void *func)
		114	{
		115	return 0;
		116	}
		117	#endif
83		118
84	#ifdef CONFIG_SMP	119	#ifdef CONFIG_SMP
85	extern void alternatives_smp_module_add(struct module mod, char name,	120	extern void alternatives_smp_module_add(struct module mod, char name,
@@ -347,6 +382,7 @@ static inline int alternatives_text_reserved(void start, void end) · 261 unmodified lines
347	#define old_len 141b-140b	382	#define old_len 141b-140b
348	#define new_len1 144f-143f	383	#define new_len1 144f-143f
349	#define new_len2 145f-144f	384	#define new_len2 145f-144f
		385	#define new_len3 146f-145f
350		386
351	/*	387	/*
352	* gas compatible max based on the idea from:	388	* gas compatible max based on the idea from:
@@ -354,7 +390,8 @@ static inline int alternatives_text_reserved(void start, void end) · 1 unmodified line
354	*	390	*
355	* The additional "-" is needed because gas uses a "true" value of -1.	391	* The additional "-" is needed because gas uses a "true" value of -1.
356	*/	392	*/
357	#define alt_max_short(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))	393	#define alt_max_2(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
		394	#define alt_max_3(a, b, c) (alt_max_2(alt_max_2(a, b), c))
358		395
359		396
360	/*	397	/*
@@ -366,13 +403,36 @@ static inline int alternatives_text_reserved(void start, void end) · 5 unmodified lines
366	140:	403	140:
367	\oldinstr	404	\oldinstr
368	141:	405	141:
369	.skip -((alt_max_short(new_len1, new_len2) - (old_len)) > 0) * \	406	.skip -((alt_max_2(new_len1, new_len2) - (old_len)) > 0) * \
370	(alt_max_short(new_len1, new_len2) - (old_len)),0x90	407	(alt_max_2(new_len1, new_len2) - (old_len)),0x90
		408	142:
		409
		410	.pushsection .altinstructions,"a"
		411	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
		412	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
		413	.popsection
		414
		415	.pushsection .altinstr_replacement,"ax"
		416	143:
		417	\newinstr1
		418	144:
		419	\newinstr2
		420	145:
		421	.popsection
		422	.endm
		423
		424	.macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3
		425	140:
		426	\oldinstr
		427	141:
		428	.skip -((alt_max_3(new_len1, new_len2, new_len3) - (old_len)) > 0) * \
		429	(alt_max_3(new_len1, new_len2, new_len3) - (old_len)),0x90
371	142:	430	142:
372		431
373	.pushsection .altinstructions,"a"	432	.pushsection .altinstructions,"a"
374	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f	433	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
375	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f	434	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
		435	altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f
376	.popsection	436	.popsection
377		437
378	.pushsection .altinstr_replacement,"ax"	438	.pushsection .altinstr_replacement,"ax"
@@ -381,6 +441,8 @@ static inline int alternatives_text_reserved(void start, void end) · 2 unmodified lines
381	144:	441	144:
382	\newinstr2	442	\newinstr2
383	145:	443	145:
		444	\newinstr3
		445	146:
384	.popsection	446	.popsection
385	.endm	447	.endm
386		448

arch/x86/include/asm/cpufeatures.h+1−2

@@ -305,8 +305,7 @@ · 304 unmodified lines
305	#define X86_FEATURE_USE_IBPB_FW (1132+16) / "" Use IBPB during runtime firmware calls */	305	#define X86_FEATURE_USE_IBPB_FW (1132+16) / "" Use IBPB during runtime firmware calls */
306	#define X86_FEATURE_RSB_VMEXIT_LITE (1132+17) / "" Fill RSB on VM exit when EIBRS is enabled */	306	#define X86_FEATURE_RSB_VMEXIT_LITE (1132+17) / "" Fill RSB on VM exit when EIBRS is enabled */
307	#define X86_FEATURE_SGX_EDECCSSA (1132+18) / "" SGX EDECCSSA user leaf function */	307	#define X86_FEATURE_SGX_EDECCSSA (1132+18) / "" SGX EDECCSSA user leaf function */
308		308	#define X86_FEATURE_CALL_DEPTH (1132+19) / "" Call depth tracking for RSB stuffing */
309
310	#define X86_FEATURE_MSR_TSX_CTRL (1132+20) / "" MSR IA32_TSX_CTRL (Intel) implemented */	309	#define X86_FEATURE_MSR_TSX_CTRL (1132+20) / "" MSR IA32_TSX_CTRL (Intel) implemented */
311		310
312	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */	311	/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */

arch/x86/include/asm/current.h+29−3

@@ -3,16 +3,42 @@ · 2 unmodified lines
3	#define _ASM_X86_CURRENT_H	3	#define _ASM_X86_CURRENT_H
4		4
5	#include <linux/compiler.h>	5	#include <linux/compiler.h>
6	#include <asm/percpu.h>
7		6
8	#ifndef __ASSEMBLY__	7	#ifndef __ASSEMBLY__
		8
		9	#include <linux/cache.h>
		10	#include <asm/percpu.h>
		11
9	struct task_struct;	12	struct task_struct;
10		13
11	DECLARE_PER_CPU(struct task_struct *, current_task);	14	struct pcpu_hot {
		15	union {
		16	struct {
		17	struct task_struct *current_task;
		18	int preempt_count;
		19	int cpu_number;
		20	#ifdef CONFIG_CALL_DEPTH_TRACKING
		21	u64 call_depth;
		22	#endif
		23	unsigned long top_of_stack;
		24	void *hardirq_stack_ptr;
		25	u16 softirq_pending;
		26	#ifdef CONFIG_X86_64
		27	bool hardirq_stack_inuse;
		28	#else
		29	void *softirq_stack_ptr;
		30	#endif
		31	};
		32	u8 pad[64];
		33	};
		34	};
		35	static_assert(sizeof(struct pcpu_hot) == 64);
		36
		37	DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
12		38
13	static __always_inline struct task_struct *get_current(void)	39	static __always_inline struct task_struct *get_current(void)
14	{	40	{
15	return this_cpu_read_stable(current_task);	41	return this_cpu_read_stable(pcpu_hot.current_task);
16	}	42	}
17		43
18	#define current get_current()	44	#define current get_current()

arch/x86/include/asm/debugreg.h+1−1

@@ -2,8 +2,8 @@ · 1 unmodified line
2	#ifndef _ASM_X86_DEBUGREG_H	2	#ifndef _ASM_X86_DEBUGREG_H
3	#define _ASM_X86_DEBUGREG_H	3	#define _ASM_X86_DEBUGREG_H
4		4
5
6	#include <linux/bug.h>	5	#include <linux/bug.h>
		6	#include <linux/percpu.h>
7	#include <uapi/asm/debugreg.h>	7	#include <uapi/asm/debugreg.h>
8		8
9	DECLARE_PER_CPU(unsigned long, cpu_dr7);	9	DECLARE_PER_CPU(unsigned long, cpu_dr7);

arch/x86/include/asm/disabled-features.h+8−1

 # define DISABLE_UNRET		(1 << (X86_FEATURE_UNRET & 31))
 #endif
+#ifdef CONFIG_CALL_DEPTH_TRACKING
+# define DISABLE_CALL_DEPTH_TRACKING	0
+#else
+# define DISABLE_CALL_DEPTH_TRACKING	(1 << (X86_FEATURE_CALL_DEPTH & 31))
+#endif
 #ifdef CONFIG_INTEL_IOMMU_SVM
 # define DISABLE_ENQCMD		0
 #else
 #define DISABLED_MASK8	(DISABLE_XENPV|DISABLE_TDX_GUEST)
 #define DISABLED_MASK9	(DISABLE_SGX)
 #define DISABLED_MASK10	0
-#define DISABLED_MASK11	(DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET)
+#define DISABLED_MASK11	(DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
+			 DISABLE_CALL_DEPTH_TRACKING)
 #define DISABLED_MASK12	0
 #define DISABLED_MASK13	0
 #define DISABLED_MASK14	0

arch/x86/include/asm/hardirq.h+2−1

 #define _ASM_X86_HARDIRQ_H
 #include <linux/threads.h>
+#include <asm/current.h>
 typedef struct {
-	u16	     __softirq_pending;
 #if IS_ENABLED(CONFIG_KVM_INTEL)
 	u8	     kvm_cpu_l1tf_flush_l1d;
 #endif
 extern u64 arch_irq_stat(void);
 #define arch_irq_stat		arch_irq_stat
+#define local_softirq_pending_ref       pcpu_hot.softirq_pending
 #if IS_ENABLED(CONFIG_KVM_INTEL)
 static inline void kvm_set_cpu_l1tf_flush_l1d(void)

arch/x86/include/asm/irq_stack.h+6−6

@@ -116,7 +116,7 @@ · 115 unmodified lines
116	ASM_CALL_ARG2	116	ASM_CALL_ARG2
117		117
118	#define call_on_irqstack(func, asm_call, argconstr...) \	118	#define call_on_irqstack(func, asm_call, argconstr...) \
119	call_on_stack(__this_cpu_read(hardirq_stack_ptr), \	119	call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \
120	func, asm_call, argconstr)	120	func, asm_call, argconstr)
121		121
122	/* Macros to assert type correctness for run__on_irqstack macros /	122	/* Macros to assert type correctness for run__on_irqstack macros /
@@ -135,7 +135,7 @@ · 12 unmodified lines
135	* User mode entry and interrupt on the irq stack do not \	135	* User mode entry and interrupt on the irq stack do not \
136	* switch stacks. If from user mode the task stack is empty. \	136	* switch stacks. If from user mode the task stack is empty. \
137	*/ \	137	*/ \
138	if (user_mode(regs) \|\| __this_cpu_read(hardirq_stack_inuse)) { \	138	if (user_mode(regs) \|\| __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
139	irq_enter_rcu(); \	139	irq_enter_rcu(); \
140	func(c_args); \	140	func(c_args); \
141	irq_exit_rcu(); \	141	irq_exit_rcu(); \
@@ -146,9 +146,9 @@ · 4 unmodified lines
146	* places. Invoke the stack switch macro with the call \	146	* places. Invoke the stack switch macro with the call \
147	* sequence which matches the above direct invocation. \	147	* sequence which matches the above direct invocation. \
148	*/ \	148	*/ \
149	__this_cpu_write(hardirq_stack_inuse, true); \	149	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
150	call_on_irqstack(func, asm_call, constr); \	150	call_on_irqstack(func, asm_call, constr); \
151	__this_cpu_write(hardirq_stack_inuse, false); \	151	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
152	} \	152	} \
153	}	153	}
154		154
@@ -212,9 +212,9 @@ · 57 unmodified lines
212	*/	212	*/
213	#define do_softirq_own_stack() \	213	#define do_softirq_own_stack() \
214	{ \	214	{ \
215	__this_cpu_write(hardirq_stack_inuse, true); \	215	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \
216	call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \	216	call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \
217	__this_cpu_write(hardirq_stack_inuse, false); \	217	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
218	}	218	}
219		219
220	#endif	220	#endif

arch/x86/include/asm/linkage.h+55−8

@@ -12,13 +12,26 @@ · 11 unmodified lines
12	#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))	12	#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
13	#endif /* CONFIG_X86_32 */	13	#endif /* CONFIG_X86_32 */
14		14
15	#ifdef __ASSEMBLY__	15	#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT, 0x90;
16
17	#if defined(CONFIG_X86_64) \|\| defined(CONFIG_X86_ALIGNMENT_16)
18	#define __ALIGN .p2align 4, 0x90
19	#define __ALIGN_STR __stringify(__ALIGN)	16	#define __ALIGN_STR __stringify(__ALIGN)
		17
		18	#if defined(CONFIG_CALL_PADDING) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
		19	#define FUNCTION_PADDING .skip CONFIG_FUNCTION_ALIGNMENT, 0x90;
		20	#else
		21	#define FUNCTION_PADDING
		22	#endif
		23
		24	#if (CONFIG_FUNCTION_ALIGNMENT > 8) && !defined(__DISABLE_EXPORTS) && !defined(BULID_VDSO)
		25	# define __FUNC_ALIGN __ALIGN; FUNCTION_PADDING
		26	#else
		27	# define __FUNC_ALIGN __ALIGN
20	#endif	28	#endif
21		29
		30	#define ASM_FUNC_ALIGN __stringify(__FUNC_ALIGN)
		31	#define SYM_F_ALIGN __FUNC_ALIGN
		32
		33	#ifdef __ASSEMBLY__
		34
22	#if defined(CONFIG_RETHUNK) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)	35	#if defined(CONFIG_RETHUNK) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)
23	#define RET jmp __x86_return_thunk	36	#define RET jmp __x86_return_thunk
24	#else /* CONFIG_RETPOLINE */	37	#else /* CONFIG_RETPOLINE */
@@ -43,11 +56,45 @@ · 18 unmodified lines
43		56
44	#endif /* __ASSEMBLY__ */	57	#endif /* __ASSEMBLY__ */
45		58
		59	/*
		60	* Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_PADDING) the
		61	* CFI symbol layout changes.
		62	*
		63	* Without CALL_THUNKS:
		64	*
		65	* .align FUNCTION_ALIGNMENT
		66	* __cfi_##name:
		67	* .skip FUNCTION_PADDING, 0x90
		68	* .byte 0xb8
		69	* .long __kcfi_typeid_##name
		70	* name:
		71	*
		72	* With CALL_THUNKS:
		73	*
		74	* .align FUNCTION_ALIGNMENT
		75	* __cfi_##name:
		76	* .byte 0xb8
		77	* .long __kcfi_typeid_##name
		78	* .skip FUNCTION_PADDING, 0x90
		79	* name:
		80	*
		81	* In both cases the whole thing is FUNCTION_ALIGNMENT aligned and sized.
		82	*/
		83
		84	#ifdef CONFIG_CALL_PADDING
		85	#define CFI_PRE_PADDING
		86	#define CFI_POST_PADDING .skip CONFIG_FUNCTION_PADDING_BYTES, 0x90;
		87	#else
		88	#define CFI_PRE_PADDING .skip CONFIG_FUNCTION_PADDING_BYTES, 0x90;
		89	#define CFI_POST_PADDING
		90	#endif
		91
46	#define __CFI_TYPE(name) \	92	#define __CFI_TYPE(name) \
47	SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \	93	SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \
48	.fill 11, 1, 0x90 ASM_NL \	94	CFI_PRE_PADDING \
49	.byte 0xb8 ASM_NL \	95	.byte 0xb8 ASM_NL \
50	.long __kcfi_typeid_##name ASM_NL \	96	.long __kcfi_typeid_##name ASM_NL \
		97	CFI_POST_PADDING \
51	SYM_FUNC_END(__cfi_##name)	98	SYM_FUNC_END(__cfi_##name)
52		99
53	/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */	100	/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
@@ -57,7 +104,7 @@ · 3 unmodified lines
57		104
58	/* SYM_FUNC_START -- use for global functions */	105	/* SYM_FUNC_START -- use for global functions */
59	#define SYM_FUNC_START(name) \	106	#define SYM_FUNC_START(name) \
60	SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \	107	SYM_START(name, SYM_L_GLOBAL, SYM_F_ALIGN) \
61	ENDBR	108	ENDBR
62		109
63	/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */	110	/* SYM_FUNC_START_NOALIGN -- use for global functions, w/o alignment */
@@ -67,7 +114,7 @@ · 3 unmodified lines
67		114
68	/* SYM_FUNC_START_LOCAL -- use for local functions */	115	/* SYM_FUNC_START_LOCAL -- use for local functions */
69	#define SYM_FUNC_START_LOCAL(name) \	116	#define SYM_FUNC_START_LOCAL(name) \
70	SYM_START(name, SYM_L_LOCAL, SYM_A_ALIGN) \	117	SYM_START(name, SYM_L_LOCAL, SYM_F_ALIGN) \
71	ENDBR	118	ENDBR
72		119
73	/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */	120	/* SYM_FUNC_START_LOCAL_NOALIGN -- use for local functions, w/o alignment */
@@ -77,7 +124,7 @@ · 3 unmodified lines
77		124
78	/* SYM_FUNC_START_WEAK -- use for weak functions */	125	/* SYM_FUNC_START_WEAK -- use for weak functions */
79	#define SYM_FUNC_START_WEAK(name) \	126	#define SYM_FUNC_START_WEAK(name) \
80	SYM_START(name, SYM_L_WEAK, SYM_A_ALIGN) \	127	SYM_START(name, SYM_L_WEAK, SYM_F_ALIGN) \
81	ENDBR	128	ENDBR
82		129
83	/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */	130	/* SYM_FUNC_START_WEAK_NOALIGN -- use for weak functions, w/o alignment */

arch/x86/include/asm/nospec-branch.h+170−6

@@ -12,8 +12,104 @@ · 11 unmodified lines
12	#include <asm/msr-index.h>	12	#include <asm/msr-index.h>
13	#include <asm/unwind_hints.h>	13	#include <asm/unwind_hints.h>
14	#include <asm/percpu.h>	14	#include <asm/percpu.h>
		15	#include <asm/current.h>
15		16
16	#define RETPOLINE_THUNK_SIZE 32	17	/*
		18	* Call depth tracking for Intel SKL CPUs to address the RSB underflow
		19	* issue in software.
		20	*
		21	* The tracking does not use a counter. It uses uses arithmetic shift
		22	* right on call entry and logical shift left on return.
		23	*
		24	* The depth tracking variable is initialized to 0x8000.... when the call
		25	* depth is zero. The arithmetic shift right sign extends the MSB and
		26	* saturates after the 12th call. The shift count is 5 for both directions
		27	* so the tracking covers 12 nested calls.
		28	*
		29	* Call
		30	* 0: 0x8000000000000000 0x0000000000000000
		31	* 1: 0xfc00000000000000 0xf000000000000000
		32	* ...
		33	* 11: 0xfffffffffffffff8 0xfffffffffffffc00
		34	* 12: 0xffffffffffffffff 0xffffffffffffffe0
		35	*
		36	* After a return buffer fill the depth is credited 12 calls before the
		37	* next stuffing has to take place.
		38	*
		39	* There is a inaccuracy for situations like this:
		40	*
		41	* 10 calls
		42	* 5 returns
		43	* 3 calls
		44	* 4 returns
		45	* 3 calls
		46	* ....
		47	*
		48	* The shift count might cause this to be off by one in either direction,
		49	* but there is still a cushion vs. the RSB depth. The algorithm does not
		50	* claim to be perfect and it can be speculated around by the CPU, but it
		51	* is considered that it obfuscates the problem enough to make exploitation
		52	* extremly difficult.
		53	*/
		54	#define RET_DEPTH_SHIFT 5
		55	#define RSB_RET_STUFF_LOOPS 16
		56	#define RET_DEPTH_INIT 0x8000000000000000ULL
		57	#define RET_DEPTH_INIT_FROM_CALL 0xfc00000000000000ULL
		58	#define RET_DEPTH_CREDIT 0xffffffffffffffffULL
		59
		60	#ifdef CONFIG_CALL_THUNKS_DEBUG
		61	# define CALL_THUNKS_DEBUG_INC_CALLS \
		62	incq %gs:__x86_call_count;
		63	# define CALL_THUNKS_DEBUG_INC_RETS \
		64	incq %gs:__x86_ret_count;
		65	# define CALL_THUNKS_DEBUG_INC_STUFFS \
		66	incq %gs:__x86_stuffs_count;
		67	# define CALL_THUNKS_DEBUG_INC_CTXSW \
		68	incq %gs:__x86_ctxsw_count;
		69	#else
		70	# define CALL_THUNKS_DEBUG_INC_CALLS
		71	# define CALL_THUNKS_DEBUG_INC_RETS
		72	# define CALL_THUNKS_DEBUG_INC_STUFFS
		73	# define CALL_THUNKS_DEBUG_INC_CTXSW
		74	#endif
		75
		76	#if defined(CONFIG_CALL_DEPTH_TRACKING) && !defined(COMPILE_OFFSETS)
		77
		78	#include <asm/asm-offsets.h>
		79
		80	#define CREDIT_CALL_DEPTH \
		81	movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
		82
		83	#define ASM_CREDIT_CALL_DEPTH \
		84	movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
		85
		86	#define RESET_CALL_DEPTH \
		87	mov $0x80, %rax; \
		88	shl $56, %rax; \
		89	movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
		90
		91	#define RESET_CALL_DEPTH_FROM_CALL \
		92	mov $0xfc, %rax; \
		93	shl $56, %rax; \
		94	movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
		95	CALL_THUNKS_DEBUG_INC_CALLS
		96
		97	#define INCREMENT_CALL_DEPTH \
		98	sarq $5, %gs:pcpu_hot + X86_call_depth; \
		99	CALL_THUNKS_DEBUG_INC_CALLS
		100
		101	#define ASM_INCREMENT_CALL_DEPTH \
		102	sarq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
		103	CALL_THUNKS_DEBUG_INC_CALLS
		104
		105	#else
		106	#define CREDIT_CALL_DEPTH
		107	#define ASM_CREDIT_CALL_DEPTH
		108	#define RESET_CALL_DEPTH
		109	#define INCREMENT_CALL_DEPTH
		110	#define ASM_INCREMENT_CALL_DEPTH
		111	#define RESET_CALL_DEPTH_FROM_CALL
		112	#endif
17		113
18	/*	114	/*
19	* Fill the CPU return stack buffer.	115	* Fill the CPU return stack buffer.
@@ -32,6 +128,7 @@ · 12 unmodified lines
32	* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.	128	* from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
33	*/	129	*/
34		130
		131	#define RETPOLINE_THUNK_SIZE 32
35	#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */	132	#define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
36		133
37	/*	134	/*
@@ -60,7 +157,9 @@ · 22 unmodified lines
60	dec reg; \	157	dec reg; \
61	jnz 771b; \	158	jnz 771b; \
62	/* barrier for jnz misprediction */ \	159	/* barrier for jnz misprediction */ \
63	lfence;	160	lfence; \
		161	ASM_CREDIT_CALL_DEPTH \
		162	CALL_THUNKS_DEBUG_INC_CTXSW
64	#else	163	#else
65	/*	164	/*
66	* i386 doesn't unconditionally have LFENCE, as such it can't	165	* i386 doesn't unconditionally have LFENCE, as such it can't
@@ -185,11 +284,32 @@ · 118 unmodified lines
185	* where we have a stack but before any RET instruction.	284	* where we have a stack but before any RET instruction.
186	*/	285	*/
187	.macro UNTRAIN_RET	286	.macro UNTRAIN_RET
188	#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY)	287	#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY) \|\| \
		288	defined(CONFIG_CALL_DEPTH_TRACKING)
189	ANNOTATE_UNRET_END	289	ANNOTATE_UNRET_END
190	ALTERNATIVE_2 "", \	290	ALTERNATIVE_3 "", \
191	CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \	291	CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
192	"call entry_ibpb", X86_FEATURE_ENTRY_IBPB	292	"call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
		293	__stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
		294	#endif
		295	.endm
		296
		297	.macro UNTRAIN_RET_FROM_CALL
		298	#if defined(CONFIG_CPU_UNRET_ENTRY) \|\| defined(CONFIG_CPU_IBPB_ENTRY) \|\| \
		299	defined(CONFIG_CALL_DEPTH_TRACKING)
		300	ANNOTATE_UNRET_END
		301	ALTERNATIVE_3 "", \
		302	CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
		303	"call entry_ibpb", X86_FEATURE_ENTRY_IBPB, \
		304	__stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH
		305	#endif
		306	.endm
		307
		308
		309	.macro CALL_DEPTH_ACCOUNT
		310	#ifdef CONFIG_CALL_DEPTH_TRACKING
		311	ALTERNATIVE "", \
		312	__stringify(ASM_INCREMENT_CALL_DEPTH), X86_FEATURE_CALL_DEPTH
193	#endif	313	#endif
194	.endm	314	.endm
195		315
@@ -203,11 +323,45 @@ · 7 unmodified lines
203		323
204	typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];	324	typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
205	extern retpoline_thunk_t __x86_indirect_thunk_array[];	325	extern retpoline_thunk_t __x86_indirect_thunk_array[];
		326	extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
		327	extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
206		328
207	extern void __x86_return_thunk(void);	329	extern void __x86_return_thunk(void);
208	extern void zen_untrain_ret(void);	330	extern void zen_untrain_ret(void);
209	extern void entry_ibpb(void);	331	extern void entry_ibpb(void);
210		332
		333	#ifdef CONFIG_CALL_THUNKS
		334	extern void (*x86_return_thunk)(void);
		335	#else
		336	#define x86_return_thunk (&__x86_return_thunk)
		337	#endif
		338
		339	#ifdef CONFIG_CALL_DEPTH_TRACKING
		340	extern void __x86_return_skl(void);
		341
		342	static inline void x86_set_skl_return_thunk(void)
		343	{
		344	x86_return_thunk = &__x86_return_skl;
		345	}
		346
		347	#define CALL_DEPTH_ACCOUNT \
		348	ALTERNATIVE("", \
		349	__stringify(INCREMENT_CALL_DEPTH), \
		350	X86_FEATURE_CALL_DEPTH)
		351
		352	#ifdef CONFIG_CALL_THUNKS_DEBUG
		353	DECLARE_PER_CPU(u64, __x86_call_count);
		354	DECLARE_PER_CPU(u64, __x86_ret_count);
		355	DECLARE_PER_CPU(u64, __x86_stuffs_count);
		356	DECLARE_PER_CPU(u64, __x86_ctxsw_count);
		357	#endif
		358	#else
		359	static inline void x86_set_skl_return_thunk(void) {}
		360
		361	#define CALL_DEPTH_ACCOUNT ""
		362
		363	#endif
		364
211	#ifdef CONFIG_RETPOLINE	365	#ifdef CONFIG_RETPOLINE
212		366
213	#define GEN(reg) \	367	#define GEN(reg) \
@@ -215,6 +369,16 @@ extern void entry_ibpb(void); · 1 unmodified line
215	#include <asm/GEN-for-each-reg.h>	369	#include <asm/GEN-for-each-reg.h>
216	#undef GEN	370	#undef GEN
217		371
		372	#define GEN(reg) \
		373	extern retpoline_thunk_t __x86_indirect_call_thunk_ ## reg;
		374	#include <asm/GEN-for-each-reg.h>
		375	#undef GEN
		376
		377	#define GEN(reg) \
		378	extern retpoline_thunk_t __x86_indirect_jump_thunk_ ## reg;
		379	#include <asm/GEN-for-each-reg.h>
		380	#undef GEN
		381
218	#ifdef CONFIG_X86_64	382	#ifdef CONFIG_X86_64
219		383
220	/*	384	/*

arch/x86/include/asm/paravirt.h+15−2

 /* Various instructions on x86 need to be replaced for
  * para-virtualization: those hooks are defined here. */
+#include <asm/paravirt_types.h>
 #ifdef CONFIG_PARAVIRT
 #include <asm/pgtable_types.h>
 #include <asm/asm.h>
 #include <asm/nospec-branch.h>
-#include <asm/paravirt_types.h>
 #ifndef __ASSEMBLY__
 #include <linux/bug.h>
 #include <linux/types.h>
 	asm(".pushsection " section ", \"ax\";"				\
 	    ".globl " PV_THUNK_NAME(func) ";"				\
 	    ".type " PV_THUNK_NAME(func) ", @function;"			\
+	    ASM_FUNC_ALIGN						\
 	    PV_THUNK_NAME(func) ":"					\
 	    ASM_ENDBR							\
 	    FRAME_BEGIN							\
 #undef PVOP_VCALL4
 #undef PVOP_CALL4
+#define DEFINE_PARAVIRT_ASM(func, instr, sec)		\
+	asm (".pushsection " #sec ", \"ax\"\n"		\
+	     ".global " #func "\n\t"			\
+	     ".type " #func ", @function\n\t"		\
+	     ASM_FUNC_ALIGN "\n"			\
+	     #func ":\n\t"				\
+	     ASM_ENDBR					\
+	     instr "\n\t"				\
+	     ASM_RET					\
+	     ".size " #func ", . - " #func "\n\t"	\
+	     ".popsection")
 extern void default_banner(void);
 #else  /* __ASSEMBLY__ */

arch/x86/include/asm/paravirt_types.h+19−15

@@ -2,6 +2,24 @@ · 1 unmodified line
2	#ifndef _ASM_X86_PARAVIRT_TYPES_H	2	#ifndef _ASM_X86_PARAVIRT_TYPES_H
3	#define _ASM_X86_PARAVIRT_TYPES_H	3	#define _ASM_X86_PARAVIRT_TYPES_H
4		4
		5	#ifndef __ASSEMBLY__
		6	/* These all sit in the .parainstructions section to tell us what to patch. */
		7	struct paravirt_patch_site {
		8	u8 instr; / original instructions */
		9	u8 type; /* type of this instruction */
		10	u8 len; /* length of original instruction */
		11	};
		12
		13	/* Lazy mode for batching updates / context switch */
		14	enum paravirt_lazy_mode {
		15	PARAVIRT_LAZY_NONE,
		16	PARAVIRT_LAZY_MMU,
		17	PARAVIRT_LAZY_CPU,
		18	};
		19	#endif
		20
		21	#ifdef CONFIG_PARAVIRT
		22
5	#ifndef __ASSEMBLY__	23	#ifndef __ASSEMBLY__
6		24
7	#include <asm/desc_defs.h>	25	#include <asm/desc_defs.h>
@@ -534,13 +552,6 @@ int paravirt_disable_iospace(void); · 526 unmodified lines
534	__PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \	552	__PVOP_VCALL(op, PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
535	PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))	553	PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
536		554
537	/* Lazy mode for batching updates / context switch */
538	enum paravirt_lazy_mode {
539	PARAVIRT_LAZY_NONE,
540	PARAVIRT_LAZY_MMU,
541	PARAVIRT_LAZY_CPU,
542	};
543
544	enum paravirt_lazy_mode paravirt_get_lazy_mode(void);	555	enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
545	void paravirt_start_context_switch(struct task_struct *prev);	556	void paravirt_start_context_switch(struct task_struct *prev);
546	void paravirt_end_context_switch(struct task_struct *next);	557	void paravirt_end_context_switch(struct task_struct *next);
@@ -556,16 +567,9 @@ unsigned long paravirt_ret0(void); · 9 unmodified lines
556		567
557	#define paravirt_nop ((void *)_paravirt_nop)	568	#define paravirt_nop ((void *)_paravirt_nop)
558		569
559	/* These all sit in the .parainstructions section to tell us what to patch. */
560	struct paravirt_patch_site {
561	u8 instr; / original instructions */
562	u8 type; /* type of this instruction */
563	u8 len; /* length of original instruction */
564	};
565
566	extern struct paravirt_patch_site __parainstructions[],	570	extern struct paravirt_patch_site __parainstructions[],
567	__parainstructions_end[];	571	__parainstructions_end[];
568		572
569	#endif /* __ASSEMBLY__ */	573	#endif /* __ASSEMBLY__ */
570		574	#endif /* CONFIG_PARAVIRT */
571	#endif /* _ASM_X86_PARAVIRT_TYPES_H */	575	#endif /* _ASM_X86_PARAVIRT_TYPES_H */

arch/x86/include/asm/preempt.h+14−13

 #include <asm/rmwcc.h>
 #include <asm/percpu.h>
+#include <asm/current.h>
 #include <linux/thread_info.h>
 #include <linux/static_call_types.h>
-DECLARE_PER_CPU(int, __preempt_count);
 /* We use the MSB mostly because its available */
 #define PREEMPT_NEED_RESCHED	0x80000000
  */
 static __always_inline int preempt_count(void)
 {
-	return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
+	return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED;
 }
 static __always_inline void preempt_count_set(int pc)
 	int old, new;
 	do {
-		old = raw_cpu_read_4(__preempt_count);
+		old = raw_cpu_read_4(pcpu_hot.preempt_count);
 		new = (old & PREEMPT_NEED_RESCHED) |
 			(pc & ~PREEMPT_NEED_RESCHED);
-	} while (raw_cpu_cmpxchg_4(__preempt_count, old, new) != old);
+	} while (raw_cpu_cmpxchg_4(pcpu_hot.preempt_count, old, new) != old);
 }
 /*
 #define init_task_preempt_count(p) do { } while (0)
 #define init_idle_preempt_count(p, cpu) do { \
-	per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
+	per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \
 } while (0)
 /*
 static __always_inline void set_preempt_need_resched(void)
 {
-	raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
+	raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED);
 }
 static __always_inline void clear_preempt_need_resched(void)
 {
-	raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
+	raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED);
 }
 static __always_inline bool test_preempt_need_resched(void)
 {
-	return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
+	return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED);
 }
 /*
 static __always_inline void __preempt_count_add(int val)
 {
-	raw_cpu_add_4(__preempt_count, val);
+	raw_cpu_add_4(pcpu_hot.preempt_count, val);
 }
 static __always_inline void __preempt_count_sub(int val)
 {
-	raw_cpu_add_4(__preempt_count, -val);
+	raw_cpu_add_4(pcpu_hot.preempt_count, -val);
 }
 /*
  */
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
-	return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
+	return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e,
+			       __percpu_arg([var]));
 }
 /*
  */
 static __always_inline bool should_resched(int preempt_offset)
 {
-	return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
+	return unlikely(raw_cpu_read_4(pcpu_hot.preempt_count) == preempt_offset);
 }
 #ifdef CONFIG_PREEMPTION

arch/x86/include/asm/processor.h+2−9

 	char		stack[IRQ_STACK_SIZE];
 } __aligned(IRQ_STACK_SIZE);
-DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
 #ifdef CONFIG_X86_64
 struct fixed_percpu_data {
 	/*
 	return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu);
 }
-DECLARE_PER_CPU(void *, hardirq_stack_ptr);
-DECLARE_PER_CPU(bool, hardirq_stack_inuse);
 extern asmlinkage void ignore_sysret(void);
 /* Save actual FS/GS selectors and bases to current->thread */
 #ifdef CONFIG_STACKPROTECTOR
 DECLARE_PER_CPU(unsigned long, __stack_chk_guard);
 #endif
-DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
-DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
 #endif	/* !X86_64 */
 struct perf_event;
 	 *  and around vm86 mode and sp0 on x86_64 is special because of the
 	 *  entry trampoline.
 	 */
-	return this_cpu_read_stable(cpu_current_top_of_stack);
+	return this_cpu_read_stable(pcpu_hot.top_of_stack);
 }
 static __always_inline bool on_thread_stack(void)
 /* Defined in head.S */
 extern struct desc_ptr		early_gdt_descr;
-extern void switch_to_new_gdt(int);
+extern void switch_gdt_and_percpu_base(int);
 extern void load_direct_gdt(int);
 extern void load_fixmap_gdt(int);
-extern void load_percpu_segment(int);
 extern void cpu_init(void);
 extern void cpu_init_secondary(void);
 extern void cpu_init_exception_handling(void);

arch/x86/include/asm/qspinlock_paravirt.h+20−27

@@ -14,8 +14,6 @@ · 13 unmodified lines
14		14
15	__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");	15	__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
16	#define __pv_queued_spin_unlock __pv_queued_spin_unlock	16	#define __pv_queued_spin_unlock __pv_queued_spin_unlock
17	#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
18	#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
19		17
20	/*	18	/*
21	* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock	19	* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
@@ -37,32 +35,27 @@ __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text"); · 15 unmodified lines
37	* rsi = lockval (second argument)	35	* rsi = lockval (second argument)
38	* rdx = internal variable (set to 0)	36	* rdx = internal variable (set to 0)
39	*/	37	*/
40	asm (".pushsection .spinlock.text, \"ax\";"	38	#define PV_UNLOCK_ASM \
41	".globl " PV_UNLOCK ";"	39	FRAME_BEGIN \
42	".type " PV_UNLOCK ", @function;"	40	"push %rdx\n\t" \
43	".align 4,0x90;"	41	"mov $0x1,%eax\n\t" \
44	PV_UNLOCK ": "	42	"xor %edx,%edx\n\t" \
45	ASM_ENDBR	43	LOCK_PREFIX "cmpxchg %dl,(%rdi)\n\t" \
46	FRAME_BEGIN	44	"cmp $0x1,%al\n\t" \
47	"push %rdx;"	45	"jne .slowpath\n\t" \
48	"mov $0x1,%eax;"	46	"pop %rdx\n\t" \
49	"xor %edx,%edx;"	47	FRAME_END \
50	LOCK_PREFIX "cmpxchg %dl,(%rdi);"	48	ASM_RET \
51	"cmp $0x1,%al;"	49	".slowpath:\n\t" \
52	"jne .slowpath;"	50	"push %rsi\n\t" \
53	"pop %rdx;"	51	"movzbl %al,%esi\n\t" \
		52	"call __raw_callee_save___pv_queued_spin_unlock_slowpath\n\t" \
		53	"pop %rsi\n\t" \
		54	"pop %rdx\n\t" \
54	FRAME_END	55	FRAME_END
55	ASM_RET	56
56	".slowpath: "	57	DEFINE_PARAVIRT_ASM(__raw_callee_save___pv_queued_spin_unlock,
57	"push %rsi;"	58	PV_UNLOCK_ASM, .spinlock.text);
58	"movzbl %al,%esi;"
59	"call " PV_UNLOCK_SLOWPATH ";"
60	"pop %rsi;"
61	"pop %rdx;"
62	FRAME_END
63	ASM_RET
64	".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
65	".popsection");
66		59
67	#else /* CONFIG_64BIT */	60	#else /* CONFIG_64BIT */
68		61

arch/x86/include/asm/smp.h+5−7

 #define _ASM_X86_SMP_H
 #ifndef __ASSEMBLY__
 #include <linux/cpumask.h>
-#include <asm/percpu.h>
-#include <asm/thread_info.h>
 #include <asm/cpumask.h>
+#include <asm/current.h>
+#include <asm/thread_info.h>
 extern int smp_num_siblings;
 extern unsigned int num_processors;
 DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_l2c_shared_map);
 DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
 DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id);
-DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
 DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
 DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid);
 /*
  * This function is needed by all SMP systems. It must _always_ be valid
- * from the initial startup. We map APIC_BASE very early in page_setup(),
+ * from the initial startup.
- * so this is correct in the x86 case.
  */
-#define raw_smp_processor_id()  this_cpu_read(cpu_number)
+#define raw_smp_processor_id()  this_cpu_read(pcpu_hot.cpu_number)
-#define __smp_processor_id() __this_cpu_read(cpu_number)
+#define __smp_processor_id() __this_cpu_read(pcpu_hot.cpu_number)
 #ifdef CONFIG_X86_32
 extern int safe_smp_processor_id(void);

arch/x86/include/asm/text-patching.h+1−0

@@ -45,6 +45,7 @@ extern void text_poke(void addr, const void *opcode, size_t len); · 44 unmodified lines
45	extern void text_poke_sync(void);	45	extern void text_poke_sync(void);
46	extern void text_poke_kgdb(void addr, const void *opcode, size_t len);	46	extern void text_poke_kgdb(void addr, const void *opcode, size_t len);
47	extern void text_poke_copy(void addr, const void *opcode, size_t len);	47	extern void text_poke_copy(void addr, const void *opcode, size_t len);
		48	extern void text_poke_copy_locked(void addr, const void *opcode, size_t len, bool core_ok);
48	extern void text_poke_set(void addr, int c, size_t len);	49	extern void text_poke_set(void addr, int c, size_t len);
49	extern int poke_int3_handler(struct pt_regs *regs);	50	extern int poke_int3_handler(struct pt_regs *regs);
50	extern void text_poke_bp(void addr, const void opcode, size_t len, const void *emulate);	51	extern void text_poke_bp(void addr, const void opcode, size_t len, const void *emulate);

arch/x86/kernel/Makefile+2−0

@@ -143,6 +143,8 @@ obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o · 142 unmodified lines
143		143
144	obj-$(CONFIG_CFI_CLANG) += cfi.o	144	obj-$(CONFIG_CFI_CLANG) += cfi.o
145		145
		146	obj-$(CONFIG_CALL_THUNKS) += callthunks.o
		147
146	###	148	###
147	# 64 bit specific files	149	# 64 bit specific files
148	ifeq ($(CONFIG_X86_64),y)	150	ifeq ($(CONFIG_X86_64),y)

arch/x86/kernel/alternative.c+495−34

@@ -116,6 +116,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len) · 115 unmodified lines
116		116
117	extern s32 __retpoline_sites[], __retpoline_sites_end[];	117	extern s32 __retpoline_sites[], __retpoline_sites_end[];
118	extern s32 __return_sites[], __return_sites_end[];	118	extern s32 __return_sites[], __return_sites_end[];
		119	extern s32 __cfi_sites[], __cfi_sites_end[];
119	extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];	120	extern s32 __ibt_endbr_seal[], __ibt_endbr_seal_end[];
120	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];	121	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
121	extern s32 __smp_locks[], __smp_locks_end[];	122	extern s32 __smp_locks[], __smp_locks_end[];
@@ -377,6 +378,56 @@ static int emit_indirect(int op, int reg, u8 *bytes) · 255 unmodified lines
377	return i;	378	return i;
378	}	379	}
379		380
		381	static inline bool is_jcc32(struct insn *insn)
		382	{
		383	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */
		384	return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;
		385	}
		386
		387	static int emit_call_track_retpoline(void addr, struct insn insn, int reg, u8 *bytes)
		388	{
		389	u8 op = insn->opcode.bytes[0];
		390	int i = 0;
		391
		392	/*
		393	* Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
		394	* tail-calls. Deal with them.
		395	*/
		396	if (is_jcc32(insn)) {
		397	bytes[i++] = op;
		398	op = insn->opcode.bytes[1];
		399	goto clang_jcc;
		400	}
		401
		402	if (insn->length == 6)
		403	bytes[i++] = 0x2e; /* CS-prefix */
		404
		405	switch (op) {
		406	case CALL_INSN_OPCODE:
		407	__text_gen_insn(bytes+i, op, addr+i,
		408	__x86_indirect_call_thunk_array[reg],
		409	CALL_INSN_SIZE);
		410	i += CALL_INSN_SIZE;
		411	break;
		412
		413	case JMP32_INSN_OPCODE:
		414	clang_jcc:
		415	__text_gen_insn(bytes+i, op, addr+i,
		416	__x86_indirect_jump_thunk_array[reg],
		417	JMP32_INSN_SIZE);
		418	i += JMP32_INSN_SIZE;
		419	break;
		420
		421	default:
		422	WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
		423	return -1;
		424	}
		425
		426	WARN_ON_ONCE(i != insn->length);
		427
		428	return i;
		429	}
		430
380	/*	431	/*
381	* Rewrite the compiler generated retpoline thunk calls.	432	* Rewrite the compiler generated retpoline thunk calls.
382	*	433	*
@@ -409,8 +460,12 @@ static int patch_retpoline(void addr, struct insn insn, u8 *bytes) · 26 unmodified lines
409	BUG_ON(reg == 4);	460	BUG_ON(reg == 4);
410		461
411	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&	462	if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
412	!cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE))	463	!cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
		464	if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
		465	return emit_call_track_retpoline(addr, insn, reg, bytes);
		466
413	return -1;	467	return -1;
		468	}
414		469
415	op = insn->opcode.bytes[0];	470	op = insn->opcode.bytes[0];
416		471
@@ -427,8 +482,7 @@ static int patch_retpoline(void addr, struct insn insn, u8 *bytes) · 10 unmodified lines
427	* [ NOP ]	482	* [ NOP ]
428	* 1:	483	* 1:
429	*/	484	*/
430	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */	485	if (is_jcc32(insn)) {
431	if (op == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80) {
432	cc = insn->opcode.bytes[1] & 0xf;	486	cc = insn->opcode.bytes[1] & 0xf;
433	cc ^= 1; /* invert condition */	487	cc ^= 1; /* invert condition */
434		488
@@ -518,6 +572,11 @@ void __init_or_module noinline apply_retpolines(s32 start, s32 end) · 83 unmodified lines
518	}	572	}
519		573
520	#ifdef CONFIG_RETHUNK	574	#ifdef CONFIG_RETHUNK
		575
		576	#ifdef CONFIG_CALL_THUNKS
		577	void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;
		578	#endif
		579
521	/*	580	/*
522	* Rewrite the compiler generated return thunk tail-calls.	581	* Rewrite the compiler generated return thunk tail-calls.
523	*	582	*
@@ -533,14 +592,18 @@ static int patch_return(void addr, struct insn insn, u8 *bytes) · 9 unmodified lines
533	{	592	{
534	int i = 0;	593	int i = 0;
535		594
536	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))	595	if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
537	return -1;	596	if (x86_return_thunk == __x86_return_thunk)
		597	return -1;
538		598
539	bytes[i++] = RET_INSN_OPCODE;	599	i = JMP32_INSN_SIZE;
		600	__text_gen_insn(bytes, JMP32_INSN_OPCODE, addr, x86_return_thunk, i);
		601	} else {
		602	bytes[i++] = RET_INSN_OPCODE;
		603	}
540		604
541	for (; i < insn->length;)	605	for (; i < insn->length;)
542	bytes[i++] = INT3_INSN_OPCODE;	606	bytes[i++] = INT3_INSN_OPCODE;
543
544	return i;	607	return i;
545	}	608	}
546		609
@@ -594,6 +657,28 @@ void __init_or_module noinline apply_returns(s32 start, s32 end) { } · 47 unmodified lines
594		657
595	#ifdef CONFIG_X86_KERNEL_IBT	658	#ifdef CONFIG_X86_KERNEL_IBT
596		659
		660	static void poison_endbr(void *addr, bool warn)
		661	{
		662	u32 endbr, poison = gen_endbr_poison();
		663
		664	if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))
		665	return;
		666
		667	if (!is_endbr(endbr)) {
		668	WARN_ON_ONCE(warn);
		669	return;
		670	}
		671
		672	DPRINTK("ENDBR at: %pS (%px)", addr, addr);
		673
		674	/*
		675	* When we have IBT, the lack of ENDBR will trigger #CP
		676	*/
		677	DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);
		678	DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);
		679	text_poke_early(addr, &poison, 4);
		680	}
		681
597	/*	682	/*
598	* Generated by: objtool --ibt	683	* Generated by: objtool --ibt
599	*/	684	*/
@@ -602,31 +687,391 @@ void __init_or_module noinline apply_ibt_endbr(s32 start, s32 end) · 2 unmodified lines
602	s32 *s;	687	s32 *s;
603		688
604	for (s = start; s < end; s++) {	689	for (s = start; s < end; s++) {
605	u32 endbr, poison = gen_endbr_poison();
606	void addr = (void )s + *s;	690	void addr = (void )s + *s;
607		691
608	if (WARN_ON_ONCE(get_kernel_nofault(endbr, addr)))	692	poison_endbr(addr, true);
		693	if (IS_ENABLED(CONFIG_FINEIBT))
		694	poison_endbr(addr - 16, false);
		695	}
		696	}
		697
		698	#else
		699
		700	void __init_or_module apply_ibt_endbr(s32 start, s32 end) { }
		701
		702	#endif /* CONFIG_X86_KERNEL_IBT */
		703
		704	#ifdef CONFIG_FINEIBT
		705
		706	enum cfi_mode {
		707	CFI_DEFAULT,
		708	CFI_OFF,
		709	CFI_KCFI,
		710	CFI_FINEIBT,
		711	};
		712
		713	static enum cfi_mode cfi_mode __ro_after_init = CFI_DEFAULT;
		714	static bool cfi_rand __ro_after_init = true;
		715	static u32 cfi_seed __ro_after_init;
		716
		717	/*
		718	* Re-hash the CFI hash with a boot-time seed while making sure the result is
		719	* not a valid ENDBR instruction.
		720	*/
		721	static u32 cfi_rehash(u32 hash)
		722	{
		723	hash ^= cfi_seed;
		724	while (unlikely(is_endbr(hash) \|\| is_endbr(-hash))) {
		725	bool lsb = hash & 1;
		726	hash >>= 1;
		727	if (lsb)
		728	hash ^= 0x80200003;
		729	}
		730	return hash;
		731	}
		732
		733	static __init int cfi_parse_cmdline(char *str)
		734	{
		735	if (!str)
		736	return -EINVAL;
		737
		738	while (str) {
		739	char *next = strchr(str, ',');
		740	if (next) {
		741	*next = 0;
		742	next++;
		743	}
		744
		745	if (!strcmp(str, "auto")) {
		746	cfi_mode = CFI_DEFAULT;
		747	} else if (!strcmp(str, "off")) {
		748	cfi_mode = CFI_OFF;
		749	cfi_rand = false;
		750	} else if (!strcmp(str, "kcfi")) {
		751	cfi_mode = CFI_KCFI;
		752	} else if (!strcmp(str, "fineibt")) {
		753	cfi_mode = CFI_FINEIBT;
		754	} else if (!strcmp(str, "norand")) {
		755	cfi_rand = false;
		756	} else {
		757	pr_err("Ignoring unknown cfi option (%s).", str);
		758	}
		759
		760	str = next;
		761	}
		762
		763	return 0;
		764	}
		765	early_param("cfi", cfi_parse_cmdline);
		766
		767	/*
		768	* kCFI FineIBT
		769	*
		770	* __cfi_\func: __cfi_\func:
		771	* movl $0x12345678,%eax // 5 endbr64 // 4
		772	* nop subl $0x12345678,%r10d // 7
		773	* nop jz 1f // 2
		774	* nop ud2 // 2
		775	* nop 1: nop // 1
		776	* nop
		777	* nop
		778	* nop
		779	* nop
		780	* nop
		781	* nop
		782	* nop
		783	*
		784	*
		785	* caller: caller:
		786	* movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
		787	* addl $-15(%r11),%r10d // 4 sub $16,%r11 // 4
		788	* je 1f // 2 nop4 // 4
		789	* ud2 // 2
		790	* 1: call __x86_indirect_thunk_r11 // 5 call *%r11; nop2; // 5
		791	*
		792	*/
		793
		794	asm( ".pushsection .rodata \n"
		795	"fineibt_preamble_start: \n"
		796	" endbr64 \n"
		797	" subl $0x12345678, %r10d \n"
		798	" je fineibt_preamble_end \n"
		799	" ud2 \n"
		800	" nop \n"
		801	"fineibt_preamble_end: \n"
		802	".popsection\n"
		803	);
		804
		805	extern u8 fineibt_preamble_start[];
		806	extern u8 fineibt_preamble_end[];
		807
		808	#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
		809	#define fineibt_preamble_hash 7
		810
		811	asm( ".pushsection .rodata \n"
		812	"fineibt_caller_start: \n"
		813	" movl $0x12345678, %r10d \n"
		814	" sub $16, %r11 \n"
		815	ASM_NOP4
		816	"fineibt_caller_end: \n"
		817	".popsection \n"
		818	);
		819
		820	extern u8 fineibt_caller_start[];
		821	extern u8 fineibt_caller_end[];
		822
		823	#define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
		824	#define fineibt_caller_hash 2
		825
		826	#define fineibt_caller_jmp (fineibt_caller_size - 2)
		827
		828	static u32 decode_preamble_hash(void *addr)
		829	{
		830	u8 *p = addr;
		831
		832	/* b8 78 56 34 12 mov $0x12345678,%eax */
		833	if (p[0] == 0xb8)
		834	return (u32 )(addr + 1);
		835
		836	return 0; /* invalid hash value */
		837	}
		838
		839	static u32 decode_caller_hash(void *addr)
		840	{
		841	u8 *p = addr;
		842
		843	/* 41 ba 78 56 34 12 mov $0x12345678,%r10d */
		844	if (p[0] == 0x41 && p[1] == 0xba)
		845	return -(u32 )(addr + 2);
		846
		847	/* e8 0c 78 56 34 12 jmp.d8 +12 */
		848	if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
		849	return -(u32 )(addr + 2);
		850
		851	return 0; /* invalid hash value */
		852	}
		853
		854	/* .retpoline_sites */
		855	static int cfi_disable_callers(s32 start, s32 end)
		856	{
		857	/*
		858	* Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
		859	* in tact for later usage. Also see decode_caller_hash() and
		860	* cfi_rewrite_callers().
		861	*/
		862	const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp };
		863	s32 *s;
		864
		865	for (s = start; s < end; s++) {
		866	void addr = (void )s + *s;
		867	u32 hash;
		868
		869	addr -= fineibt_caller_size;
		870	hash = decode_caller_hash(addr);
		871	if (!hash) /* nocfi callers */
609	continue;	872	continue;
610		873
611	if (WARN_ON_ONCE(!is_endbr(endbr)))	874	text_poke_early(addr, jmp, 2);
		875	}
		876
		877	return 0;
		878	}
		879
		880	static int cfi_enable_callers(s32 start, s32 end)
		881	{
		882	/*
		883	* Re-enable kCFI, undo what cfi_disable_callers() did.
		884	*/
		885	const u8 mov[] = { 0x41, 0xba };
		886	s32 *s;
		887
		888	for (s = start; s < end; s++) {
		889	void addr = (void )s + *s;
		890	u32 hash;
		891
		892	addr -= fineibt_caller_size;
		893	hash = decode_caller_hash(addr);
		894	if (!hash) /* nocfi callers */
612	continue;	895	continue;
613		896
614	DPRINTK("ENDBR at: %pS (%px)", addr, addr);	897	text_poke_early(addr, mov, 2);
		898	}
615		899
616	/*	900	return 0;
617	* When we have IBT, the lack of ENDBR will trigger #CP	901	}
618	*/	902
619	DUMP_BYTES(((u8*)addr), 4, "%px: orig: ", addr);	903	/* .cfi_sites */
620	DUMP_BYTES(((u8*)&poison), 4, "%px: repl: ", addr);	904	static int cfi_rand_preamble(s32 start, s32 end)
621	text_poke_early(addr, &poison, 4);	905	{
		906	s32 *s;
		907
		908	for (s = start; s < end; s++) {
		909	void addr = (void )s + *s;
		910	u32 hash;
		911
		912	hash = decode_preamble_hash(addr);
		913	if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
		914	addr, addr, 5, addr))
		915	return -EINVAL;
		916
		917	hash = cfi_rehash(hash);
		918	text_poke_early(addr + 1, &hash, 4);
		919	}
		920
		921	return 0;
		922	}
		923
		924	static int cfi_rewrite_preamble(s32 start, s32 end)
		925	{
		926	s32 *s;
		927
		928	for (s = start; s < end; s++) {
		929	void addr = (void )s + *s;
		930	u32 hash;
		931
		932	hash = decode_preamble_hash(addr);
		933	if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
		934	addr, addr, 5, addr))
		935	return -EINVAL;
		936
		937	text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
		938	WARN_ON((u32 )(addr + fineibt_preamble_hash) != 0x12345678);
		939	text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
		940	}
		941
		942	return 0;
		943	}
		944
		945	/* .retpoline_sites */
		946	static int cfi_rand_callers(s32 start, s32 end)
		947	{
		948	s32 *s;
		949
		950	for (s = start; s < end; s++) {
		951	void addr = (void )s + *s;
		952	u32 hash;
		953
		954	addr -= fineibt_caller_size;
		955	hash = decode_caller_hash(addr);
		956	if (hash) {
		957	hash = -cfi_rehash(hash);
		958	text_poke_early(addr + 2, &hash, 4);
		959	}
		960	}
		961
		962	return 0;
		963	}
		964
		965	static int cfi_rewrite_callers(s32 start, s32 end)
		966	{
		967	s32 *s;
		968
		969	for (s = start; s < end; s++) {
		970	void addr = (void )s + *s;
		971	u32 hash;
		972
		973	addr -= fineibt_caller_size;
		974	hash = decode_caller_hash(addr);
		975	if (hash) {
		976	text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
		977	WARN_ON((u32 )(addr + fineibt_caller_hash) != 0x12345678);
		978	text_poke_early(addr + fineibt_caller_hash, &hash, 4);
		979	}
		980	/* rely on apply_retpolines() */
		981	}
		982
		983	return 0;
		984	}
		985
		986	static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
		987	s32 start_cfi, s32 end_cfi, bool builtin)
		988	{
		989	int ret;
		990
		991	if (WARN_ONCE(fineibt_preamble_size != 16,
		992	"FineIBT preamble wrong size: %ld", fineibt_preamble_size))
		993	return;
		994
		995	if (cfi_mode == CFI_DEFAULT) {
		996	cfi_mode = CFI_KCFI;
		997	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
		998	cfi_mode = CFI_FINEIBT;
		999	}
		1000
		1001	/*
		1002	* Rewrite the callers to not use the __cfi_ stubs, such that we might
		1003	* rewrite them. This disables all CFI. If this succeeds but any of the
		1004	* later stages fails, we're without CFI.
		1005	*/
		1006	ret = cfi_disable_callers(start_retpoline, end_retpoline);
		1007	if (ret)
		1008	goto err;
		1009
		1010	if (cfi_rand) {
		1011	if (builtin)
		1012	cfi_seed = get_random_u32();
		1013
		1014	ret = cfi_rand_preamble(start_cfi, end_cfi);
		1015	if (ret)
		1016	goto err;
		1017
		1018	ret = cfi_rand_callers(start_retpoline, end_retpoline);
		1019	if (ret)
		1020	goto err;
		1021	}
		1022
		1023	switch (cfi_mode) {
		1024	case CFI_OFF:
		1025	if (builtin)
		1026	pr_info("Disabling CFI\n");
		1027	return;
		1028
		1029	case CFI_KCFI:
		1030	ret = cfi_enable_callers(start_retpoline, end_retpoline);
		1031	if (ret)
		1032	goto err;
		1033
		1034	if (builtin)
		1035	pr_info("Using kCFI\n");
		1036	return;
		1037
		1038	case CFI_FINEIBT:
		1039	ret = cfi_rewrite_preamble(start_cfi, end_cfi);
		1040	if (ret)
		1041	goto err;
		1042
		1043	ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
		1044	if (ret)
		1045	goto err;
		1046
		1047	if (builtin)
		1048	pr_info("Using FineIBT CFI\n");
		1049	return;
		1050
		1051	default:
		1052	break;
622	}	1053	}
		1054
		1055	err:
		1056	pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n");
623	}	1057	}
624		1058
625	#else	1059	#else
626		1060
627	void __init_or_module apply_ibt_endbr(s32 start, s32 end) { }	1061	static void __apply_fineibt(s32 start_retpoline, s32 end_retpoline,
		1062	s32 start_cfi, s32 end_cfi, bool builtin)
		1063	{
		1064	}
628		1065
629	#endif /* CONFIG_X86_KERNEL_IBT */	1066	#endif
		1067
		1068	void apply_fineibt(s32 start_retpoline, s32 end_retpoline,
		1069	s32 start_cfi, s32 end_cfi)
		1070	{
		1071	return __apply_fineibt(start_retpoline, end_retpoline,
		1072	start_cfi, end_cfi,
		1073	/* .builtin = */ false);
		1074	}
630		1075
631	#ifdef CONFIG_SMP	1076	#ifdef CONFIG_SMP
632	static void alternatives_smp_lock(const s32 start, const s32 end,	1077	static void alternatives_smp_lock(const s32 start, const s32 end,
@@ -934,6 +1379,9 @@ void __init alternative_instructions(void) · 301 unmodified lines
934	*/	1379	*/
935	apply_paravirt(__parainstructions, __parainstructions_end);	1380	apply_paravirt(__parainstructions, __parainstructions_end);
936		1381
		1382	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,
		1383	__cfi_sites, __cfi_sites_end, true);
		1384
937	/*	1385	/*
938	* Rewrite the retpolines, must be done before alternatives since	1386	* Rewrite the retpolines, must be done before alternatives since
939	* those can rewrite the retpoline thunks.	1387	* those can rewrite the retpoline thunks.
@@ -947,6 +1395,12 @@ void __init alternative_instructions(void) · 7 unmodified lines
947	*/	1395	*/
948	apply_alternatives(__alt_instructions, __alt_instructions_end);	1396	apply_alternatives(__alt_instructions, __alt_instructions_end);
949		1397
		1398	/*
		1399	* Now all calls are established. Apply the call thunks if
		1400	* required.
		1401	*/
		1402	callthunks_patch_builtin_calls();
		1403
950	apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);	1404	apply_ibt_endbr(__ibt_endbr_seal, __ibt_endbr_seal_end);
951		1405
952	#ifdef CONFIG_SMP	1406	#ifdef CONFIG_SMP
@@ -1236,27 +1690,15 @@ void text_poke_kgdb(void addr, const void *opcode, size_t len) · 283 unmodified lines
1236	return __text_poke(text_poke_memcpy, addr, opcode, len);	1690	return __text_poke(text_poke_memcpy, addr, opcode, len);
1237	}	1691	}
1238		1692
1239	/**	1693	void text_poke_copy_locked(void addr, const void *opcode, size_t len,
1240	* text_poke_copy - Copy instructions into (an unused part of) RX memory	1694	bool core_ok)
1241	* @addr: address to modify
1242	* @opcode: source of the copy
1243	* @len: length to copy, could be more than 2x PAGE_SIZE
1244	*
1245	* Not safe against concurrent execution; useful for JITs to dump
1246	* new code blocks into unused regions of RX memory. Can be used in
1247	* conjunction with synchronize_rcu_tasks() to wait for existing
1248	* execution to quiesce after having made sure no existing functions
1249	* pointers are live.
1250	*/
1251	void text_poke_copy(void addr, const void *opcode, size_t len)
1252	{	1695	{
1253	unsigned long start = (unsigned long)addr;	1696	unsigned long start = (unsigned long)addr;
1254	size_t patched = 0;	1697	size_t patched = 0;
1255		1698
1256	if (WARN_ON_ONCE(core_kernel_text(start)))	1699	if (WARN_ON_ONCE(!core_ok && core_kernel_text(start)))
1257	return NULL;	1700	return NULL;
1258		1701
1259	mutex_lock(&text_mutex);
1260	while (patched < len) {	1702	while (patched < len) {
1261	unsigned long ptr = start + patched;	1703	unsigned long ptr = start + patched;
1262	size_t s;	1704	size_t s;
@@ -1266,6 +1708,25 @@ void text_poke_copy(void addr, const void *opcode, size_t len) · 3 unmodified lines
1266	__text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s);	1708	__text_poke(text_poke_memcpy, (void *)ptr, opcode + patched, s);
1267	patched += s;	1709	patched += s;
1268	}	1710	}
		1711	return addr;
		1712	}
		1713
		1714	/**
		1715	* text_poke_copy - Copy instructions into (an unused part of) RX memory
		1716	* @addr: address to modify
		1717	* @opcode: source of the copy
		1718	* @len: length to copy, could be more than 2x PAGE_SIZE
		1719	*
		1720	* Not safe against concurrent execution; useful for JITs to dump
		1721	* new code blocks into unused regions of RX memory. Can be used in
		1722	* conjunction with synchronize_rcu_tasks() to wait for existing
		1723	* execution to quiesce after having made sure no existing functions
		1724	* pointers are live.
		1725	*/
		1726	void text_poke_copy(void addr, const void *opcode, size_t len)
		1727	{
		1728	mutex_lock(&text_mutex);
		1729	addr = text_poke_copy_locked(addr, opcode, len, false);
1269	mutex_unlock(&text_mutex);	1730	mutex_unlock(&text_mutex);
1270	return addr;	1731	return addr;
1271	}	1732	}

arch/x86/kernel/asm-offsets.c+5−0

@@ -107,4 +107,9 @@ static void __used common(void) · 106 unmodified lines
107	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);	107	OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
108	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);	108	OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
109	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);	109	OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
		110	OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack);
		111	#ifdef CONFIG_CALL_DEPTH_TRACKING
		112	OFFSET(X86_call_depth, pcpu_hot, call_depth);
		113	#endif
		114
110	}	115	}

arch/x86/kernel/asm-offsets_64.c+1−1

@@ -57,7 +57,7 @@ int main(void) · 56 unmodified lines
57	BLANK();	57	BLANK();
58		58
59	#ifdef CONFIG_STACKPROTECTOR	59	#ifdef CONFIG_STACKPROTECTOR
60	DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary));	60	OFFSET(FIXED_stack_canary, fixed_percpu_data, stack_canary);
61	BLANK();	61	BLANK();
62	#endif	62	#endif
63	return 0;	63	return 0;

arch/x86/kernel/callthunks.c+388−0

@@ -0,0 +1,388 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	#define pr_fmt(fmt) "callthunks: " fmt
	4
	5	#include <linux/debugfs.h>
	6	#include <linux/kallsyms.h>
	7	#include <linux/memory.h>
	8	#include <linux/moduleloader.h>
	9	#include <linux/static_call.h>
	10
	11	#include <asm/alternative.h>
	12	#include <asm/asm-offsets.h>
	13	#include <asm/cpu.h>
	14	#include <asm/ftrace.h>
	15	#include <asm/insn.h>
	16	#include <asm/kexec.h>
	17	#include <asm/nospec-branch.h>
	18	#include <asm/paravirt.h>
	19	#include <asm/sections.h>
	20	#include <asm/switch_to.h>
	21	#include <asm/sync_core.h>
	22	#include <asm/text-patching.h>
	23	#include <asm/xen/hypercall.h>
	24
	25	static int __initdata_or_module debug_callthunks;
	26
	27	#define prdbg(fmt, args...) \
	28	do { \
	29	if (debug_callthunks) \
	30	printk(KERN_DEBUG pr_fmt(fmt), ##args); \
	31	} while(0)
	32
	33	static int __init debug_thunks(char *str)
	34	{
	35	debug_callthunks = 1;
	36	return 1;
	37	}
	38	__setup("debug-callthunks", debug_thunks);
	39
	40	#ifdef CONFIG_CALL_THUNKS_DEBUG
	41	DEFINE_PER_CPU(u64, __x86_call_count);
	42	DEFINE_PER_CPU(u64, __x86_ret_count);
	43	DEFINE_PER_CPU(u64, __x86_stuffs_count);
	44	DEFINE_PER_CPU(u64, __x86_ctxsw_count);
	45	EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
	46	EXPORT_SYMBOL_GPL(__x86_call_count);
	47	#endif
	48
	49	extern s32 __call_sites[], __call_sites_end[];
	50
	51	struct thunk_desc {
	52	void *template;
	53	unsigned int template_size;
	54	};
	55
	56	struct core_text {
	57	unsigned long base;
	58	unsigned long end;
	59	const char *name;
	60	};
	61
	62	static bool thunks_initialized __ro_after_init;
	63
	64	static const struct core_text builtin_coretext = {
	65	.base = (unsigned long)_text,
	66	.end = (unsigned long)_etext,
	67	.name = "builtin",
	68	};
	69
	70	asm (
	71	".pushsection .rodata \n"
	72	".global skl_call_thunk_template \n"
	73	"skl_call_thunk_template: \n"
	74	__stringify(INCREMENT_CALL_DEPTH)" \n"
	75	".global skl_call_thunk_tail \n"
	76	"skl_call_thunk_tail: \n"
	77	".popsection \n"
	78	);
	79
	80	extern u8 skl_call_thunk_template[];
	81	extern u8 skl_call_thunk_tail[];
	82
	83	#define SKL_TMPL_SIZE \
	84	((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
	85
	86	extern void error_entry(void);
	87	extern void xen_error_entry(void);
	88	extern void paranoid_entry(void);
	89
	90	static inline bool within_coretext(const struct core_text ct, void addr)
	91	{
	92	unsigned long p = (unsigned long)addr;
	93
	94	return ct->base <= p && p < ct->end;
	95	}
	96
	97	static inline bool within_module_coretext(void *addr)
	98	{
	99	bool ret = false;
	100
	101	#ifdef CONFIG_MODULES
	102	struct module *mod;
	103
	104	preempt_disable();
	105	mod = __module_address((unsigned long)addr);
	106	if (mod && within_module_core((unsigned long)addr, mod))
	107	ret = true;
	108	preempt_enable();
	109	#endif
	110	return ret;
	111	}
	112
	113	static bool is_coretext(const struct core_text ct, void addr)
	114	{
	115	if (ct && within_coretext(ct, addr))
	116	return true;
	117	if (within_coretext(&builtin_coretext, addr))
	118	return true;
	119	return within_module_coretext(addr);
	120	}
	121
	122	static __init_or_module bool skip_addr(void *dest)
	123	{
	124	if (dest == error_entry)
	125	return true;
	126	if (dest == paranoid_entry)
	127	return true;
	128	if (dest == xen_error_entry)
	129	return true;
	130	/* Does FILL_RSB... */
	131	if (dest == __switch_to_asm)
	132	return true;
	133	/* Accounts directly */
	134	if (dest == ret_from_fork)
	135	return true;
	136	#ifdef CONFIG_HOTPLUG_CPU
	137	if (dest == start_cpu0)
	138	return true;
	139	#endif
	140	#ifdef CONFIG_FUNCTION_TRACER
	141	if (dest == __fentry__)
	142	return true;
	143	#endif
	144	#ifdef CONFIG_KEXEC_CORE
	145	if (dest >= (void *)relocate_kernel &&
	146	dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
	147	return true;
	148	#endif
	149	#ifdef CONFIG_XEN
	150	if (dest >= (void *)hypercall_page &&
	151	dest < (void*)hypercall_page + PAGE_SIZE)
	152	return true;
	153	#endif
	154	return false;
	155	}
	156
	157	static __init_or_module void call_get_dest(void addr)
	158	{
	159	struct insn insn;
	160	void *dest;
	161	int ret;
	162
	163	ret = insn_decode_kernel(&insn, addr);
	164	if (ret)
	165	return ERR_PTR(ret);
	166
	167	/* Patched out call? */
	168	if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
	169	return NULL;
	170
	171	dest = addr + insn.length + insn.immediate.value;
	172	if (skip_addr(dest))
	173	return NULL;
	174	return dest;
	175	}
	176
	177	static const u8 nops[] = {
	178	0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	179	0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	180	0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	181	0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
	182	};
	183
	184	static __init_or_module void patch_dest(void dest, bool direct)
	185	{
	186	unsigned int tsize = SKL_TMPL_SIZE;
	187	u8 *pad = dest - tsize;
	188
	189	/* Already patched? */
	190	if (!bcmp(pad, skl_call_thunk_template, tsize))
	191	return pad;
	192
	193	/* Ensure there are nops */
	194	if (bcmp(pad, nops, tsize)) {
	195	pr_warn_once("Invalid padding area for %pS\n", dest);
	196	return NULL;
	197	}
	198
	199	if (direct)
	200	memcpy(pad, skl_call_thunk_template, tsize);
	201	else
	202	text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
	203	return pad;
	204	}
	205
	206	static __init_or_module void patch_call(void addr, const struct core_text ct)
	207	{
	208	void pad, dest;
	209	u8 bytes[8];
	210
	211	if (!within_coretext(ct, addr))
	212	return;
	213
	214	dest = call_get_dest(addr);
	215	if (!dest \|\| WARN_ON_ONCE(IS_ERR(dest)))
	216	return;
	217
	218	if (!is_coretext(ct, dest))
	219	return;
	220
	221	pad = patch_dest(dest, within_coretext(ct, dest));
	222	if (!pad)
	223	return;
	224
	225	prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
	226	dest, dest, pad);
	227	__text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
	228	text_poke_early(addr, bytes, CALL_INSN_SIZE);
	229	}
	230
	231	static __init_or_module void
	232	patch_call_sites(s32 start, s32 end, const struct core_text *ct)
	233	{
	234	s32 *s;
	235
	236	for (s = start; s < end; s++)
	237	patch_call((void )s + s, ct);
	238	}
	239
	240	static __init_or_module void
	241	patch_paravirt_call_sites(struct paravirt_patch_site *start,
	242	struct paravirt_patch_site *end,
	243	const struct core_text *ct)
	244	{
	245	struct paravirt_patch_site *p;
	246
	247	for (p = start; p < end; p++)
	248	patch_call(p->instr, ct);
	249	}
	250
	251	static __init_or_module void
	252	callthunks_setup(struct callthunk_sites cs, const struct core_text ct)
	253	{
	254	prdbg("Patching call sites %s\n", ct->name);
	255	patch_call_sites(cs->call_start, cs->call_end, ct);
	256	patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
	257	prdbg("Patching call sites done%s\n", ct->name);
	258	}
	259
	260	void __init callthunks_patch_builtin_calls(void)
	261	{
	262	struct callthunk_sites cs = {
	263	.call_start = __call_sites,
	264	.call_end = __call_sites_end,
	265	.pv_start = __parainstructions,
	266	.pv_end = __parainstructions_end
	267	};
	268
	269	if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
	270	return;
	271
	272	pr_info("Setting up call depth tracking\n");
	273	mutex_lock(&text_mutex);
	274	callthunks_setup(&cs, &builtin_coretext);
	275	static_call_force_reinit();
	276	thunks_initialized = true;
	277	mutex_unlock(&text_mutex);
	278	}
	279
	280	void callthunks_translate_call_dest(void dest)
	281	{
	282	void *target;
	283
	284	lockdep_assert_held(&text_mutex);
	285
	286	if (!thunks_initialized \|\| skip_addr(dest))
	287	return dest;
	288
	289	if (!is_coretext(NULL, dest))
	290	return dest;
	291
	292	target = patch_dest(dest, false);
	293	return target ? : dest;
	294	}
	295
	296	bool is_callthunk(void *addr)
	297	{
	298	unsigned int tmpl_size = SKL_TMPL_SIZE;
	299	void *tmpl = skl_call_thunk_template;
	300	unsigned long dest;
	301
	302	dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
	303	if (!thunks_initialized \|\| skip_addr((void *)dest))
	304	return false;
	305
	306	return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
	307	}
	308
	309	#ifdef CONFIG_BPF_JIT
	310	int x86_call_depth_emit_accounting(u8 *pprog, void func)
	311	{
	312	unsigned int tmpl_size = SKL_TMPL_SIZE;
	313	void *tmpl = skl_call_thunk_template;
	314
	315	if (!thunks_initialized)
	316	return 0;
	317
	318	/* Is function call target a thunk? */
	319	if (func && is_callthunk(func))
	320	return 0;
	321
	322	memcpy(*pprog, tmpl, tmpl_size);
	323	*pprog += tmpl_size;
	324	return tmpl_size;
	325	}
	326	#endif
	327
	328	#ifdef CONFIG_MODULES
	329	void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
	330	struct module *mod)
	331	{
	332	struct core_text ct = {
	333	.base = (unsigned long)mod->core_layout.base,
	334	.end = (unsigned long)mod->core_layout.base + mod->core_layout.size,
	335	.name = mod->name,
	336	};
	337
	338	if (!thunks_initialized)
	339	return;
	340
	341	mutex_lock(&text_mutex);
	342	callthunks_setup(cs, &ct);
	343	mutex_unlock(&text_mutex);
	344	}
	345	#endif /* CONFIG_MODULES */
	346
	347	#if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
	348	static int callthunks_debug_show(struct seq_file m, void p)
	349	{
	350	unsigned long cpu = (unsigned long)m->private;
	351
	352	seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
	353	per_cpu(__x86_call_count, cpu),
	354	per_cpu(__x86_ret_count, cpu),
	355	per_cpu(__x86_stuffs_count, cpu),
	356	per_cpu(__x86_ctxsw_count, cpu));
	357	return 0;
	358	}
	359
	360	static int callthunks_debug_open(struct inode inode, struct file file)
	361	{
	362	return single_open(file, callthunks_debug_show, inode->i_private);
	363	}
	364
	365	static const struct file_operations dfs_ops = {
	366	.open = callthunks_debug_open,
	367	.read = seq_read,
	368	.llseek = seq_lseek,
	369	.release = single_release,
	370	};
	371
	372	static int __init callthunks_debugfs_init(void)
	373	{
	374	struct dentry *dir;
	375	unsigned long cpu;
	376
	377	dir = debugfs_create_dir("callthunks", NULL);
	378	for_each_possible_cpu(cpu) {
	379	void arg = (void )cpu;
	380	char name [10];
	381
	382	sprintf(name, "cpu%lu", cpu);
	383	debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
	384	}
	385	return 0;
	386	}
	387	__initcall(callthunks_debugfs_init);
	388	#endif

arch/x86/kernel/cpu/Makefile+0−3

@@ -17,9 +17,6 @@ KMSAN_SANITIZE_common.o := n · 16 unmodified lines
17	# As above, instrumenting secondary CPU boot code causes boot hangs.	17	# As above, instrumenting secondary CPU boot code causes boot hangs.
18	KCSAN_SANITIZE_common.o := n	18	KCSAN_SANITIZE_common.o := n
19		19
20	# Make sure load_percpu_segment has no stackprotector
21	CFLAGS_common.o := -fno-stack-protector
22
23	obj-y := cacheinfo.o scattered.o topology.o	20	obj-y := cacheinfo.o scattered.o topology.o
24	obj-y += common.o	21	obj-y += common.o
25	obj-y += rdrand.o	22	obj-y += rdrand.o

arch/x86/kernel/cpu/bugs.c+32−2

 	RETBLEED_MITIGATION_IBPB,
 	RETBLEED_MITIGATION_IBRS,
 	RETBLEED_MITIGATION_EIBRS,
+	RETBLEED_MITIGATION_STUFF,
 };
 enum retbleed_mitigation_cmd {
 	RETBLEED_CMD_AUTO,
 	RETBLEED_CMD_UNRET,
 	RETBLEED_CMD_IBPB,
+	RETBLEED_CMD_STUFF,
 };
 static const char * const retbleed_strings[] = {
 	[RETBLEED_MITIGATION_IBPB]	= "Mitigation: IBPB",
 	[RETBLEED_MITIGATION_IBRS]	= "Mitigation: IBRS",
 	[RETBLEED_MITIGATION_EIBRS]	= "Mitigation: Enhanced IBRS",
+	[RETBLEED_MITIGATION_STUFF]	= "Mitigation: Stuffing",
 };
 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
 			retbleed_cmd = RETBLEED_CMD_UNRET;
 		} else if (!strcmp(str, "ibpb")) {
 			retbleed_cmd = RETBLEED_CMD_IBPB;
+		} else if (!strcmp(str, "stuff")) {
+			retbleed_cmd = RETBLEED_CMD_STUFF;
 		} else if (!strcmp(str, "nosmt")) {
 			retbleed_nosmt = true;
+		} else if (!strcmp(str, "force")) {
+			setup_force_cpu_bug(X86_BUG_RETBLEED);
 		} else {
 			pr_err("Ignoring unknown retbleed option (%s).", str);
 		}
 		}
 		break;
+	case RETBLEED_CMD_STUFF:
+		if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) &&
+		    spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {
+			retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
+		} else {
+			if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING))
+				pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
+			else
+				pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n");
+			goto do_cmd_auto;
+		}
+		break;
 do_cmd_auto:
 	case RETBLEED_CMD_AUTO:
 	default:
 		mitigate_smt = true;
 		break;
+	case RETBLEED_MITIGATION_STUFF:
+		setup_force_cpu_cap(X86_FEATURE_RETHUNK);
+		setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
+		x86_set_skl_return_thunk();
+		break;
 	default:
 		break;
 	}
 	/*
 	 * Let IBRS trump all on Intel without affecting the effects of the
-	 * retbleed= cmdline option.
+	 * retbleed= cmdline option except for call depth based stuffing
 	 */
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
 		switch (spectre_v2_enabled) {
 			retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
 			break;
 		default:
-			pr_err(RETBLEED_INTEL_MSG);
+			if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
+				pr_err(RETBLEED_INTEL_MSG);
 		}
 	}
 		if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) &&
 		    boot_cpu_has_bug(X86_BUG_RETBLEED) &&
 		    retbleed_cmd != RETBLEED_CMD_OFF &&
+		    retbleed_cmd != RETBLEED_CMD_STUFF &&
 		    boot_cpu_has(X86_FEATURE_IBRS) &&
 		    boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
 			mode = SPECTRE_V2_IBRS;

arch/x86/kernel/cpu/common.c+44−53

@@ -610,6 +610,7 @@ static __always_inline void setup_cet(struct cpuinfo_x86 *c) · 609 unmodified lines
610		610
611	if (!ibt_selftest()) {	611	if (!ibt_selftest()) {
612	pr_err("IBT selftest: Failed!\n");	612	pr_err("IBT selftest: Failed!\n");
		613	wrmsrl(MSR_IA32_S_CET, 0);
613	setup_clear_cpu_cap(X86_FEATURE_IBT);	614	setup_clear_cpu_cap(X86_FEATURE_IBT);
614	return;	615	return;
615	}	616	}
@@ -702,16 +703,6 @@ static const char table_lookup_model(struct cpuinfo_x86 c) · 86 unmodified lines
702	__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));	703	__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
703	__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));	704	__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
704		705
705	void load_percpu_segment(int cpu)
706	{
707	#ifdef CONFIG_X86_32
708	loadsegment(fs, __KERNEL_PERCPU);
709	#else
710	__loadsegment_simple(gs, 0);
711	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
712	#endif
713	}
714
715	#ifdef CONFIG_X86_32	706	#ifdef CONFIG_X86_32
716	/* The 32-bit entry code needs to find cpu_entry_area. */	707	/* The 32-bit entry code needs to find cpu_entry_area. */
717	DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);	708	DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
@@ -739,16 +730,45 @@ void load_fixmap_gdt(int cpu) · 21 unmodified lines
739	}	730	}
740	EXPORT_SYMBOL_GPL(load_fixmap_gdt);	731	EXPORT_SYMBOL_GPL(load_fixmap_gdt);
741		732
742	/*	733	/**
743	* Current gdt points %fs at the "master" per-cpu area: after this,	734	* switch_gdt_and_percpu_base - Switch to direct GDT and runtime per CPU base
744	* it's on the real one.	735	* @cpu: The CPU number for which this is invoked
		736	*
		737	* Invoked during early boot to switch from early GDT and early per CPU to
		738	* the direct GDT and the runtime per CPU area. On 32-bit the percpu base
		739	* switch is implicit by loading the direct GDT. On 64bit this requires
		740	* to update GSBASE.
745	*/	741	*/
746	void switch_to_new_gdt(int cpu)	742	void __init switch_gdt_and_percpu_base(int cpu)
747	{	743	{
748	/* Load the original GDT */
749	load_direct_gdt(cpu);	744	load_direct_gdt(cpu);
750	/* Reload the per-cpu base */	745
751	load_percpu_segment(cpu);	746	#ifdef CONFIG_X86_64
		747	/*
		748	* No need to load %gs. It is already correct.
		749	*
		750	* Writing %gs on 64bit would zero GSBASE which would make any per
		751	* CPU operation up to the point of the wrmsrl() fault.
		752	*
		753	* Set GSBASE to the new offset. Until the wrmsrl() happens the
		754	* early mapping is still valid. That means the GSBASE update will
		755	* lose any prior per CPU data which was not copied over in
		756	* setup_per_cpu_areas().
		757	*
		758	* This works even with stackprotector enabled because the
		759	* per CPU stack canary is 0 in both per CPU areas.
		760	*/
		761	wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
		762	#else
		763	/*
		764	* %fs is already set to __KERNEL_PERCPU, but after switching GDT
		765	* it is required to load FS again so that the 'hidden' part is
		766	* updated from the new GDT. Up to this point the early per CPU
		767	* translation is active. Any content of the early per CPU data
		768	* which was not copied over in setup_per_cpu_areas() is lost.
		769	*/
		770	loadsegment(fs, __KERNEL_PERCPU);
		771	#endif
752	}	772	}
753		773
754	static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};	774	static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
@@ -1993,27 +2013,18 @@ static __init int setup_clearcpuid(char *arg) · 1238 unmodified lines
1993	}	2013	}
1994	__setup("clearcpuid=", setup_clearcpuid);	2014	__setup("clearcpuid=", setup_clearcpuid);
1995		2015
		2016	DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
		2017	.current_task = &init_task,
		2018	.preempt_count = INIT_PREEMPT_COUNT,
		2019	.top_of_stack = TOP_OF_INIT_STACK,
		2020	};
		2021	EXPORT_PER_CPU_SYMBOL(pcpu_hot);
		2022
1996	#ifdef CONFIG_X86_64	2023	#ifdef CONFIG_X86_64
1997	DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,	2024	DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
1998	fixed_percpu_data) __aligned(PAGE_SIZE) __visible;	2025	fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
1999	EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);	2026	EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
2000		2027
2001	/*
2002	* The following percpu variables are hot. Align current_task to
2003	* cacheline size such that they fall in the same cacheline.
2004	*/
2005	DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
2006	&init_task;
2007	EXPORT_PER_CPU_SYMBOL(current_task);
2008
2009	DEFINE_PER_CPU(void *, hardirq_stack_ptr);
2010	DEFINE_PER_CPU(bool, hardirq_stack_inuse);
2011
2012	DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2013	EXPORT_PER_CPU_SYMBOL(__preempt_count);
2014
2015	DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
2016
2017	static void wrmsrl_cstar(unsigned long val)	2028	static void wrmsrl_cstar(unsigned long val)
2018	{	2029	{
2019	/*	2030	/*
@@ -2064,20 +2075,6 @@ void syscall_init(void) · 44 unmodified lines
2064		2075
2065	#else /* CONFIG_X86_64 */	2076	#else /* CONFIG_X86_64 */
2066		2077
2067	DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
2068	EXPORT_PER_CPU_SYMBOL(current_task);
2069	DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
2070	EXPORT_PER_CPU_SYMBOL(__preempt_count);
2071
2072	/*
2073	* On x86_32, vm86 modifies tss.sp0, so sp0 isn't a reliable way to find
2074	* the top of the kernel stack. Use an extra percpu variable to track the
2075	* top of the kernel stack directly.
2076	*/
2077	DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
2078	(unsigned long)&init_thread_union + THREAD_SIZE;
2079	EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
2080
2081	#ifdef CONFIG_STACKPROTECTOR	2078	#ifdef CONFIG_STACKPROTECTOR
2082	DEFINE_PER_CPU(unsigned long, __stack_chk_guard);	2079	DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
2083	EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);	2080	EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
@@ -2248,12 +2245,6 @@ void cpu_init(void) · 164 unmodified lines
2248	boot_cpu_has(X86_FEATURE_TSC) \|\| boot_cpu_has(X86_FEATURE_DE))	2245	boot_cpu_has(X86_FEATURE_TSC) \|\| boot_cpu_has(X86_FEATURE_DE))
2249	cr4_clear_bits(X86_CR4_VME\|X86_CR4_PVI\|X86_CR4_TSD\|X86_CR4_DE);	2246	cr4_clear_bits(X86_CR4_VME\|X86_CR4_PVI\|X86_CR4_TSD\|X86_CR4_DE);
2250		2247
2251	/*
2252	* Initialize the per-CPU GDT with the boot GDT,
2253	* and set up the GDT descriptor:
2254	*/
2255	switch_to_new_gdt(cpu);
2256
2257	if (IS_ENABLED(CONFIG_X86_64)) {	2248	if (IS_ENABLED(CONFIG_X86_64)) {
2258	loadsegment(fs, 0);	2249	loadsegment(fs, 0);
2259	memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);	2250	memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);

arch/x86/kernel/dumpstack_32.c+2−2

@@ -37,7 +37,7 @@ const char *stack_type_name(enum stack_type type) · 36 unmodified lines
37		37
38	static bool in_hardirq_stack(unsigned long stack, struct stack_info info)	38	static bool in_hardirq_stack(unsigned long stack, struct stack_info info)
39	{	39	{
40	unsigned long begin = (unsigned long )this_cpu_read(hardirq_stack_ptr);	40	unsigned long begin = (unsigned long )this_cpu_read(pcpu_hot.hardirq_stack_ptr);
41	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));	41	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));
42		42
43	/*	43	/*
@@ -62,7 +62,7 @@ static bool in_hardirq_stack(unsigned long stack, struct stack_info info) · 18 unmodified lines
62		62
63	static bool in_softirq_stack(unsigned long stack, struct stack_info info)	63	static bool in_softirq_stack(unsigned long stack, struct stack_info info)
64	{	64	{
65	unsigned long begin = (unsigned long )this_cpu_read(softirq_stack_ptr);	65	unsigned long begin = (unsigned long )this_cpu_read(pcpu_hot.softirq_stack_ptr);
66	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));	66	unsigned long *end = begin + (THREAD_SIZE / sizeof(long));
67		67
68	/*	68	/*

arch/x86/kernel/dumpstack_64.c+1−1

@@ -134,7 +134,7 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac · 133 unmodified lines
134		134
135	static __always_inline bool in_irq_stack(unsigned long stack, struct stack_info info)	135	static __always_inline bool in_irq_stack(unsigned long stack, struct stack_info info)
136	{	136	{
137	unsigned long end = (unsigned long )this_cpu_read(hardirq_stack_ptr);	137	unsigned long end = (unsigned long )this_cpu_read(pcpu_hot.hardirq_stack_ptr);
138	unsigned long *begin;	138	unsigned long *begin;
139		139
140	/*	140	/*

arch/x86/kernel/ftrace.c+13−7

 static const char *ftrace_call_replace(unsigned long ip, unsigned long addr)
 {
+	/*
+	 * No need to translate into a callthunk. The trampoline does
+	 * the depth accounting itself.
+	 */
 	return text_gen_insn(CALL_INSN_OPCODE, (void *)ip, (void *)addr);
 }
 	unsigned long size;
 	unsigned long *ptr;
 	void *trampoline;
-	void *ip;
+	void *ip, *dest;
 	/* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
 	unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
 	unsigned const char retq[] = { RET_INSN_OPCODE, INT3_INSN_OPCODE };
 	ip = trampoline + size;
 	if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
-		__text_gen_insn(ip, JMP32_INSN_OPCODE, ip, &__x86_return_thunk, JMP32_INSN_SIZE);
+		__text_gen_insn(ip, JMP32_INSN_OPCODE, ip, x86_return_thunk, JMP32_INSN_SIZE);
 	else
 		memcpy(ip, retq, sizeof(retq));
 	/* put in the call to the function */
 	mutex_lock(&text_mutex);
 	call_offset -= start_offset;
+	/*
+	 * No need to translate into a callthunk. The trampoline does
+	 * the depth accounting before the call already.
+	 */
+	dest = ftrace_ops_get_func(ops);
 	memcpy(trampoline + call_offset,
-	       text_gen_insn(CALL_INSN_OPCODE,
+	       text_gen_insn(CALL_INSN_OPCODE, trampoline + call_offset, dest),
-			     trampoline + call_offset,
+	       CALL_INSN_SIZE);
-			     ftrace_ops_get_func(ops)), CALL_INSN_SIZE);
 	mutex_unlock(&text_mutex);
 	/* ALLOC_TRAMP flags lets us know we created it */
 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
-	set_vm_flush_reset_perms(trampoline);
 	if (likely(system_state != SYSTEM_BOOTING))
 		set_memory_ro((unsigned long)trampoline, npages);
 	set_memory_x((unsigned long)trampoline, npages);

arch/x86/kernel/ftrace_64.S+34−3

@@ -3,8 +3,9 @@ · 2 unmodified lines
3	* Copyright (C) 2014 Steven Rostedt, Red Hat Inc	3	* Copyright (C) 2014 Steven Rostedt, Red Hat Inc
4	*/	4	*/
5		5
6	#include <linux/linkage.h>
7	#include <linux/cfi_types.h>	6	#include <linux/cfi_types.h>
		7	#include <linux/linkage.h>
		8	#include <asm/asm-offsets.h>
8	#include <asm/ptrace.h>	9	#include <asm/ptrace.h>
9	#include <asm/ftrace.h>	10	#include <asm/ftrace.h>
10	#include <asm/export.h>	11	#include <asm/export.h>
@@ -131,16 +132,19 @@ · 120 unmodified lines
131	.endm	132	.endm
132		133
133	SYM_TYPED_FUNC_START(ftrace_stub)	134	SYM_TYPED_FUNC_START(ftrace_stub)
		135	CALL_DEPTH_ACCOUNT
134	RET	136	RET
135	SYM_FUNC_END(ftrace_stub)	137	SYM_FUNC_END(ftrace_stub)
136		138
137	SYM_TYPED_FUNC_START(ftrace_stub_graph)	139	SYM_TYPED_FUNC_START(ftrace_stub_graph)
		140	CALL_DEPTH_ACCOUNT
138	RET	141	RET
139	SYM_FUNC_END(ftrace_stub_graph)	142	SYM_FUNC_END(ftrace_stub_graph)
140		143
141	#ifdef CONFIG_DYNAMIC_FTRACE	144	#ifdef CONFIG_DYNAMIC_FTRACE
142		145
143	SYM_FUNC_START(__fentry__)	146	SYM_FUNC_START(__fentry__)
		147	CALL_DEPTH_ACCOUNT
144	RET	148	RET
145	SYM_FUNC_END(__fentry__)	149	SYM_FUNC_END(__fentry__)
146	EXPORT_SYMBOL(__fentry__)	150	EXPORT_SYMBOL(__fentry__)
@@ -149,6 +153,8 @@ SYM_FUNC_START(ftrace_caller) · 2 unmodified lines
149	/* save_mcount_regs fills in first two parameters */	153	/* save_mcount_regs fills in first two parameters */
150	save_mcount_regs	154	save_mcount_regs
151		155
		156	CALL_DEPTH_ACCOUNT
		157
152	/* Stack - skipping return address of ftrace_caller */	158	/* Stack - skipping return address of ftrace_caller */
153	leaq MCOUNT_REG_SIZE+8(%rsp), %rcx	159	leaq MCOUNT_REG_SIZE+8(%rsp), %rcx
154	movq %rcx, RSP(%rsp)	160	movq %rcx, RSP(%rsp)
@@ -164,6 +170,9 @@ SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL) · 9 unmodified lines
164	/* Only ops with REGS flag set should have CS register set */	170	/* Only ops with REGS flag set should have CS register set */
165	movq $0, CS(%rsp)	171	movq $0, CS(%rsp)
166		172
		173	/* Account for the function call below */
		174	CALL_DEPTH_ACCOUNT
		175
167	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)	176	SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
168	ANNOTATE_NOENDBR	177	ANNOTATE_NOENDBR
169	call ftrace_stub	178	call ftrace_stub
@@ -193,6 +202,8 @@ SYM_FUNC_START(ftrace_regs_caller) · 23 unmodified lines
193	save_mcount_regs 8	202	save_mcount_regs 8
194	/* save_mcount_regs fills in first two parameters */	203	/* save_mcount_regs fills in first two parameters */
195		204
		205	CALL_DEPTH_ACCOUNT
		206
196	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)	207	SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
197	ANNOTATE_NOENDBR	208	ANNOTATE_NOENDBR
198	/* Load the ftrace_ops into the 3rd parameter */	209	/* Load the ftrace_ops into the 3rd parameter */
@@ -223,6 +234,9 @@ SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL) · 24 unmodified lines
223	/* regs go into 4th parameter */	234	/* regs go into 4th parameter */
224	leaq (%rsp), %rcx	235	leaq (%rsp), %rcx
225		236
		237	/* Account for the function call below */
		238	CALL_DEPTH_ACCOUNT
		239
226	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)	240	SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
227	ANNOTATE_NOENDBR	241	ANNOTATE_NOENDBR
228	call ftrace_stub	242	call ftrace_stub
@@ -275,7 +289,20 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) · 46 unmodified lines
275	/* Restore flags */	289	/* Restore flags */
276	popfq	290	popfq
277	UNWIND_HINT_FUNC	291	UNWIND_HINT_FUNC
278	RET	292
		293	/*
		294	* The above left an extra return value on the stack; effectively
		295	* doing a tail-call without using a register. This PUSH;RET
		296	* pattern unbalances the RSB, inject a pointless CALL to rebalance.
		297	*/
		298	ANNOTATE_INTRA_FUNCTION_CALL
		299	CALL .Ldo_rebalance
		300	int3
		301	.Ldo_rebalance:
		302	add $8, %rsp
		303	ALTERNATIVE __stringify(RET), \
		304	__stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
		305	X86_FEATURE_CALL_DEPTH
279		306
280	SYM_FUNC_END(ftrace_regs_caller)	307	SYM_FUNC_END(ftrace_regs_caller)
281	STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)	308	STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
@@ -284,6 +311,8 @@ STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller) · 2 unmodified lines
284	#else /* ! CONFIG_DYNAMIC_FTRACE */	311	#else /* ! CONFIG_DYNAMIC_FTRACE */
285		312
286	SYM_FUNC_START(__fentry__)	313	SYM_FUNC_START(__fentry__)
		314	CALL_DEPTH_ACCOUNT
		315
287	cmpq $ftrace_stub, ftrace_trace_function	316	cmpq $ftrace_stub, ftrace_trace_function
288	jnz trace	317	jnz trace
289	RET	318	RET
@@ -337,6 +366,8 @@ SYM_CODE_START(return_to_handler) · 47 unmodified lines
337	int3	366	int3
338	.Ldo_rop:	367	.Ldo_rop:
339	mov %rdi, (%rsp)	368	mov %rdi, (%rsp)
340	RET	369	ALTERNATIVE __stringify(RET), \
		370	__stringify(ANNOTATE_UNRET_SAFE; ret; int3), \
		371	X86_FEATURE_CALL_DEPTH
341	SYM_CODE_END(return_to_handler)	372	SYM_CODE_END(return_to_handler)
342	#endif	373	#endif

arch/x86/kernel/head_64.S+1−0

@@ -370,6 +370,7 @@ SYM_CODE_END(secondary_startup_64) · 369 unmodified lines
370	* start_secondary() via .Ljump_to_C_code.	370	* start_secondary() via .Ljump_to_C_code.
371	*/	371	*/
372	SYM_CODE_START(start_cpu0)	372	SYM_CODE_START(start_cpu0)
		373	ANNOTATE_NOENDBR
373	UNWIND_HINT_EMPTY	374	UNWIND_HINT_EMPTY
374	movq initial_stack(%rip), %rsp	375	movq initial_stack(%rip), %rsp
375	jmp .Ljump_to_C_code	376	jmp .Ljump_to_C_code

arch/x86/kernel/irq_32.c+5−8

 static inline void print_stack_overflow(void) { }
 #endif
-DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
-DEFINE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
 static void call_on_stack(void *func, void *stack)
 {
 	asm volatile("xchgl	%%ebx,%%esp	\n"
 	u32 *isp, *prev_esp, arg1;
 	curstk = (struct irq_stack *) current_stack();
-	irqstk = __this_cpu_read(hardirq_stack_ptr);
+	irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr);
 	/*
 	 * this is where we switch to the IRQ stack. However, if we are
 	int node = cpu_to_node(cpu);
 	struct page *ph, *ps;
-	if (per_cpu(hardirq_stack_ptr, cpu))
+	if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
 		return 0;
 	ph = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
 		return -ENOMEM;
 	}
-	per_cpu(hardirq_stack_ptr, cpu) = page_address(ph);
+	per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph);
-	per_cpu(softirq_stack_ptr, cpu) = page_address(ps);
+	per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps);
 	return 0;
 }
 	struct irq_stack *irqstk;
 	u32 *isp, *prev_esp;
-	irqstk = __this_cpu_read(softirq_stack_ptr);
+	irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr);
 	/* build the stack frame on the softirq stack */
 	isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));

arch/x86/kernel/irq_64.c+3−3

@@ -50,7 +50,7 @@ static int map_irq_stack(unsigned int cpu) · 49 unmodified lines
50	return -ENOMEM;	50	return -ENOMEM;
51		51
52	/* Store actual TOS to avoid adjustment in the hotpath */	52	/* Store actual TOS to avoid adjustment in the hotpath */
53	per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;	53	per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
54	return 0;	54	return 0;
55	}	55	}
56	#else	56	#else
@@ -63,14 +63,14 @@ static int map_irq_stack(unsigned int cpu) · 6 unmodified lines
63	void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);	63	void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);
64		64
65	/* Store actual TOS to avoid adjustment in the hotpath */	65	/* Store actual TOS to avoid adjustment in the hotpath */
66	per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;	66	per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
67	return 0;	67	return 0;
68	}	68	}
69	#endif	69	#endif
70		70
71	int irq_init_percpu_irqstack(unsigned int cpu)	71	int irq_init_percpu_irqstack(unsigned int cpu)
72	{	72	{
73	if (per_cpu(hardirq_stack_ptr, cpu))	73	if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
74	return 0;	74	return 0;
75	return map_irq_stack(cpu);	75	return map_irq_stack(cpu);
76	}	76	}

arch/x86/kernel/kprobes/core.c+0−1

@@ -414,7 +414,6 @@ void *alloc_insn_page(void) · 413 unmodified lines
414	if (!page)	414	if (!page)
415	return NULL;	415	return NULL;
416		416
417	set_vm_flush_reset_perms(page);
418	/*	417	/*
419	* First make the page read-only, and only then make it executable to	418	* First make the page read-only, and only then make it executable to
420	* prevent it from being W+X in between.	419	* prevent it from being W+X in between.

arch/x86/kernel/kvm.c+6−12

  * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
  * restoring to/from the stack.
  */
-asm(
+#define PV_VCPU_PREEMPTED_ASM						     \
-".pushsection .text;"
+ "movq   __per_cpu_offset(,%rdi,8), %rax\n\t"				     \
-".global __raw_callee_save___kvm_vcpu_is_preempted;"
+ "cmpb   $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax)\n\t" \
-".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
+ "setne  %al\n\t"
-"__raw_callee_save___kvm_vcpu_is_preempted:"
-ASM_ENDBR
-"movq	__per_cpu_offset(,%rdi,8), %rax;"
-"cmpb	$0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
-"setne	%al;"
-ASM_RET
-".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;"
-".popsection");
+DEFINE_PARAVIRT_ASM(__raw_callee_save___kvm_vcpu_is_preempted,
+		    PV_VCPU_PREEMPTED_ASM, .text);
 #endif
 static void __init kvm_guest_init(void)

arch/x86/kernel/module.c+42−5

@@ -74,10 +74,11 @@ void *module_alloc(unsigned long size) · 73 unmodified lines
74	return NULL;	74	return NULL;
75		75
76	p = __vmalloc_node_range(size, MODULE_ALIGN,	76	p = __vmalloc_node_range(size, MODULE_ALIGN,
77	MODULES_VADDR + get_module_load_offset(),	77	MODULES_VADDR + get_module_load_offset(),
78	MODULES_END, gfp_mask,	78	MODULES_END, gfp_mask, PAGE_KERNEL,
79	PAGE_KERNEL, VM_DEFER_KMEMLEAK, NUMA_NO_NODE,	79	VM_FLUSH_RESET_PERMS \| VM_DEFER_KMEMLEAK,
80	__builtin_return_address(0));	80	NUMA_NO_NODE, __builtin_return_address(0));
		81
81	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {	82	if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) {
82	vfree(p);	83	vfree(p);
83	return NULL;	84	return NULL;
@@ -253,7 +254,8 @@ int module_finalize(const Elf_Ehdr *hdr, · 169 unmodified lines
253	{	254	{
254	const Elf_Shdr s, alt = NULL, *locks = NULL,	255	const Elf_Shdr s, alt = NULL, *locks = NULL,
255	para = NULL, orc = NULL, *orc_ip = NULL,	256	para = NULL, orc = NULL, *orc_ip = NULL,
256	retpolines = NULL, returns = NULL, *ibt_endbr = NULL;	257	retpolines = NULL, returns = NULL, *ibt_endbr = NULL,
		258	calls = NULL, cfi = NULL;
257	char secstrings = (void )hdr + sechdrs[hdr->e_shstrndx].sh_offset;	259	char secstrings = (void )hdr + sechdrs[hdr->e_shstrndx].sh_offset;
258		260
259	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {	261	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -271,6 +273,10 @@ int module_finalize(const Elf_Ehdr *hdr, · 11 unmodified lines
271	retpolines = s;	273	retpolines = s;
272	if (!strcmp(".return_sites", secstrings + s->sh_name))	274	if (!strcmp(".return_sites", secstrings + s->sh_name))
273	returns = s;	275	returns = s;
		276	if (!strcmp(".call_sites", secstrings + s->sh_name))
		277	calls = s;
		278	if (!strcmp(".cfi_sites", secstrings + s->sh_name))
		279	cfi = s;
274	if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))	280	if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
275	ibt_endbr = s;	281	ibt_endbr = s;
276	}	282	}
@@ -283,6 +289,22 @@ int module_finalize(const Elf_Ehdr *hdr, · 6 unmodified lines
283	void pseg = (void )para->sh_addr;	289	void pseg = (void )para->sh_addr;
284	apply_paravirt(pseg, pseg + para->sh_size);	290	apply_paravirt(pseg, pseg + para->sh_size);
285	}	291	}
		292	if (retpolines \|\| cfi) {
		293	void rseg = NULL, cseg = NULL;
		294	unsigned int rsize = 0, csize = 0;
		295
		296	if (retpolines) {
		297	rseg = (void *)retpolines->sh_addr;
		298	rsize = retpolines->sh_size;
		299	}
		300
		301	if (cfi) {
		302	cseg = (void *)cfi->sh_addr;
		303	csize = cfi->sh_size;
		304	}
		305
		306	apply_fineibt(rseg, rseg + rsize, cseg, cseg + csize);
		307	}
286	if (retpolines) {	308	if (retpolines) {
287	void rseg = (void )retpolines->sh_addr;	309	void rseg = (void )retpolines->sh_addr;
288	apply_retpolines(rseg, rseg + retpolines->sh_size);	310	apply_retpolines(rseg, rseg + retpolines->sh_size);
@@ -296,6 +318,21 @@ int module_finalize(const Elf_Ehdr *hdr, · 7 unmodified lines
296	void aseg = (void )alt->sh_addr;	318	void aseg = (void )alt->sh_addr;
297	apply_alternatives(aseg, aseg + alt->sh_size);	319	apply_alternatives(aseg, aseg + alt->sh_size);
298	}	320	}
		321	if (calls \|\| para) {
		322	struct callthunk_sites cs = {};
		323
		324	if (calls) {
		325	cs.call_start = (void *)calls->sh_addr;
		326	cs.call_end = (void *)calls->sh_addr + calls->sh_size;
		327	}
		328
		329	if (para) {
		330	cs.pv_start = (void *)para->sh_addr;
		331	cs.pv_end = (void *)para->sh_addr + para->sh_size;
		332	}
		333
		334	callthunks_patch_module_calls(&cs, me);
		335	}
299	if (ibt_endbr) {	336	if (ibt_endbr) {
300	void iseg = (void )ibt_endbr->sh_addr;	337	void iseg = (void )ibt_endbr->sh_addr;
301	apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);	338	apply_ibt_endbr(iseg, iseg + ibt_endbr->sh_size);

arch/x86/kernel/paravirt.c+2−19

@@ -37,27 +37,10 @@ · 36 unmodified lines
37	* nop stub, which must not clobber anything including the stack to	37	* nop stub, which must not clobber anything including the stack to
38	* avoid confusing the entry prologues.	38	* avoid confusing the entry prologues.
39	*/	39	*/
40	extern void _paravirt_nop(void);	40	DEFINE_PARAVIRT_ASM(_paravirt_nop, "", .entry.text);
41	asm (".pushsection .entry.text, \"ax\"\n"
42	".global _paravirt_nop\n"
43	"_paravirt_nop:\n\t"
44	ASM_ENDBR
45	ASM_RET
46	".size _paravirt_nop, . - _paravirt_nop\n\t"
47	".type _paravirt_nop, @function\n\t"
48	".popsection");
49		41
50	/* stub always returning 0. */	42	/* stub always returning 0. */
51	asm (".pushsection .entry.text, \"ax\"\n"	43	DEFINE_PARAVIRT_ASM(paravirt_ret0, "xor %eax,%eax", .entry.text);
52	".global paravirt_ret0\n"
53	"paravirt_ret0:\n\t"
54	ASM_ENDBR
55	"xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
56	ASM_RET
57	".size paravirt_ret0, . - paravirt_ret0\n\t"
58	".type paravirt_ret0, @function\n\t"
59	".popsection");
60
61		44
62	void __init default_banner(void)	45	void __init default_banner(void)
63	{	46	{

arch/x86/kernel/process_32.c+3−3

@@ -191,13 +191,13 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 190 unmodified lines
191	arch_end_context_switch(next_p);	191	arch_end_context_switch(next_p);
192		192
193	/*	193	/*
194	* Reload esp0 and cpu_current_top_of_stack. This changes	194	* Reload esp0 and pcpu_hot.top_of_stack. This changes
195	* current_thread_info(). Refresh the SYSENTER configuration in	195	* current_thread_info(). Refresh the SYSENTER configuration in
196	* case prev or next is vm86.	196	* case prev or next is vm86.
197	*/	197	*/
198	update_task_stack(next_p);	198	update_task_stack(next_p);
199	refresh_sysenter_cs(next);	199	refresh_sysenter_cs(next);
200	this_cpu_write(cpu_current_top_of_stack,	200	this_cpu_write(pcpu_hot.top_of_stack,
201	(unsigned long)task_stack_page(next_p) +	201	(unsigned long)task_stack_page(next_p) +
202	THREAD_SIZE);	202	THREAD_SIZE);
203		203
@@ -207,7 +207,7 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 3 unmodified lines
207	if (prev->gs \| next->gs)	207	if (prev->gs \| next->gs)
208	loadsegment(gs, next->gs);	208	loadsegment(gs, next->gs);
209		209
210	this_cpu_write(current_task, next_p);	210	raw_cpu_write(pcpu_hot.current_task, next_p);
211		211
212	switch_fpu_finish();	212	switch_fpu_finish();
213		213

arch/x86/kernel/process_64.c+3−3

@@ -563,7 +563,7 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 562 unmodified lines
563	int cpu = smp_processor_id();	563	int cpu = smp_processor_id();
564		564
565	WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&	565	WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
566	this_cpu_read(hardirq_stack_inuse));	566	this_cpu_read(pcpu_hot.hardirq_stack_inuse));
567		567
568	if (!test_thread_flag(TIF_NEED_FPU_LOAD))	568	if (!test_thread_flag(TIF_NEED_FPU_LOAD))
569	switch_fpu_prepare(prev_fpu, cpu);	569	switch_fpu_prepare(prev_fpu, cpu);
@@ -617,8 +617,8 @@ __switch_to(struct task_struct prev_p, struct task_struct next_p) · 47 unmodified lines
617	/*	617	/*
618	* Switch the PDA and FPU contexts.	618	* Switch the PDA and FPU contexts.
619	*/	619	*/
620	this_cpu_write(current_task, next_p);	620	raw_cpu_write(pcpu_hot.current_task, next_p);
621	this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));	621	raw_cpu_write(pcpu_hot.top_of_stack, task_top_of_stack(next_p));
622		622
623	switch_fpu_finish();	623	switch_fpu_finish();
624		624

arch/x86/kernel/relocate_kernel_64.S+3−2

 	.text
 	.align PAGE_SIZE
 	.code64
+SYM_CODE_START_NOALIGN(relocate_range)
 SYM_CODE_START_NOALIGN(relocate_kernel)
 	UNWIND_HINT_EMPTY
 	ANNOTATE_NOENDBR
 	int3
 SYM_CODE_END(swap_pages)
-	.globl kexec_control_code_size
+	.skip KEXEC_CONTROL_CODE_MAX_SIZE - (. - relocate_kernel), 0xcc
-.set kexec_control_code_size, . - relocate_kernel
+SYM_CODE_END(relocate_range);

arch/x86/kernel/setup_percpu.c+2−5

 #include <asm/cpumask.h>
 #include <asm/cpu.h>
-DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
-EXPORT_PER_CPU_SYMBOL(cpu_number);
 #ifdef CONFIG_X86_64
 #define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
 #else
 	for_each_possible_cpu(cpu) {
 		per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu];
 		per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
-		per_cpu(cpu_number, cpu) = cpu;
+		per_cpu(pcpu_hot.cpu_number, cpu) = cpu;
 		setup_percpu_segment(cpu);
 		/*
 		 * Copy data used in early init routines from the
 		 * area.  Reload any changed state for the boot CPU.
 		 */
 		if (!cpu)
-			switch_to_new_gdt(cpu);
+			switch_gdt_and_percpu_base(cpu);
 	}
 	/* indicate the early static arrays will soon be gone */

arch/x86/kernel/smpboot.c+7−3

@@ -1048,7 +1048,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) · 1047 unmodified lines
1048	/* Just in case we booted with a single CPU. */	1048	/* Just in case we booted with a single CPU. */
1049	alternatives_enable_smp();	1049	alternatives_enable_smp();
1050		1050
1051	per_cpu(current_task, cpu) = idle;	1051	per_cpu(pcpu_hot.current_task, cpu) = idle;
1052	cpu_init_stack_canary(cpu, idle);	1052	cpu_init_stack_canary(cpu, idle);
1053		1053
1054	/* Initialize the interrupt stack(s) */	1054	/* Initialize the interrupt stack(s) */
@@ -1058,7 +1058,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle) · 3 unmodified lines
1058		1058
1059	#ifdef CONFIG_X86_32	1059	#ifdef CONFIG_X86_32
1060	/* Stack for startup_32 can be just as for start_secondary onwards */	1060	/* Stack for startup_32 can be just as for start_secondary onwards */
1061	per_cpu(cpu_current_top_of_stack, cpu) = task_top_of_stack(idle);	1061	per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle);
1062	#else	1062	#else
1063	initial_gs = per_cpu_offset(cpu);	1063	initial_gs = per_cpu_offset(cpu);
1064	#endif	1064	#endif
@@ -1453,7 +1453,11 @@ void arch_thaw_secondary_cpus_end(void) · 388 unmodified lines
1453	void __init native_smp_prepare_boot_cpu(void)	1453	void __init native_smp_prepare_boot_cpu(void)
1454	{	1454	{
1455	int me = smp_processor_id();	1455	int me = smp_processor_id();
1456	switch_to_new_gdt(me);	1456
		1457	/* SMP handles this from setup_per_cpu_areas() */
		1458	if (!IS_ENABLED(CONFIG_SMP))
		1459	switch_gdt_and_percpu_base(me);
		1460
1457	/* already set me in cpu_online_mask in boot_cpu_init() */	1461	/* already set me in cpu_online_mask in boot_cpu_init() */
1458	cpumask_set_cpu(me, cpu_callout_mask);	1462	cpumask_set_cpu(me, cpu_callout_mask);
1459	cpu_set_state_online(me);	1463	cpu_set_state_online(me);

arch/x86/kernel/static_call.c+2−1

 	switch (type) {
 	case CALL:
+		func = callthunks_translate_call_dest(func);
 		code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
 		if (func == &__static_call_return0) {
 			emulate = code;
 	case RET:
 		if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
-			code = text_gen_insn(JMP32_INSN_OPCODE, insn, &__x86_return_thunk);
+			code = text_gen_insn(JMP32_INSN_OPCODE, insn, x86_return_thunk);
 		else
 			code = &retinsn;
 		break;

arch/x86/kernel/traps.c+2−2

@@ -858,7 +858,7 @@ DEFINE_IDTENTRY_RAW(exc_int3) · 857 unmodified lines
858	*/	858	*/
859	asmlinkage __visible noinstr struct pt_regs sync_regs(struct pt_regs eregs)	859	asmlinkage __visible noinstr struct pt_regs sync_regs(struct pt_regs eregs)
860	{	860	{
861	struct pt_regs regs = (struct pt_regs )this_cpu_read(cpu_current_top_of_stack) - 1;	861	struct pt_regs regs = (struct pt_regs )this_cpu_read(pcpu_hot.top_of_stack) - 1;
862	if (regs != eregs)	862	if (regs != eregs)
863	regs = eregs;	863	regs = eregs;
864	return regs;	864	return regs;
@@ -876,7 +876,7 @@ asmlinkage __visible noinstr struct pt_regs vc_switch_off_ist(struct pt_regs r · 11 unmodified lines
876	* trust it and switch to the current kernel stack	876	* trust it and switch to the current kernel stack
877	*/	877	*/
878	if (ip_within_syscall_gap(regs)) {	878	if (ip_within_syscall_gap(regs)) {
879	sp = this_cpu_read(cpu_current_top_of_stack);	879	sp = this_cpu_read(pcpu_hot.top_of_stack);
880	goto sync;	880	goto sync;
881	}	881	}
882		882

arch/x86/kernel/unwind_orc.c+20−1

@@ -136,6 +136,21 @@ static struct orc_entry null_orc_entry = { · 135 unmodified lines
136	.type = UNWIND_HINT_TYPE_CALL	136	.type = UNWIND_HINT_TYPE_CALL
137	};	137	};
138		138
		139	#ifdef CONFIG_CALL_THUNKS
		140	static struct orc_entry *orc_callthunk_find(unsigned long ip)
		141	{
		142	if (!is_callthunk((void *)ip))
		143	return NULL;
		144
		145	return &null_orc_entry;
		146	}
		147	#else
		148	static struct orc_entry *orc_callthunk_find(unsigned long ip)
		149	{
		150	return NULL;
		151	}
		152	#endif
		153
139	/* Fake frame pointer entry -- used as a fallback for generated code */	154	/* Fake frame pointer entry -- used as a fallback for generated code */
140	static struct orc_entry orc_fp_entry = {	155	static struct orc_entry orc_fp_entry = {
141	.type = UNWIND_HINT_TYPE_CALL,	156	.type = UNWIND_HINT_TYPE_CALL,
@@ -189,7 +204,11 @@ static struct orc_entry *orc_find(unsigned long ip) · 47 unmodified lines
189	if (orc)	204	if (orc)
190	return orc;	205	return orc;
191		206
192	return orc_ftrace_find(ip);	207	orc = orc_ftrace_find(ip);
		208	if (orc)
		209	return orc;
		210
		211	return orc_callthunk_find(ip);
193	}	212	}
194		213
195	#ifdef CONFIG_MODULES	214	#ifdef CONFIG_MODULES

arch/x86/kernel/vmlinux.lds.S+23−14

 		CPUIDLE_TEXT
 		LOCK_TEXT
 		KPROBES_TEXT
-		ALIGN_ENTRY_TEXT_BEGIN
-		ENTRY_TEXT
-		ALIGN_ENTRY_TEXT_END
 		SOFTIRQENTRY_TEXT
-		STATIC_CALL_TEXT
-		*(.gnu.warning)
 #ifdef CONFIG_RETPOLINE
 		__indirect_thunk_start = .;
 		*(.text.__x86.*)
 		__indirect_thunk_end = .;
 #endif
+		STATIC_CALL_TEXT
+		ALIGN_ENTRY_TEXT_BEGIN
+		ENTRY_TEXT
+		ALIGN_ENTRY_TEXT_END
+		*(.gnu.warning)
 	} :text =0xcccc
 	/* End of text section, which should occupy whole number of pages */
 		*(.return_sites)
 		__return_sites_end = .;
 	}
+	. = ALIGN(8);
+	.call_sites : AT(ADDR(.call_sites) - LOAD_OFFSET) {
+		__call_sites = .;
+		*(.call_sites)
+		__call_sites_end = .;
+	}
 #endif
 #ifdef CONFIG_X86_KERNEL_IBT
 	}
 #endif
+#ifdef CONFIG_FINEIBT
+	. = ALIGN(8);
+	.cfi_sites : AT(ADDR(.cfi_sites) - LOAD_OFFSET) {
+		__cfi_sites = .;
+		*(.cfi_sites)
+		__cfi_sites_end = .;
+	}
+#endif
 	/*
 	 * struct alt_inst entries. From the header (alternative.h):
 	 * "Alternative instructions for different CPU types or capabilities"
 #endif
 #endif /* CONFIG_X86_64 */
-#ifdef CONFIG_KEXEC_CORE
-#include <asm/kexec.h>
-. = ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
-           "kexec control code size is too big");
-#endif

arch/x86/kvm/svm/vmenter.S+1−0

@@ -1,6 +1,7 @@
1	/* SPDX-License-Identifier: GPL-2.0 */	1	/* SPDX-License-Identifier: GPL-2.0 */
2	#include <linux/linkage.h>	2	#include <linux/linkage.h>
3	#include <asm/asm.h>	3	#include <asm/asm.h>
		4	#include <asm/asm-offsets.h>
4	#include <asm/bitsperlong.h>	5	#include <asm/bitsperlong.h>
5	#include <asm/kvm_vcpu_regs.h>	6	#include <asm/kvm_vcpu_regs.h>
6	#include <asm/nospec-branch.h>	7	#include <asm/nospec-branch.h>

arch/x86/lib/error-inject.c+1−0

@@ -11,6 +11,7 @@ asm( · 10 unmodified lines
11	".text\n"	11	".text\n"
12	".type just_return_func, @function\n"	12	".type just_return_func, @function\n"
13	".globl just_return_func\n"	13	".globl just_return_func\n"
		14	ASM_FUNC_ALIGN
14	"just_return_func:\n"	15	"just_return_func:\n"
15	ANNOTATE_NOENDBR	16	ANNOTATE_NOENDBR
16	ASM_RET	17	ASM_RET

arch/x86/lib/putuser.S+49−13

 	LOAD_TASK_SIZE_MINUS_N(0)
 	cmp %_ASM_BX,%_ASM_CX
 	jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
-	ENDBR
 	ASM_STAC
 1:	movb %al,(%_ASM_CX)
 	xor %ecx,%ecx
 	RET
 SYM_FUNC_END(__put_user_1)
 EXPORT_SYMBOL(__put_user_1)
+SYM_FUNC_START(__put_user_nocheck_1)
+	ENDBR
+	ASM_STAC
+2:	movb %al,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_1)
 EXPORT_SYMBOL(__put_user_nocheck_1)
 SYM_FUNC_START(__put_user_2)
 	LOAD_TASK_SIZE_MINUS_N(1)
 	cmp %_ASM_BX,%_ASM_CX
 	jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
-	ENDBR
 	ASM_STAC
-2:	movw %ax,(%_ASM_CX)
+3:	movw %ax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_2)
 EXPORT_SYMBOL(__put_user_2)
+SYM_FUNC_START(__put_user_nocheck_2)
+	ENDBR
+	ASM_STAC
+4:	movw %ax,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_2)
 EXPORT_SYMBOL(__put_user_nocheck_2)
 SYM_FUNC_START(__put_user_4)
 	LOAD_TASK_SIZE_MINUS_N(3)
 	cmp %_ASM_BX,%_ASM_CX
 	jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
-	ENDBR
 	ASM_STAC
-3:	movl %eax,(%_ASM_CX)
+5:	movl %eax,(%_ASM_CX)
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_4)
 EXPORT_SYMBOL(__put_user_4)
+SYM_FUNC_START(__put_user_nocheck_4)
+	ENDBR
+	ASM_STAC
+6:	movl %eax,(%_ASM_CX)
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_4)
 EXPORT_SYMBOL(__put_user_nocheck_4)
 SYM_FUNC_START(__put_user_8)
 	LOAD_TASK_SIZE_MINUS_N(7)
 	cmp %_ASM_BX,%_ASM_CX
 	jae .Lbad_put_user
-SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
-	ENDBR
 	ASM_STAC
-4:	mov %_ASM_AX,(%_ASM_CX)
+7:	mov %_ASM_AX,(%_ASM_CX)
 #ifdef CONFIG_X86_32
-5:	movl %edx,4(%_ASM_CX)
+8:	movl %edx,4(%_ASM_CX)
 #endif
 	xor %ecx,%ecx
 	ASM_CLAC
 	RET
 SYM_FUNC_END(__put_user_8)
 EXPORT_SYMBOL(__put_user_8)
+SYM_FUNC_START(__put_user_nocheck_8)
+	ENDBR
+	ASM_STAC
+9:	mov %_ASM_AX,(%_ASM_CX)
+#ifdef CONFIG_X86_32
+10:	movl %edx,4(%_ASM_CX)
+#endif
+	xor %ecx,%ecx
+	ASM_CLAC
+	RET
+SYM_FUNC_END(__put_user_nocheck_8)
 EXPORT_SYMBOL(__put_user_nocheck_8)
 SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
 	_ASM_EXTABLE_UA(2b, .Lbad_put_user_clac)
 	_ASM_EXTABLE_UA(3b, .Lbad_put_user_clac)
 	_ASM_EXTABLE_UA(4b, .Lbad_put_user_clac)
-#ifdef CONFIG_X86_32
 	_ASM_EXTABLE_UA(5b, .Lbad_put_user_clac)
+	_ASM_EXTABLE_UA(6b, .Lbad_put_user_clac)
+	_ASM_EXTABLE_UA(7b, .Lbad_put_user_clac)
+	_ASM_EXTABLE_UA(9b, .Lbad_put_user_clac)
+#ifdef CONFIG_X86_32
+	_ASM_EXTABLE_UA(8b, .Lbad_put_user_clac)
+	_ASM_EXTABLE_UA(10b, .Lbad_put_user_clac)
 #endif

arch/x86/lib/retpoline.S+99−8

@@ -5,24 +5,27 @@ · 4 unmodified lines
5	#include <asm/dwarf2.h>	5	#include <asm/dwarf2.h>
6	#include <asm/cpufeatures.h>	6	#include <asm/cpufeatures.h>
7	#include <asm/alternative.h>	7	#include <asm/alternative.h>
		8	#include <asm/asm-offsets.h>
8	#include <asm/export.h>	9	#include <asm/export.h>
9	#include <asm/nospec-branch.h>	10	#include <asm/nospec-branch.h>
10	#include <asm/unwind_hints.h>	11	#include <asm/unwind_hints.h>
		12	#include <asm/percpu.h>
11	#include <asm/frame.h>	13	#include <asm/frame.h>
12		14
13	.section .text.__x86.indirect_thunk	15	.section .text.__x86.indirect_thunk
14		16
15	.macro RETPOLINE reg	17
		18	.macro POLINE reg
16	ANNOTATE_INTRA_FUNCTION_CALL	19	ANNOTATE_INTRA_FUNCTION_CALL
17	call .Ldo_rop_\@	20	call .Ldo_rop_\@
18	.Lspec_trap_\@:	21	int3
19	UNWIND_HINT_EMPTY
20	pause
21	lfence
22	jmp .Lspec_trap_\@
23	.Ldo_rop_\@:	22	.Ldo_rop_\@:
24	mov %\reg, (%_ASM_SP)	23	mov %\reg, (%_ASM_SP)
25	UNWIND_HINT_FUNC	24	UNWIND_HINT_FUNC
		25	.endm
		26
		27	.macro RETPOLINE reg
		28	POLINE \reg
26	RET	29	RET
27	.endm	30	.endm
28		31
@@ -52,7 +55,6 @@ SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) · 23 unmodified lines
52	*/	55	*/
53		56
54	#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)	57	#define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym)
55	#define EXPORT_THUNK(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
56		58
57	.align RETPOLINE_THUNK_SIZE	59	.align RETPOLINE_THUNK_SIZE
58	SYM_CODE_START(__x86_indirect_thunk_array)	60	SYM_CODE_START(__x86_indirect_thunk_array)
@@ -64,10 +66,65 @@ SYM_CODE_START(__x86_indirect_thunk_array) · 5 unmodified lines
64	.align RETPOLINE_THUNK_SIZE	66	.align RETPOLINE_THUNK_SIZE
65	SYM_CODE_END(__x86_indirect_thunk_array)	67	SYM_CODE_END(__x86_indirect_thunk_array)
66		68
67	#define GEN(reg) EXPORT_THUNK(reg)	69	#define GEN(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg)
		70	#include <asm/GEN-for-each-reg.h>
		71	#undef GEN
		72
		73	#ifdef CONFIG_CALL_DEPTH_TRACKING
		74	.macro CALL_THUNK reg
		75	.align RETPOLINE_THUNK_SIZE
		76
		77	SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL)
		78	UNWIND_HINT_EMPTY
		79	ANNOTATE_NOENDBR
		80
		81	CALL_DEPTH_ACCOUNT
		82	POLINE \reg
		83	ANNOTATE_UNRET_SAFE
		84	ret
		85	int3
		86	.endm
		87
		88	.align RETPOLINE_THUNK_SIZE
		89	SYM_CODE_START(__x86_indirect_call_thunk_array)
		90
		91	#define GEN(reg) CALL_THUNK reg
68	#include <asm/GEN-for-each-reg.h>	92	#include <asm/GEN-for-each-reg.h>
69	#undef GEN	93	#undef GEN
70		94
		95	.align RETPOLINE_THUNK_SIZE
		96	SYM_CODE_END(__x86_indirect_call_thunk_array)
		97
		98	#define GEN(reg) __EXPORT_THUNK(__x86_indirect_call_thunk_ ## reg)
		99	#include <asm/GEN-for-each-reg.h>
		100	#undef GEN
		101
		102	.macro JUMP_THUNK reg
		103	.align RETPOLINE_THUNK_SIZE
		104
		105	SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL)
		106	UNWIND_HINT_EMPTY
		107	ANNOTATE_NOENDBR
		108	POLINE \reg
		109	ANNOTATE_UNRET_SAFE
		110	ret
		111	int3
		112	.endm
		113
		114	.align RETPOLINE_THUNK_SIZE
		115	SYM_CODE_START(__x86_indirect_jump_thunk_array)
		116
		117	#define GEN(reg) JUMP_THUNK reg
		118	#include <asm/GEN-for-each-reg.h>
		119	#undef GEN
		120
		121	.align RETPOLINE_THUNK_SIZE
		122	SYM_CODE_END(__x86_indirect_jump_thunk_array)
		123
		124	#define GEN(reg) __EXPORT_THUNK(__x86_indirect_jump_thunk_ ## reg)
		125	#include <asm/GEN-for-each-reg.h>
		126	#undef GEN
		127	#endif
71	/*	128	/*
72	* This function name is magical and is used by -mfunction-return=thunk-extern	129	* This function name is magical and is used by -mfunction-return=thunk-extern
73	* for the compiler to generate JMPs to it.	130	* for the compiler to generate JMPs to it.
@@ -140,3 +197,37 @@ __EXPORT_THUNK(zen_untrain_ret) · 66 unmodified lines
140	EXPORT_SYMBOL(__x86_return_thunk)	197	EXPORT_SYMBOL(__x86_return_thunk)
141		198
142	#endif /* CONFIG_RETHUNK */	199	#endif /* CONFIG_RETHUNK */
		200
		201	#ifdef CONFIG_CALL_DEPTH_TRACKING
		202
		203	.align 64
		204	SYM_FUNC_START(__x86_return_skl)
		205	ANNOTATE_NOENDBR
		206	/*
		207	* Keep the hotpath in a 16byte I-fetch for the non-debug
		208	* case.
		209	*/
		210	CALL_THUNKS_DEBUG_INC_RETS
		211	shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth)
		212	jz 1f
		213	ANNOTATE_UNRET_SAFE
		214	ret
		215	int3
		216	1:
		217	CALL_THUNKS_DEBUG_INC_STUFFS
		218	.rept 16
		219	ANNOTATE_INTRA_FUNCTION_CALL
		220	call 2f
		221	int3
		222	2:
		223	.endr
		224	add $(8*16), %rsp
		225
		226	CREDIT_CALL_DEPTH
		227
		228	ANNOTATE_UNRET_SAFE
		229	ret
		230	int3
		231	SYM_FUNC_END(__x86_return_skl)
		232
		233	#endif /* CONFIG_CALL_DEPTH_TRACKING */

arch/x86/net/bpf_jit_comp.c+34−11

 #include <linux/memory.h>
 #include <linux/sort.h>
 #include <asm/extable.h>
+#include <asm/ftrace.h>
 #include <asm/set_memory.h>
 #include <asm/nospec-branch.h>
 #include <asm/text-patching.h>
 	return emit_patch(pprog, func, ip, 0xE8);
 }
+static int emit_rsb_call(u8 **pprog, void *func, void *ip)
+{
+	OPTIMIZER_HIDE_VAR(func);
+	x86_call_depth_emit_accounting(pprog, func);
+	return emit_patch(pprog, func, ip, 0xE8);
+}
 static int emit_jump(u8 **pprog, void *func, void *ip)
 {
 	return emit_patch(pprog, func, ip, 0xE9);
 		EMIT2(0xFF, 0xE0 + reg);
 	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
 		OPTIMIZER_HIDE_VAR(reg);
-		emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
+		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
+			emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);
+		else
+			emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
 	} else {
 		EMIT2(0xFF, 0xE0 + reg);	/* jmp *%\reg */
 		if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS))
 	u8 *prog = *pprog;
 	if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
-		emit_jump(&prog, &__x86_return_thunk, ip);
+		emit_jump(&prog, x86_return_thunk, ip);
 	} else {
 		EMIT1(0xC3);		/* ret */
 		if (IS_ENABLED(CONFIG_SLS))
 			break;
 			/* call */
-		case BPF_JMP | BPF_CALL:
+		case BPF_JMP | BPF_CALL: {
+			int offs;
 			func = (u8 *) __bpf_call_base + imm32;
 			if (tail_call_reachable) {
 				/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
 				EMIT3_off32(0x48, 0x8B, 0x85,
 					    -round_up(bpf_prog->aux->stack_depth, 8) - 8);
-				if (!imm32 || emit_call(&prog, func, image + addrs[i - 1] + 7))
+				if (!imm32)
 					return -EINVAL;
+				offs = 7 + x86_call_depth_emit_accounting(&prog, func);
 			} else {
-				if (!imm32 || emit_call(&prog, func, image + addrs[i - 1]))
+				if (!imm32)
 					return -EINVAL;
+				offs = x86_call_depth_emit_accounting(&prog, func);
 			}
+			if (emit_call(&prog, func, image + addrs[i - 1] + offs))
+				return -EINVAL;
 			break;
+		}
 		case BPF_JMP | BPF_TAIL_CALL:
 			if (imm32)
 	/* arg2: lea rsi, [rbp - ctx_cookie_off] */
 	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
-	if (emit_call(&prog, bpf_trampoline_enter(p), prog))
+	if (emit_rsb_call(&prog, bpf_trampoline_enter(p), prog))
 		return -EINVAL;
 	/* remember prog start time returned by __bpf_prog_enter */
 	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
 			       (long) p->insnsi >> 32,
 			       (u32) (long) p->insnsi);
 	/* call JITed bpf program or interpreter */
-	if (emit_call(&prog, p->bpf_func, prog))
+	if (emit_rsb_call(&prog, p->bpf_func, prog))
 		return -EINVAL;
 	/*
 	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
 	/* arg3: lea rdx, [rbp - run_ctx_off] */
 	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
-	if (emit_call(&prog, bpf_trampoline_exit(p), prog))
+	if (emit_rsb_call(&prog, bpf_trampoline_exit(p), prog))
 		return -EINVAL;
 	*pprog = prog;
 	prog = image;
 	EMIT_ENDBR();
+	/*
+	 * This is the direct-call trampoline, as such it needs accounting
+	 * for the __fentry__ call.
+	 */
+	x86_call_depth_emit_accounting(&prog, NULL);
 	EMIT1(0x55);		 /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
 	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
 		/* arg1: mov rdi, im */
 		emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
-		if (emit_call(&prog, __bpf_tramp_enter, prog)) {
+		if (emit_rsb_call(&prog, __bpf_tramp_enter, prog)) {
 			ret = -EINVAL;
 			goto cleanup;
 		}
 			EMIT2(0xff, 0xd0); /* call *rax */
 		} else {
 			/* call original function */
-			if (emit_call(&prog, orig_call, prog)) {
+			if (emit_rsb_call(&prog, orig_call, prog)) {
 				ret = -EINVAL;
 				goto cleanup;
 			}
 		im->ip_epilogue = prog;
 		/* arg1: mov rdi, im */
 		emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
-		if (emit_call(&prog, __bpf_tramp_exit, prog)) {
+		if (emit_rsb_call(&prog, __bpf_tramp_exit, prog)) {
 			ret = -EINVAL;
 			goto cleanup;
 		}

arch/x86/power/hibernate.c+1−1

@@ -159,7 +159,7 @@ int relocate_restore_code(void) · 158 unmodified lines
159	if (!relocated_restore_code)	159	if (!relocated_restore_code)
160	return -ENOMEM;	160	return -ENOMEM;
161		161
162	memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);	162	__memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);
163		163
164	/* Make the page containing the relocated code executable */	164	/* Make the page containing the relocated code executable */
165	pgd = (pgd_t *)__va(read_cr3_pa()) +	165	pgd = (pgd_t *)__va(read_cr3_pa()) +

arch/x86/xen/enlighten_pv.c+1−1

@@ -1210,7 +1210,7 @@ static void __init xen_setup_gdt(int cpu) · 1209 unmodified lines
1210	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot;	1210	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot;
1211	pv_ops.cpu.load_gdt = xen_load_gdt_boot;	1211	pv_ops.cpu.load_gdt = xen_load_gdt_boot;
1212		1212
1213	switch_to_new_gdt(cpu);	1213	switch_gdt_and_percpu_base(cpu);
1214		1214
1215	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry;	1215	pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry;
1216	pv_ops.cpu.load_gdt = xen_load_gdt;	1216	pv_ops.cpu.load_gdt = xen_load_gdt;

include/asm-generic/vmlinux.lds.h+2−2

@@ -81,8 +81,8 @@ · 80 unmodified lines
81	#define RO_EXCEPTION_TABLE	81	#define RO_EXCEPTION_TABLE
82	#endif	82	#endif
83		83
84	/* Align . to a 8 byte boundary equals to maximum function alignment. */	84	/* Align . function alignment. */
85	#define ALIGN_FUNCTION() . = ALIGN(8)	85	#define ALIGN_FUNCTION() . = ALIGN(CONFIG_FUNCTION_ALIGNMENT)
86		86
87	/*	87	/*
88	* LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which	88	* LD_DEAD_CODE_DATA_ELIMINATION option enables -fdata-sections, which

include/linux/linkage.h+2−2

@@ -69,8 +69,8 @@ · 68 unmodified lines
69	#endif	69	#endif
70		70
71	#ifndef __ALIGN	71	#ifndef __ALIGN
72	#define __ALIGN .align 4,0x90	72	#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT
73	#define __ALIGN_STR ".align 4,0x90"	73	#define __ALIGN_STR __stringify(__ALIGN)
74	#endif	74	#endif
75		75
76	#ifdef __ASSEMBLY__	76	#ifdef __ASSEMBLY__

include/linux/static_call.h+2−0

@@ -162,6 +162,8 @@ extern void arch_static_call_transform(void site, void tramp, void *func, bool · 161 unmodified lines
162		162
163	extern int __init static_call_init(void);	163	extern int __init static_call_init(void);
164		164
		165	extern void static_call_force_reinit(void);
		166
165	struct static_call_mod {	167	struct static_call_mod {
166	struct static_call_mod *next;	168	struct static_call_mod *next;
167	struct module mod; / for vmlinux, mod == NULL */	169	struct module mod; / for vmlinux, mod == NULL */

kernel/static_call_inline.c+18−5

 extern struct static_call_tramp_key __start_static_call_tramp_key[],
 				    __stop_static_call_tramp_key[];
-static bool static_call_initialized;
+static int static_call_initialized;
+/*
+ * Must be called before early_initcall() to be effective.
+ */
+void static_call_force_reinit(void)
+{
+	if (WARN_ON_ONCE(!static_call_initialized))
+		return;
+	static_call_initialized++;
+}
 /* mutex to protect key modules/sites */
 static DEFINE_MUTEX(static_call_mutex);
 {
 	int ret;
-	if (static_call_initialized)
+	/* See static_call_force_reinit(). */
+	if (static_call_initialized == 1)
 		return 0;
 	cpus_read_lock();
 		BUG();
 	}
-	static_call_initialized = true;
 #ifdef CONFIG_MODULES
-	register_module_notifier(&static_call_module_nb);
+	if (!static_call_initialized)
+		register_module_notifier(&static_call_module_nb);
 #endif
+	static_call_initialized = 1;
 	return 0;
 }
 early_initcall(static_call_init);

kernel/trace/trace_selftest.c+8−1

 };
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
-noinline __noclone static void trace_direct_tramp(void) { }
+#ifndef CALL_DEPTH_ACCOUNT
+#define CALL_DEPTH_ACCOUNT ""
+#endif
+noinline __noclone static void trace_direct_tramp(void)
+{
+	asm(CALL_DEPTH_ACCOUNT);
+}
 #endif
 /*

lib/Kconfig.debug+1−0

@@ -469,6 +469,7 @@ config SECTION_MISMATCH_WARN_ONLY · 468 unmodified lines
469	config DEBUG_FORCE_FUNCTION_ALIGN_64B	469	config DEBUG_FORCE_FUNCTION_ALIGN_64B
470	bool "Force all function address 64B aligned"	470	bool "Force all function address 64B aligned"
471	depends on EXPERT && (X86_64 \|\| ARM64 \|\| PPC32 \|\| PPC64 \|\| ARC)	471	depends on EXPERT && (X86_64 \|\| ARM64 \|\| PPC32 \|\| PPC64 \|\| ARC)
		472	select FUNCTION_ALIGNMENT_64B
472	help	473	help
473	There are cases that a commit from one domain changes the function	474	There are cases that a commit from one domain changes the function
474	address alignment of other domains, and cause magic performance	475	address alignment of other domains, and cause magic performance

samples/ftrace/ftrace-direct-modify.c+3−0

 #include <linux/kthread.h>
 #include <linux/ftrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/nospec-branch.h>
 extern void my_direct_func1(void);
 extern void my_direct_func2(void);
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	call my_direct_func1\n"
 "	leave\n"
 "	.size		my_tramp1, .-my_tramp1\n"
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	call my_direct_func2\n"
 "	leave\n"
 	ASM_RET

samples/ftrace/ftrace-direct-multi-modify.c+3−0

 #include <linux/kthread.h>
 #include <linux/ftrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/nospec-branch.h>
 extern void my_direct_func1(unsigned long ip);
 extern void my_direct_func2(unsigned long ip);
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	pushq %rdi\n"
 "	movq 8(%rbp), %rdi\n"
 "	call my_direct_func1\n"
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	pushq %rdi\n"
 "	movq 8(%rbp), %rdi\n"
 "	call my_direct_func2\n"

samples/ftrace/ftrace-direct-multi.c+2−0

 #include <linux/ftrace.h>
 #include <linux/sched/stat.h>
 #include <asm/asm-offsets.h>
+#include <asm/nospec-branch.h>
 extern void my_direct_func(unsigned long ip);
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	pushq %rdi\n"
 "	movq 8(%rbp), %rdi\n"
 "	call my_direct_func\n"

samples/ftrace/ftrace-direct-too.c+2−0

 #include <linux/mm.h> /* for handle_mm_fault() */
 #include <linux/ftrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/nospec-branch.h>
 extern void my_direct_func(struct vm_area_struct *vma,
 			   unsigned long address, unsigned int flags);
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	pushq %rdi\n"
 "	pushq %rsi\n"
 "	pushq %rdx\n"

samples/ftrace/ftrace-direct.c+2−0

 #include <linux/sched.h> /* for wake_up_process() */
 #include <linux/ftrace.h>
 #include <asm/asm-offsets.h>
+#include <asm/nospec-branch.h>
 extern void my_direct_func(struct task_struct *p);
 	ASM_ENDBR
 "	pushq %rbp\n"
 "	movq %rsp, %rbp\n"
+	CALL_DEPTH_ACCOUNT
 "	pushq %rdi\n"
 "	call my_direct_func\n"
 "	popq %rdi\n"

scripts/Makefile.lib+3−0

 objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK)		+= --hacks=jump_label
 objtool-args-$(CONFIG_HAVE_NOINSTR_HACK)		+= --hacks=noinstr
+objtool-args-$(CONFIG_CALL_DEPTH_TRACKING)		+= --hacks=skylake
 objtool-args-$(CONFIG_X86_KERNEL_IBT)			+= --ibt
+objtool-args-$(CONFIG_FINEIBT)				+= --cfi
 objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL)	+= --mcount
 objtool-args-$(CONFIG_UNWINDER_ORC)			+= --orc
 objtool-args-$(CONFIG_RETPOLINE)			+= --retpoline
 objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE)		+= --static-call
 objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION)		+= --uaccess
 objtool-args-$(CONFIG_GCOV_KERNEL)			+= --no-unreachable
+objtool-args-$(CONFIG_PREFIX_SYMBOLS)			+= --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
 objtool-args = $(objtool-args-y)					\
 	$(if $(delay-objtool), --link)					\

tools/include/linux/interval_tree_generic.h+187−0

@@ -0,0 +1,187 @@
	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
	2	/*
	3	Interval Trees
	4	(C) 2012 Michel Lespinasse <walken@google.com>
	5
	6
	7	include/linux/interval_tree_generic.h
	8	*/
	9
	10	#include <linux/rbtree_augmented.h>
	11
	12	/*
	13	* Template for implementing interval trees
	14	*
	15	* ITSTRUCT: struct type of the interval tree nodes
	16	* ITRB: name of struct rb_node field within ITSTRUCT
	17	* ITTYPE: type of the interval endpoints
	18	* ITSUBTREE: name of ITTYPE field within ITSTRUCT holding last-in-subtree
	19	* ITSTART(n): start endpoint of ITSTRUCT node n
	20	* ITLAST(n): last endpoint of ITSTRUCT node n
	21	* ITSTATIC: 'static' or empty
	22	* ITPREFIX: prefix to use for the inline tree definitions
	23	*
	24	* Note - before using this, please consider if generic version
	25	* (interval_tree.h) would work for you...
	26	*/
	27
	28	#define INTERVAL_TREE_DEFINE(ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, \
	29	ITSTART, ITLAST, ITSTATIC, ITPREFIX) \
	30	\
	31	/* Callbacks for augmented rbtree insert and remove */ \
	32	\
	33	RB_DECLARE_CALLBACKS_MAX(static, ITPREFIX ## _augment, \
	34	ITSTRUCT, ITRB, ITTYPE, ITSUBTREE, ITLAST) \
	35	\
	36	/* Insert / remove interval nodes from the tree */ \
	37	\
	38	ITSTATIC void ITPREFIX ## _insert(ITSTRUCT *node, \
	39	struct rb_root_cached *root) \
	40	{ \
	41	struct rb_node *link = &root->rb_root.rb_node, rb_parent = NULL; \
	42	ITTYPE start = ITSTART(node), last = ITLAST(node); \
	43	ITSTRUCT *parent; \
	44	bool leftmost = true; \
	45	\
	46	while (*link) { \
	47	rb_parent = *link; \
	48	parent = rb_entry(rb_parent, ITSTRUCT, ITRB); \
	49	if (parent->ITSUBTREE < last) \
	50	parent->ITSUBTREE = last; \
	51	if (start < ITSTART(parent)) \
	52	link = &parent->ITRB.rb_left; \
	53	else { \
	54	link = &parent->ITRB.rb_right; \
	55	leftmost = false; \
	56	} \
	57	} \
	58	\
	59	node->ITSUBTREE = last; \
	60	rb_link_node(&node->ITRB, rb_parent, link); \
	61	rb_insert_augmented_cached(&node->ITRB, root, \
	62	leftmost, &ITPREFIX ## _augment); \
	63	} \
	64	\
	65	ITSTATIC void ITPREFIX ## _remove(ITSTRUCT *node, \
	66	struct rb_root_cached *root) \
	67	{ \
	68	rb_erase_augmented_cached(&node->ITRB, root, &ITPREFIX ## _augment); \
	69	} \
	70	\
	71	/* \
	72	* Iterate over intervals intersecting [start;last] \
	73	* \
	74	* Note that a node's interval intersects [start;last] iff: \
	75	* Cond1: ITSTART(node) <= last \
	76	* and \
	77	* Cond2: start <= ITLAST(node) \
	78	*/ \
	79	\
	80	static ITSTRUCT * \
	81	ITPREFIX ## _subtree_search(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
	82	{ \
	83	while (true) { \
	84	/* \
	85	* Loop invariant: start <= node->ITSUBTREE \
	86	* (Cond2 is satisfied by one of the subtree nodes) \
	87	*/ \
	88	if (node->ITRB.rb_left) { \
	89	ITSTRUCT *left = rb_entry(node->ITRB.rb_left, \
	90	ITSTRUCT, ITRB); \
	91	if (start <= left->ITSUBTREE) { \
	92	/* \
	93	* Some nodes in left subtree satisfy Cond2. \
	94	* Iterate to find the leftmost such node N. \
	95	* If it also satisfies Cond1, that's the \
	96	* match we are looking for. Otherwise, there \
	97	* is no matching interval as nodes to the \
	98	* right of N can't satisfy Cond1 either. \
	99	*/ \
	100	node = left; \
	101	continue; \
	102	} \
	103	} \
	104	if (ITSTART(node) <= last) { /* Cond1 */ \
	105	if (start <= ITLAST(node)) /* Cond2 */ \
	106	return node; /* node is leftmost match */ \
	107	if (node->ITRB.rb_right) { \
	108	node = rb_entry(node->ITRB.rb_right, \
	109	ITSTRUCT, ITRB); \
	110	if (start <= node->ITSUBTREE) \
	111	continue; \
	112	} \
	113	} \
	114	return NULL; /* No match */ \
	115	} \
	116	} \
	117	\
	118	ITSTATIC ITSTRUCT * \
	119	ITPREFIX ## _iter_first(struct rb_root_cached *root, \
	120	ITTYPE start, ITTYPE last) \
	121	{ \
	122	ITSTRUCT node, leftmost; \
	123	\
	124	if (!root->rb_root.rb_node) \
	125	return NULL; \
	126	\
	127	/* \
	128	* Fastpath range intersection/overlap between A: [a0, a1] and \
	129	* B: [b0, b1] is given by: \
	130	* \
	131	* a0 <= b1 && b0 <= a1 \
	132	* \
	133	* ... where A holds the lock range and B holds the smallest \
	134	* 'start' and largest 'last' in the tree. For the later, we \
	135	* rely on the root node, which by augmented interval tree \
	136	* property, holds the largest value in its last-in-subtree. \
	137	* This allows mitigating some of the tree walk overhead for \
	138	* for non-intersecting ranges, maintained and consulted in O(1). \
	139	*/ \
	140	node = rb_entry(root->rb_root.rb_node, ITSTRUCT, ITRB); \
	141	if (node->ITSUBTREE < start) \
	142	return NULL; \
	143	\
	144	leftmost = rb_entry(root->rb_leftmost, ITSTRUCT, ITRB); \
	145	if (ITSTART(leftmost) > last) \
	146	return NULL; \
	147	\
	148	return ITPREFIX ## _subtree_search(node, start, last); \
	149	} \
	150	\
	151	ITSTATIC ITSTRUCT * \
	152	ITPREFIX ## _iter_next(ITSTRUCT *node, ITTYPE start, ITTYPE last) \
	153	{ \
	154	struct rb_node rb = node->ITRB.rb_right, prev; \
	155	\
	156	while (true) { \
	157	/* \
	158	* Loop invariants: \
	159	* Cond1: ITSTART(node) <= last \
	160	* rb == node->ITRB.rb_right \
	161	* \
	162	* First, search right subtree if suitable \
	163	*/ \
	164	if (rb) { \
	165	ITSTRUCT *right = rb_entry(rb, ITSTRUCT, ITRB); \
	166	if (start <= right->ITSUBTREE) \
	167	return ITPREFIX ## _subtree_search(right, \
	168	start, last); \
	169	} \
	170	\
	171	/* Move up the tree until we come from a node's left child */ \
	172	do { \
	173	rb = rb_parent(&node->ITRB); \
	174	if (!rb) \
	175	return NULL; \
	176	prev = &node->ITRB; \
	177	node = rb_entry(rb, ITSTRUCT, ITRB); \
	178	rb = node->ITRB.rb_right; \
	179	} while (prev == rb); \
	180	\
	181	/* Check if the node intersects [start;last] */ \
	182	if (last < ITSTART(node)) /* !Cond1 */ \
	183	return NULL; \
	184	else if (start <= ITLAST(node)) /* Cond2 */ \
	185	return node; \
	186	} \
	187	}

tools/objtool/arch/x86/decode.c+24−0

@@ -73,6 +73,30 @@ unsigned long arch_jump_destination(struct instruction *insn) · 72 unmodified lines
73	return insn->offset + insn->len + insn->immediate;	73	return insn->offset + insn->len + insn->immediate;
74	}	74	}
75		75
		76	bool arch_pc_relative_reloc(struct reloc *reloc)
		77	{
		78	/*
		79	* All relocation types where P (the address of the target)
		80	* is included in the computation.
		81	*/
		82	switch (reloc->type) {
		83	case R_X86_64_PC8:
		84	case R_X86_64_PC16:
		85	case R_X86_64_PC32:
		86	case R_X86_64_PC64:
		87
		88	case R_X86_64_PLT32:
		89	case R_X86_64_GOTPC32:
		90	case R_X86_64_GOTPCREL:
		91	return true;
		92
		93	default:
		94	break;
		95	}
		96
		97	return false;
		98	}
		99
76	#define ADD_OP(op) \	100	#define ADD_OP(op) \
77	if (!(op = calloc(1, sizeof(*op)))) \	101	if (!(op = calloc(1, sizeof(*op)))) \
78	return -1; \	102	return -1; \

tools/objtool/builtin-check.c+8−1

 		found = true;
 	}
+	if (!str || strstr(str, "skylake")) {
+		opts.hack_skylake = true;
+		found = true;
+	}
 	return found ? 0 : -1;
 }
 const struct option check_options[] = {
 	OPT_GROUP("Actions:"),
-	OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr", "patch toolchain bugs/limitations", parse_hacks),
+	OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr,skylake", "patch toolchain bugs/limitations", parse_hacks),
 	OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"),
 	OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"),
 	OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"),
 	OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"),
 	OPT_BOOLEAN(0,   "rethunk", &opts.rethunk, "validate and annotate rethunk usage"),
 	OPT_BOOLEAN(0,   "unret", &opts.unret, "validate entry unret placement"),
+	OPT_INTEGER(0,   "prefix", &opts.prefix, "generate prefix symbols"),
 	OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"),
 	OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"),
 	OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"),
 	OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"),
+	OPT_BOOLEAN(0  , "cfi", &opts.cfi, "annotate kernel control flow integrity (kCFI) function preambles"),
 	OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump),
 	OPT_GROUP("Options:"),

tools/objtool/check.c+310−81

@@ -62,12 +62,12 @@ static struct instruction next_insn_same_func(struct objtool_file file, · 61 unmodified lines
62	struct instruction *insn)	62	struct instruction *insn)
63	{	63	{
64	struct instruction *next = list_next_entry(insn, list);	64	struct instruction *next = list_next_entry(insn, list);
65	struct symbol *func = insn->func;	65	struct symbol *func = insn_func(insn);
66		66
67	if (!func)	67	if (!func)
68	return NULL;	68	return NULL;
69		69
70	if (&next->list != &file->insn_list && next->func == func)	70	if (&next->list != &file->insn_list && insn_func(next) == func)
71	return next;	71	return next;
72		72
73	/* Check if we're already in the subfunction: */	73	/* Check if we're already in the subfunction: */
@@ -83,7 +83,7 @@ static struct instruction prev_insn_same_sym(struct objtool_file file, · 9 unmodified lines
83	{	83	{
84	struct instruction *prev = list_prev_entry(insn, list);	84	struct instruction *prev = list_prev_entry(insn, list);
85		85
86	if (&prev->list != &file->insn_list && prev->func == insn->func)	86	if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
87	return prev;	87	return prev;
88		88
89	return NULL;	89	return NULL;
@@ -129,16 +129,13 @@ static bool is_jump_table_jump(struct instruction *insn) · 39 unmodified lines
129	static bool is_sibling_call(struct instruction *insn)	129	static bool is_sibling_call(struct instruction *insn)
130	{	130	{
131	/*	131	/*
132	* Assume only ELF functions can make sibling calls. This ensures	132	* Assume only STT_FUNC calls have jump-tables.
133	* sibling call detection consistency between vmlinux.o and individual
134	* objects.
135	*/	133	*/
136	if (!insn->func)	134	if (insn_func(insn)) {
137	return false;	135	/* An indirect jump is either a sibling call or a jump to a table. */
138		136	if (insn->type == INSN_JUMP_DYNAMIC)
139	/* An indirect jump is either a sibling call or a jump to a table. */	137	return !is_jump_table_jump(insn);
140	if (insn->type == INSN_JUMP_DYNAMIC)	138	}
141	return !is_jump_table_jump(insn);
142		139
143	/* add_jump_destinations() sets insn->call_dest for sibling calls. */	140	/* add_jump_destinations() sets insn->call_dest for sibling calls. */
144	return (is_static_jump(insn) && insn->call_dest);	141	return (is_static_jump(insn) && insn->call_dest);
@@ -207,7 +204,7 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 62 unmodified lines
207	return false;	204	return false;
208		205
209	insn = find_insn(file, func->sec, func->offset);	206	insn = find_insn(file, func->sec, func->offset);
210	if (!insn->func)	207	if (!insn_func(insn))
211	return false;	208	return false;
212		209
213	func_for_each_insn(file, func, insn) {	210	func_for_each_insn(file, func, insn) {
@@ -243,7 +240,7 @@ static bool __dead_end_function(struct objtool_file file, struct symbol func, · 29 unmodified lines
243	return false;	240	return false;
244	}	241	}
245		242
246	return __dead_end_function(file, dest->func, recursion+1);	243	return __dead_end_function(file, insn_func(dest), recursion+1);
247	}	244	}
248	}	245	}
249		246
@@ -382,6 +379,15 @@ static int decode_instructions(struct objtool_file *file) · 132 unmodified lines
382	!strncmp(sec->name, ".text.__x86.", 12))	379	!strncmp(sec->name, ".text.__x86.", 12))
383	sec->noinstr = true;	380	sec->noinstr = true;
384		381
		382	/*
		383	* .init.text code is ran before userspace and thus doesn't
		384	* strictly need retpolines, except for modules which are
		385	* loaded late, they very much do need retpoline in their
		386	* .init.text
		387	*/
		388	if (!strcmp(sec->name, ".init.text") && !opts.module)
		389	sec->init = true;
		390
385	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {	391	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
386	insn = malloc(sizeof(*insn));	392	insn = malloc(sizeof(*insn));
387	if (!insn) {	393	if (!insn) {
@@ -418,7 +424,10 @@ static int decode_instructions(struct objtool_file *file) · 30 unmodified lines
418	}	424	}
419		425
420	list_for_each_entry(func, &sec->symbol_list, list) {	426	list_for_each_entry(func, &sec->symbol_list, list) {
421	if (func->type != STT_FUNC \|\| func->alias != func)	427	if (func->type != STT_NOTYPE && func->type != STT_FUNC)
		428	continue;
		429
		430	if (func->return_thunk \|\| func->alias != func)
422	continue;	431	continue;
423		432
424	if (!find_insn(file, sec, func->offset)) {	433	if (!find_insn(file, sec, func->offset)) {
@@ -428,9 +437,11 @@ static int decode_instructions(struct objtool_file *file) · 3 unmodified lines
428	}	437	}
429		438
430	sym_for_each_insn(file, func, insn) {	439	sym_for_each_insn(file, func, insn) {
431	insn->func = func;	440	insn->sym = func;
432	if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {	441	if (func->type == STT_FUNC &&
433	if (insn->offset == insn->func->offset) {	442	insn->type == INSN_ENDBR &&
		443	list_empty(&insn->call_node)) {
		444	if (insn->offset == func->offset) {
434	list_add_tail(&insn->call_node, &file->endbr_list);	445	list_add_tail(&insn->call_node, &file->endbr_list);
435	file->nr_endbr++;	446	file->nr_endbr++;
436	} else {	447	} else {
@@ -850,6 +861,68 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) · 413 unmodified lines
850	return 0;	861	return 0;
851	}	862	}
852		863
		864	static int create_cfi_sections(struct objtool_file *file)
		865	{
		866	struct section sec, s;
		867	struct symbol *sym;
		868	unsigned int *loc;
		869	int idx;
		870
		871	sec = find_section_by_name(file->elf, ".cfi_sites");
		872	if (sec) {
		873	INIT_LIST_HEAD(&file->call_list);
		874	WARN("file already has .cfi_sites section, skipping");
		875	return 0;
		876	}
		877
		878	idx = 0;
		879	for_each_sec(file, s) {
		880	if (!s->text)
		881	continue;
		882
		883	list_for_each_entry(sym, &s->symbol_list, list) {
		884	if (sym->type != STT_FUNC)
		885	continue;
		886
		887	if (strncmp(sym->name, "__cfi_", 6))
		888	continue;
		889
		890	idx++;
		891	}
		892	}
		893
		894	sec = elf_create_section(file->elf, ".cfi_sites", 0, sizeof(unsigned int), idx);
		895	if (!sec)
		896	return -1;
		897
		898	idx = 0;
		899	for_each_sec(file, s) {
		900	if (!s->text)
		901	continue;
		902
		903	list_for_each_entry(sym, &s->symbol_list, list) {
		904	if (sym->type != STT_FUNC)
		905	continue;
		906
		907	if (strncmp(sym->name, "__cfi_", 6))
		908	continue;
		909
		910	loc = (unsigned int *)sec->data->d_buf + idx;
		911	memset(loc, 0, sizeof(unsigned int));
		912
		913	if (elf_add_reloc_to_insn(file->elf, sec,
		914	idx * sizeof(unsigned int),
		915	R_X86_64_PC32,
		916	s, sym->offset))
		917	return -1;
		918
		919	idx++;
		920	}
		921	}
		922
		923	return 0;
		924	}
		925
853	static int create_mcount_loc_sections(struct objtool_file *file)	926	static int create_mcount_loc_sections(struct objtool_file *file)
854	{	927	{
855	struct section *sec;	928	struct section *sec;
@@ -893,6 +966,49 @@ static int create_mcount_loc_sections(struct objtool_file *file) · 37 unmodified lines
893	return 0;	966	return 0;
894	}	967	}
895		968
		969	static int create_direct_call_sections(struct objtool_file *file)
		970	{
		971	struct instruction *insn;
		972	struct section *sec;
		973	unsigned int *loc;
		974	int idx;
		975
		976	sec = find_section_by_name(file->elf, ".call_sites");
		977	if (sec) {
		978	INIT_LIST_HEAD(&file->call_list);
		979	WARN("file already has .call_sites section, skipping");
		980	return 0;
		981	}
		982
		983	if (list_empty(&file->call_list))
		984	return 0;
		985
		986	idx = 0;
		987	list_for_each_entry(insn, &file->call_list, call_node)
		988	idx++;
		989
		990	sec = elf_create_section(file->elf, ".call_sites", 0, sizeof(unsigned int), idx);
		991	if (!sec)
		992	return -1;
		993
		994	idx = 0;
		995	list_for_each_entry(insn, &file->call_list, call_node) {
		996
		997	loc = (unsigned int *)sec->data->d_buf + idx;
		998	memset(loc, 0, sizeof(unsigned int));
		999
		1000	if (elf_add_reloc_to_insn(file->elf, sec,
		1001	idx * sizeof(unsigned int),
		1002	R_X86_64_PC32,
		1003	insn->sec, insn->offset))
		1004	return -1;
		1005
		1006	idx++;
		1007	}
		1008
		1009	return 0;
		1010	}
		1011
896	/*	1012	/*
897	* Warnings shouldn't be reported for ignored functions.	1013	* Warnings shouldn't be reported for ignored functions.
898	*/	1014	*/
@@ -1280,6 +1396,9 @@ static void annotate_call_site(struct objtool_file *file, · 381 unmodified lines
1280	return;	1396	return;
1281	}	1397	}
1282		1398
		1399	if (insn->type == INSN_CALL && !insn->sec->init)
		1400	list_add_tail(&insn->call_node, &file->call_list);
		1401
1283	if (!sibling && dead_end_function(file, sym))	1402	if (!sibling && dead_end_function(file, sym))
1284	insn->dead_end = true;	1403	insn->dead_end = true;
1285	}	1404	}
@@ -1350,27 +1469,50 @@ static void add_return_call(struct objtool_file file, struct instruction insn, · 64 unmodified lines
1350	list_add_tail(&insn->call_node, &file->return_thunk_list);	1469	list_add_tail(&insn->call_node, &file->return_thunk_list);
1351	}	1470	}
1352		1471
1353	static bool same_function(struct instruction insn1, struct instruction insn2)	1472	static bool is_first_func_insn(struct objtool_file *file,
		1473	struct instruction insn, struct symbol sym)
1354	{	1474	{
1355	return insn1->func->pfunc == insn2->func->pfunc;	1475	if (insn->offset == sym->offset)
1356	}
1357
1358	static bool is_first_func_insn(struct objtool_file file, struct instruction insn)
1359	{
1360	if (insn->offset == insn->func->offset)
1361	return true;	1476	return true;
1362		1477
		1478	/* Allow direct CALL/JMP past ENDBR */
1363	if (opts.ibt) {	1479	if (opts.ibt) {
1364	struct instruction *prev = prev_insn_same_sym(file, insn);	1480	struct instruction *prev = prev_insn_same_sym(file, insn);
1365		1481
1366	if (prev && prev->type == INSN_ENDBR &&	1482	if (prev && prev->type == INSN_ENDBR &&
1367	insn->offset == insn->func->offset + prev->len)	1483	insn->offset == sym->offset + prev->len)
1368	return true;	1484	return true;
1369	}	1485	}
1370		1486
1371	return false;	1487	return false;
1372	}	1488	}
1373		1489
		1490	/*
		1491	* A sibling call is a tail-call to another symbol -- to differentiate from a
		1492	* recursive tail-call which is to the same symbol.
		1493	*/
		1494	static bool jump_is_sibling_call(struct objtool_file *file,
		1495	struct instruction from, struct instruction to)
		1496	{
		1497	struct symbol *fs = from->sym;
		1498	struct symbol *ts = to->sym;
		1499
		1500	/* Not a sibling call if from/to a symbol hole */
		1501	if (!fs \|\| !ts)
		1502	return false;
		1503
		1504	/* Not a sibling call if not targeting the start of a symbol. */
		1505	if (!is_first_func_insn(file, to, ts))
		1506	return false;
		1507
		1508	/* Disallow sibling calls into STT_NOTYPE */
		1509	if (ts->type == STT_NOTYPE)
		1510	return false;
		1511
		1512	/* Must not be self to be a sibling */
		1513	return fs->pfunc != ts->pfunc;
		1514	}
		1515
1374	/*	1516	/*
1375	* Find the destination instructions for all jumps.	1517	* Find the destination instructions for all jumps.
1376	*/	1518	*/
@@ -1405,7 +1547,7 @@ static int add_jump_destinations(struct objtool_file *file) · 28 unmodified lines
1405	} else if (reloc->sym->return_thunk) {	1547	} else if (reloc->sym->return_thunk) {
1406	add_return_call(file, insn, true);	1548	add_return_call(file, insn, true);
1407	continue;	1549	continue;
1408	} else if (insn->func) {	1550	} else if (insn_func(insn)) {
1409	/*	1551	/*
1410	* External sibling call or internal sibling call with	1552	* External sibling call or internal sibling call with
1411	* STT_FUNC reloc.	1553	* STT_FUNC reloc.
@@ -1447,8 +1589,8 @@ static int add_jump_destinations(struct objtool_file *file) · 35 unmodified lines
1447	/*	1589	/*
1448	* Cross-function jump.	1590	* Cross-function jump.
1449	*/	1591	*/
1450	if (insn->func && jump_dest->func &&	1592	if (insn_func(insn) && insn_func(jump_dest) &&
1451	insn->func != jump_dest->func) {	1593	insn_func(insn) != insn_func(jump_dest)) {
1452		1594
1453	/*	1595	/*
1454	* For GCC 8+, create parent/child links for any cold	1596	* For GCC 8+, create parent/child links for any cold
@@ -1465,22 +1607,22 @@ static int add_jump_destinations(struct objtool_file *file) · 10 unmodified lines
1465	* case where the parent function's only reference to a	1607	* case where the parent function's only reference to a
1466	* subfunction is through a jump table.	1608	* subfunction is through a jump table.
1467	*/	1609	*/
1468	if (!strstr(insn->func->name, ".cold") &&	1610	if (!strstr(insn_func(insn)->name, ".cold") &&
1469	strstr(jump_dest->func->name, ".cold")) {	1611	strstr(insn_func(jump_dest)->name, ".cold")) {
1470	insn->func->cfunc = jump_dest->func;	1612	insn_func(insn)->cfunc = insn_func(jump_dest);
1471	jump_dest->func->pfunc = insn->func;	1613	insn_func(jump_dest)->pfunc = insn_func(insn);
1472
1473	} else if (!same_function(insn, jump_dest) &&
1474	is_first_func_insn(file, jump_dest)) {
1475	/*
1476	* Internal sibling call without reloc or with
1477	* STT_SECTION reloc.
1478	*/
1479	add_call_dest(file, insn, jump_dest->func, true);
1480	continue;
1481	}	1614	}
1482	}	1615	}
1483		1616
		1617	if (jump_is_sibling_call(file, insn, jump_dest)) {
		1618	/*
		1619	* Internal sibling call without reloc or with
		1620	* STT_SECTION reloc.
		1621	*/
		1622	add_call_dest(file, insn, insn_func(jump_dest), true);
		1623	continue;
		1624	}
		1625
1484	insn->jump_dest = jump_dest;	1626	insn->jump_dest = jump_dest;
1485	}	1627	}
1486		1628
@@ -1527,7 +1669,7 @@ static int add_call_destinations(struct objtool_file *file) · 40 unmodified lines
1527	return -1;	1669	return -1;
1528	}	1670	}
1529		1671
1530	if (insn->func && insn->call_dest->type != STT_FUNC) {	1672	if (insn_func(insn) && insn->call_dest->type != STT_FUNC) {
1531	WARN_FUNC("unsupported call to non-function",	1673	WARN_FUNC("unsupported call to non-function",
1532	insn->sec, insn->offset);	1674	insn->sec, insn->offset);
1533	return -1;	1675	return -1;
@@ -1623,7 +1765,7 @@ static int handle_group_alt(struct objtool_file *file, · 89 unmodified lines
1623	nop->offset = special_alt->new_off + special_alt->new_len;	1765	nop->offset = special_alt->new_off + special_alt->new_len;
1624	nop->len = special_alt->orig_len - special_alt->new_len;	1766	nop->len = special_alt->orig_len - special_alt->new_len;
1625	nop->type = INSN_NOP;	1767	nop->type = INSN_NOP;
1626	nop->func = orig_insn->func;	1768	nop->sym = orig_insn->sym;
1627	nop->alt_group = new_alt_group;	1769	nop->alt_group = new_alt_group;
1628	nop->ignore = orig_insn->ignore_alts;	1770	nop->ignore = orig_insn->ignore_alts;
1629	}	1771	}
@@ -1643,7 +1785,7 @@ static int handle_group_alt(struct objtool_file *file, · 13 unmodified lines
1643	last_new_insn = insn;	1785	last_new_insn = insn;
1644		1786
1645	insn->ignore = orig_insn->ignore_alts;	1787	insn->ignore = orig_insn->ignore_alts;
1646	insn->func = orig_insn->func;	1788	insn->sym = orig_insn->sym;
1647	insn->alt_group = new_alt_group;	1789	insn->alt_group = new_alt_group;
1648		1790
1649	/*	1791	/*
@@ -1655,7 +1797,7 @@ static int handle_group_alt(struct objtool_file *file, · 5 unmodified lines
1655	* accordingly.	1797	* accordingly.
1656	*/	1798	*/
1657	alt_reloc = insn_reloc(file, insn);	1799	alt_reloc = insn_reloc(file, insn);
1658	if (alt_reloc &&	1800	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1659	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {	1801	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1660		1802
1661	WARN_FUNC("unsupported relocation in alternatives section",	1803	WARN_FUNC("unsupported relocation in alternatives section",
@@ -1837,7 +1979,7 @@ static int add_jump_table(struct objtool_file file, struct instruction insn, · 175 unmodified lines
1837	struct reloc *reloc = table;	1979	struct reloc *reloc = table;
1838	struct instruction *dest_insn;	1980	struct instruction *dest_insn;
1839	struct alternative *alt;	1981	struct alternative *alt;
1840	struct symbol *pfunc = insn->func->pfunc;	1982	struct symbol *pfunc = insn_func(insn)->pfunc;
1841	unsigned int prev_offset = 0;	1983	unsigned int prev_offset = 0;
1842		1984
1843	/*	1985	/*
@@ -1864,7 +2006,7 @@ static int add_jump_table(struct objtool_file file, struct instruction insn, · 20 unmodified lines
1864	break;	2006	break;
1865		2007
1866	/* Make sure the destination is in the same function: */	2008	/* Make sure the destination is in the same function: */
1867	if (!dest_insn->func \|\| dest_insn->func->pfunc != pfunc)	2009	if (!insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != pfunc)
1868	break;	2010	break;
1869		2011
1870	alt = malloc(sizeof(*alt));	2012	alt = malloc(sizeof(*alt));
@@ -1904,7 +2046,7 @@ static struct reloc find_jump_table(struct objtool_file file, · 33 unmodified lines
1904	* it.	2046	* it.
1905	*/	2047	*/
1906	for (;	2048	for (;
1907	insn && insn->func && insn->func->pfunc == func;	2049	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
1908	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {	2050	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1909		2051
1910	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)	2052	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
@@ -1921,7 +2063,7 @@ static struct reloc find_jump_table(struct objtool_file file, · 10 unmodified lines
1921	if (!table_reloc)	2063	if (!table_reloc)
1922	continue;	2064	continue;
1923	dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);	2065	dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1924	if (!dest_insn \|\| !dest_insn->func \|\| dest_insn->func->pfunc != func)	2066	if (!dest_insn \|\| !insn_func(dest_insn) \|\| insn_func(dest_insn)->pfunc != func)
1925	continue;	2067	continue;
1926		2068
1927	return table_reloc;	2069	return table_reloc;
@@ -2370,6 +2512,13 @@ static int decode_sections(struct objtool_file *file) · 442 unmodified lines
2370	if (ret)	2512	if (ret)
2371	return ret;	2513	return ret;
2372		2514
		2515	/*
		2516	* Must be before add_{jump_call}_destination.
		2517	*/
		2518	ret = classify_symbols(file);
		2519	if (ret)
		2520	return ret;
		2521
2373	ret = decode_instructions(file);	2522	ret = decode_instructions(file);
2374	if (ret)	2523	if (ret)
2375	return ret;	2524	return ret;
@@ -2388,13 +2537,6 @@ static int decode_sections(struct objtool_file *file) · 12 unmodified lines
2388	if (ret)	2537	if (ret)
2389	return ret;	2538	return ret;
2390		2539
2391	/*
2392	* Must be before add_{jump_call}_destination.
2393	*/
2394	ret = classify_symbols(file);
2395	if (ret)
2396	return ret;
2397
2398	/*	2540	/*
2399	* Must be before add_jump_destinations(), which depends on 'func'	2541	* Must be before add_jump_destinations(), which depends on 'func'
2400	* being set for alternatives, to enable proper sibling call detection.	2542	* being set for alternatives, to enable proper sibling call detection.
@@ -2603,7 +2745,7 @@ static int update_cfi_state(struct instruction *insn, · 202 unmodified lines
2603		2745
2604	/* stack operations don't make sense with an undefined CFA */	2746	/* stack operations don't make sense with an undefined CFA */
2605	if (cfa->base == CFI_UNDEFINED) {	2747	if (cfa->base == CFI_UNDEFINED) {
2606	if (insn->func) {	2748	if (insn_func(insn)) {
2607	WARN_FUNC("undefined stack state", insn->sec, insn->offset);	2749	WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2608	return -1;	2750	return -1;
2609	}	2751	}
@@ -2949,7 +3091,7 @@ static int update_cfi_state(struct instruction *insn, · 339 unmodified lines
2949	}	3091	}
2950		3092
2951	/* detect when asm code uses rbp as a scratch register */	3093	/* detect when asm code uses rbp as a scratch register */
2952	if (opts.stackval && insn->func && op->src.reg == CFI_BP &&	3094	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
2953	cfa->base != CFI_BP)	3095	cfa->base != CFI_BP)
2954	cfi->bp_scratch = true;	3096	cfi->bp_scratch = true;
2955	break;	3097	break;
@@ -3259,7 +3401,7 @@ static int validate_sibling_call(struct objtool_file *file, · 303 unmodified lines
3259	struct instruction *insn,	3401	struct instruction *insn,
3260	struct insn_state *state)	3402	struct insn_state *state)
3261	{	3403	{
3262	if (has_modified_stack_frame(insn, state)) {	3404	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3263	WARN_FUNC("sibling call from callable instruction with modified stack frame",	3405	WARN_FUNC("sibling call from callable instruction with modified stack frame",
3264	insn->sec, insn->offset);	3406	insn->sec, insn->offset);
3265	return 1;	3407	return 1;
@@ -3345,13 +3487,14 @@ static int validate_branch(struct objtool_file file, struct symbol func, · 79 unmodified lines
3345	while (1) {	3487	while (1) {
3346	next_insn = next_insn_to_validate(file, insn);	3488	next_insn = next_insn_to_validate(file, insn);
3347		3489
3348	if (func && insn->func && func != insn->func->pfunc) {	3490	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3349	/* Ignore KCFI type preambles, which always fall through */	3491	/* Ignore KCFI type preambles, which always fall through */
3350	if (!strncmp(func->name, "__cfi_", 6))	3492	if (!strncmp(func->name, "__cfi_", 6) \|\|
		3493	!strncmp(func->name, "__pfx_", 6))
3351	return 0;	3494	return 0;
3352		3495
3353	WARN("%s() falls through to next function %s()",	3496	WARN("%s() falls through to next function %s()",
3354	func->name, insn->func->name);	3497	func->name, insn_func(insn)->name);
3355	return 1;	3498	return 1;
3356	}	3499	}
3357		3500
@@ -3593,7 +3736,7 @@ static int validate_unwind_hints(struct objtool_file file, struct section sec) · 235 unmodified lines
3593		3736
3594	while (&insn->list != &file->insn_list && (!sec \|\| insn->sec == sec)) {	3737	while (&insn->list != &file->insn_list && (!sec \|\| insn->sec == sec)) {
3595	if (insn->hint && !insn->visited && !insn->ignore) {	3738	if (insn->hint && !insn->visited && !insn->ignore) {
3596	ret = validate_branch(file, insn->func, insn, state);	3739	ret = validate_branch(file, insn_func(insn), insn, state);
3597	if (ret && opts.backtrace)	3740	if (ret && opts.backtrace)
3598	BT_FUNC("<=== (hint)", insn);	3741	BT_FUNC("<=== (hint)", insn);
3599	warnings += ret;	3742	warnings += ret;
@@ -3758,13 +3901,7 @@ static int validate_retpoline(struct objtool_file *file) · 158 unmodified lines
3758	if (insn->retpoline_safe)	3901	if (insn->retpoline_safe)
3759	continue;	3902	continue;
3760		3903
3761	/*	3904	if (insn->sec->init)
3762	* .init.text code is ran before userspace and thus doesn't
3763	* strictly need retpolines, except for modules which are
3764	* loaded late, they very much do need retpoline in their
3765	* .init.text
3766	*/
3767	if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
3768	continue;	3905	continue;
3769		3906
3770	if (insn->type == INSN_RETURN) {	3907	if (insn->type == INSN_RETURN) {
@@ -3822,7 +3959,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 51 unmodified lines
3822	* In this case we'll find a piece of code (whole function) that is not	3959	* In this case we'll find a piece of code (whole function) that is not
3823	* covered by a !section symbol. Ignore them.	3960	* covered by a !section symbol. Ignore them.
3824	*/	3961	*/
3825	if (opts.link && !insn->func) {	3962	if (opts.link && !insn_func(insn)) {
3826	int size = find_symbol_hole_containing(insn->sec, insn->offset);	3963	int size = find_symbol_hole_containing(insn->sec, insn->offset);
3827	unsigned long end = insn->offset + size;	3964	unsigned long end = insn->offset + size;
3828		3965
@@ -3846,10 +3983,10 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 17 unmodified lines
3846	/*	3983	/*
3847	* If this hole jumps to a .cold function, mark it ignore too.	3984	* If this hole jumps to a .cold function, mark it ignore too.
3848	*/	3985	*/
3849	if (insn->jump_dest && insn->jump_dest->func &&	3986	if (insn->jump_dest && insn_func(insn->jump_dest) &&
3850	strstr(insn->jump_dest->func->name, ".cold")) {	3987	strstr(insn_func(insn->jump_dest)->name, ".cold")) {
3851	struct instruction *dest = insn->jump_dest;	3988	struct instruction *dest = insn->jump_dest;
3852	func_for_each_insn(file, dest->func, dest)	3989	func_for_each_insn(file, insn_func(dest), dest)
3853	dest->ignore = true;	3990	dest->ignore = true;
3854	}	3991	}
3855	}	3992	}
@@ -3857,10 +3994,10 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 1 unmodified line
3857	return false;	3994	return false;
3858	}	3995	}
3859		3996
3860	if (!insn->func)	3997	if (!insn_func(insn))
3861	return false;	3998	return false;
3862		3999
3863	if (insn->func->static_call_tramp)	4000	if (insn_func(insn)->static_call_tramp)
3864	return true;	4001	return true;
3865		4002
3866	/*	4003	/*
@@ -3891,7 +4028,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 24 unmodified lines
3891		4028
3892	if (insn->type == INSN_JUMP_UNCONDITIONAL) {	4029	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3893	if (insn->jump_dest &&	4030	if (insn->jump_dest &&
3894	insn->jump_dest->func == insn->func) {	4031	insn_func(insn->jump_dest) == insn_func(insn)) {
3895	insn = insn->jump_dest;	4032	insn = insn->jump_dest;
3896	continue;	4033	continue;
3897	}	4034	}
@@ -3899,7 +4036,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 1 unmodified line
3899	break;	4036	break;
3900	}	4037	}
3901		4038
3902	if (insn->offset + insn->len >= insn->func->offset + insn->func->len)	4039	if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
3903	break;	4040	break;
3904		4041
3905	insn = list_next_entry(insn, list);	4042	insn = list_next_entry(insn, list);
@@ -3908,6 +4045,54 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio · 2 unmodified lines
3908	return false;	4045	return false;
3909	}	4046	}
3910		4047
		4048	static int add_prefix_symbol(struct objtool_file file, struct symbol func,
		4049	struct instruction *insn)
		4050	{
		4051	if (!opts.prefix)
		4052	return 0;
		4053
		4054	for (;;) {
		4055	struct instruction *prev = list_prev_entry(insn, list);
		4056	u64 offset;
		4057
		4058	if (&prev->list == &file->insn_list)
		4059	break;
		4060
		4061	if (prev->type != INSN_NOP)
		4062	break;
		4063
		4064	offset = func->offset - prev->offset;
		4065	if (offset >= opts.prefix) {
		4066	if (offset == opts.prefix) {
		4067	/*
		4068	* Since the sec->symbol_list is ordered by
		4069	* offset (see elf_add_symbol()) the added
		4070	* symbol will not be seen by the iteration in
		4071	* validate_section().
		4072	*
		4073	* Hence the lack of list_for_each_entry_safe()
		4074	* there.
		4075	*
		4076	* The direct concequence is that prefix symbols
		4077	* don't get visited (because pointless), except
		4078	* for the logic in ignore_unreachable_insn()
		4079	* that needs the terminating insn to be visited
		4080	* otherwise it will report the hole.
		4081	*
		4082	* Hence mark the first instruction of the
		4083	* prefix symbol as visisted.
		4084	*/
		4085	prev->visited \|= VISITED_BRANCH;
		4086	elf_create_prefix_symbol(file->elf, func, opts.prefix);
		4087	}
		4088	break;
		4089	}
		4090	insn = prev;
		4091	}
		4092
		4093	return 0;
		4094	}
		4095
3911	static int validate_symbol(struct objtool_file file, struct section sec,	4096	static int validate_symbol(struct objtool_file file, struct section sec,
3912	struct symbol sym, struct insn_state state)	4097	struct symbol sym, struct insn_state state)
3913	{	4098	{
@@ -3926,9 +4111,11 @@ static int validate_symbol(struct objtool_file file, struct section sec, · 12 unmodified lines
3926	if (!insn \|\| insn->ignore \|\| insn->visited)	4111	if (!insn \|\| insn->ignore \|\| insn->visited)
3927	return 0;	4112	return 0;
3928		4113
		4114	add_prefix_symbol(file, sym, insn);
		4115
3929	state->uaccess = sym->uaccess_safe;	4116	state->uaccess = sym->uaccess_safe;
3930		4117
3931	ret = validate_branch(file, insn->func, insn, *state);	4118	ret = validate_branch(file, insn_func(insn), insn, *state);
3932	if (ret && opts.backtrace)	4119	if (ret && opts.backtrace)
3933	BT_FUNC("<=== (sym)", insn);	4120	BT_FUNC("<=== (sym)", insn);
3934	return ret;	4121	return ret;
@@ -3994,6 +4181,24 @@ static void mark_endbr_used(struct instruction *insn) · 59 unmodified lines
3994	list_del_init(&insn->call_node);	4181	list_del_init(&insn->call_node);
3995	}	4182	}
3996		4183
		4184	static bool noendbr_range(struct objtool_file file, struct instruction insn)
		4185	{
		4186	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
		4187	struct instruction *first;
		4188
		4189	if (!sym)
		4190	return false;
		4191
		4192	first = find_insn(file, sym->sec, sym->offset);
		4193	if (!first)
		4194	return false;
		4195
		4196	if (first->type != INSN_ENDBR && !first->noendbr)
		4197	return false;
		4198
		4199	return insn->offset == sym->offset + sym->len;
		4200	}
		4201
3997	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)	4202	static int validate_ibt_insn(struct objtool_file file, struct instruction insn)
3998	{	4203	{
3999	struct instruction *dest;	4204	struct instruction *dest;
@@ -4047,7 +4252,7 @@ static int validate_ibt_insn(struct objtool_file file, struct instruction insn · 47 unmodified lines
4047	continue;	4252	continue;
4048	}	4253	}
4049		4254
4050	if (dest->func && dest->func == insn->func) {	4255	if (insn_func(dest) && insn_func(dest) == insn_func(insn)) {
4051	/*	4256	/*
4052	* Anything from->to self is either _THIS_IP_ or	4257	* Anything from->to self is either _THIS_IP_ or
4053	* IRET-to-self.	4258	* IRET-to-self.
@@ -4066,9 +4271,19 @@ static int validate_ibt_insn(struct objtool_file file, struct instruction insn · 12 unmodified lines
4066	continue;	4271	continue;
4067	}	4272	}
4068		4273
		4274	/*
		4275	* Accept anything ANNOTATE_NOENDBR.
		4276	*/
4069	if (dest->noendbr)	4277	if (dest->noendbr)
4070	continue;	4278	continue;
4071		4279
		4280	/*
		4281	* Accept if this is the instruction after a symbol
		4282	* that is (no)endbr -- typical code-range usage.
		4283	*/
		4284	if (noendbr_range(file, dest))
		4285	continue;
		4286
4072	WARN_FUNC("relocation to !ENDBR: %s",	4287	WARN_FUNC("relocation to !ENDBR: %s",
4073	insn->sec, insn->offset,	4288	insn->sec, insn->offset,
4074	offstr(dest->sec, dest->offset));	4289	offstr(dest->sec, dest->offset));
@@ -4307,11 +4522,25 @@ int check(struct objtool_file *file) · 232 unmodified lines
4307	warnings += ret;	4522	warnings += ret;
4308	}	4523	}
4309		4524
		4525	if (opts.cfi) {
		4526	ret = create_cfi_sections(file);
		4527	if (ret < 0)
		4528	goto out;
		4529	warnings += ret;
		4530	}
		4531
4310	if (opts.rethunk) {	4532	if (opts.rethunk) {
4311	ret = create_return_sites_sections(file);	4533	ret = create_return_sites_sections(file);
4312	if (ret < 0)	4534	if (ret < 0)
4313	goto out;	4535	goto out;
4314	warnings += ret;	4536	warnings += ret;
		4537
		4538	if (opts.hack_skylake) {
		4539	ret = create_direct_call_sections(file);
		4540	if (ret < 0)
		4541	goto out;
		4542	warnings += ret;
		4543	}
4315	}	4544	}
4316		4545
4317	if (opts.mcount) {	4546	if (opts.mcount) {

tools/objtool/elf.c+199−97

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	#include <string.h>	16	#include <string.h>
17	#include <unistd.h>	17	#include <unistd.h>
18	#include <errno.h>	18	#include <errno.h>
		19	#include <linux/interval_tree_generic.h>
19	#include <objtool/builtin.h>	20	#include <objtool/builtin.h>
20		21
21	#include <objtool/elf.h>	22	#include <objtool/elf.h>
@@ -50,38 +51,22 @@ static inline u32 str_hash(const char *str) · 28 unmodified lines
50	__elf_table(name); \	51	__elf_table(name); \
51	})	52	})
52		53
53	static bool symbol_to_offset(struct rb_node a, const struct rb_node b)	54	static inline unsigned long __sym_start(struct symbol *s)
54	{	55	{
55	struct symbol *sa = rb_entry(a, struct symbol, node);	56	return s->offset;
56	struct symbol *sb = rb_entry(b, struct symbol, node);
57
58	if (sa->offset < sb->offset)
59	return true;
60	if (sa->offset > sb->offset)
61	return false;
62
63	if (sa->len < sb->len)
64	return true;
65	if (sa->len > sb->len)
66	return false;
67
68	sa->alias = sb;
69
70	return false;
71	}	57	}
72		58
73	static int symbol_by_offset(const void key, const struct rb_node node)	59	static inline unsigned long __sym_last(struct symbol *s)
74	{	60	{
75	const struct symbol *s = rb_entry(node, struct symbol, node);	61	return s->offset + s->len - 1;
76	const unsigned long *o = key;	62	}
77		63
78	if (*o < s->offset)	64	INTERVAL_TREE_DEFINE(struct symbol, node, unsigned long, __subtree_last,
79	return -1;	65	__sym_start, __sym_last, static, __sym)
80	if (*o >= s->offset + s->len)
81	return 1;
82		66
83	return 0;	67	#define __sym_for_each(_iter, _tree, _start, _end) \
84	}	68	for (_iter = __sym_iter_first((_tree), (_start), (_end)); \
		69	_iter; _iter = __sym_iter_next(_iter, (_start), (_end)))
85		70
86	struct symbol_hole {	71	struct symbol_hole {
87	unsigned long key;	72	unsigned long key;
@@ -147,13 +132,12 @@ static struct symbol find_symbol_by_index(struct elf elf, unsigned int idx) · 59 unmodified lines
147		132
148	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset)	133	struct symbol find_symbol_by_offset(struct section sec, unsigned long offset)
149	{	134	{
150	struct rb_node *node;	135	struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
151		136	struct symbol *iter;
152	rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
153	struct symbol *s = rb_entry(node, struct symbol, node);
154		137
155	if (s->offset == offset && s->type != STT_SECTION)	138	__sym_for_each(iter, tree, offset, offset) {
156	return s;	139	if (iter->offset == offset && iter->type != STT_SECTION)
		140	return iter;
157	}	141	}
158		142
159	return NULL;	143	return NULL;
@@ -161,13 +145,12 @@ struct symbol find_symbol_by_offset(struct section sec, unsigned long offset) · 1 unmodified line
161		145
162	struct symbol find_func_by_offset(struct section sec, unsigned long offset)	146	struct symbol find_func_by_offset(struct section sec, unsigned long offset)
163	{	147	{
164	struct rb_node *node;	148	struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
		149	struct symbol *iter;
165		150
166	rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {	151	__sym_for_each(iter, tree, offset, offset) {
167	struct symbol *s = rb_entry(node, struct symbol, node);	152	if (iter->offset == offset && iter->type == STT_FUNC)
168		153	return iter;
169	if (s->offset == offset && s->type == STT_FUNC)
170	return s;
171	}	154	}
172		155
173	return NULL;	156	return NULL;
@@ -175,13 +158,12 @@ struct symbol find_func_by_offset(struct section sec, unsigned long offset) · 1 unmodified line
175		158
176	struct symbol find_symbol_containing(const struct section sec, unsigned long offset)	159	struct symbol find_symbol_containing(const struct section sec, unsigned long offset)
177	{	160	{
178	struct rb_node *node;	161	struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
179		162	struct symbol *iter;
180	rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
181	struct symbol *s = rb_entry(node, struct symbol, node);
182		163
183	if (s->type != STT_SECTION)	164	__sym_for_each(iter, tree, offset, offset) {
184	return s;	165	if (iter->type != STT_SECTION)
		166	return iter;
185	}	167	}
186		168
187	return NULL;	169	return NULL;
@@ -202,7 +184,7 @@ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) · 14 unmodified lines
202	/*	184	/*
203	* Find the rightmost symbol for which @offset is after it.	185	* Find the rightmost symbol for which @offset is after it.
204	*/	186	*/
205	n = rb_find(&hole, &sec->symbol_tree, symbol_hole_by_offset);	187	n = rb_find(&hole, &sec->symbol_tree.rb_root, symbol_hole_by_offset);
206		188
207	/* found a symbol that contains @offset */	189	/* found a symbol that contains @offset */
208	if (n)	190	if (n)
@@ -224,13 +206,12 @@ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) · 15 unmodified lines
224		206
225	struct symbol find_func_containing(struct section sec, unsigned long offset)	207	struct symbol find_func_containing(struct section sec, unsigned long offset)
226	{	208	{
227	struct rb_node *node;	209	struct rb_root_cached tree = (struct rb_root_cached )&sec->symbol_tree;
228		210	struct symbol *iter;
229	rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) {
230	struct symbol *s = rb_entry(node, struct symbol, node);
231		211
232	if (s->type == STT_FUNC)	212	__sym_for_each(iter, tree, offset, offset) {
233	return s;	213	if (iter->type == STT_FUNC)
		214	return iter;
234	}	215	}
235		216
236	return NULL;	217	return NULL;
@@ -373,7 +354,9 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 136 unmodified lines
373	{	354	{
374	struct list_head *entry;	355	struct list_head *entry;
375	struct rb_node *pnode;	356	struct rb_node *pnode;
		357	struct symbol *iter;
376		358
		359	INIT_LIST_HEAD(&sym->reloc_list);
377	INIT_LIST_HEAD(&sym->pv_target);	360	INIT_LIST_HEAD(&sym->pv_target);
378	sym->alias = sym;	361	sym->alias = sym;
379		362
@@ -386,7 +369,12 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 6 unmodified lines
386	sym->offset = sym->sym.st_value;	369	sym->offset = sym->sym.st_value;
387	sym->len = sym->sym.st_size;	370	sym->len = sym->sym.st_size;
388		371
389	rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset);	372	__sym_for_each(iter, &sym->sec->symbol_tree, sym->offset, sym->offset) {
		373	if (iter->offset == sym->offset && iter->type == sym->type)
		374	iter->alias = sym;
		375	}
		376
		377	__sym_insert(sym, &sym->sec->symbol_tree);
390	pnode = rb_prev(&sym->node);	378	pnode = rb_prev(&sym->node);
391	if (pnode)	379	if (pnode)
392	entry = &rb_entry(pnode, struct symbol, node)->list;	380	entry = &rb_entry(pnode, struct symbol, node)->list;
@@ -401,7 +389,7 @@ static void elf_add_symbol(struct elf elf, struct symbol sym) · 8 unmodified lines
401	* can exist within a function, confusing the sorting.	389	* can exist within a function, confusing the sorting.
402	*/	390	*/
403	if (!sym->len)	391	if (!sym->len)
404	rb_erase(&sym->node, &sym->sec->symbol_tree);	392	__sym_remove(sym, &sym->sec->symbol_tree);
405	}	393	}
406		394
407	static int read_symbols(struct elf *elf)	395	static int read_symbols(struct elf *elf)
@@ -570,6 +558,7 @@ int elf_add_reloc(struct elf elf, struct section sec, unsigned long offset, · 162 unmodified lines
570	reloc->sym = sym;	558	reloc->sym = sym;
571	reloc->addend = addend;	559	reloc->addend = addend;
572		560
		561	list_add_tail(&reloc->sym_reloc_entry, &sym->reloc_list);
573	list_add_tail(&reloc->list, &sec->reloc->reloc_list);	562	list_add_tail(&reloc->list, &sec->reloc->reloc_list);
574	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));	563	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
575		564
@@ -586,21 +575,10 @@ int elf_add_reloc(struct elf elf, struct section sec, unsigned long offset, · 10 unmodified lines
586	*/	575	*/
587	static void elf_dirty_reloc_sym(struct elf elf, struct symbol sym)	576	static void elf_dirty_reloc_sym(struct elf elf, struct symbol sym)
588	{	577	{
589	struct section *sec;	578	struct reloc *reloc;
590
591	list_for_each_entry(sec, &elf->sections, list) {
592	struct reloc *reloc;
593
594	if (sec->changed)
595	continue;
596		579
597	list_for_each_entry(reloc, &sec->reloc_list, list) {	580	list_for_each_entry(reloc, &sym->reloc_list, sym_reloc_entry)
598	if (reloc->sym == sym) {	581	reloc->sec->changed = true;
599	sec->changed = true;
600	break;
601	}
602	}
603	}
604	}	582	}
605		583
606	/*	584	/*
@@ -647,6 +625,12 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 40 unmodified lines
647		625
648	/* end-of-list */	626	/* end-of-list */
649	if (!symtab_data) {	627	if (!symtab_data) {
		628	/*
		629	* Over-allocate to avoid O(n^2) symbol creation
		630	* behaviour. The down side is that libelf doesn't
		631	* like this; see elf_truncate_section() for the fixup.
		632	*/
		633	int num = max(1U, sym->idx/3);
650	void *buf;	634	void *buf;
651		635
652	if (idx) {	636	if (idx) {
@@ -660,28 +644,34 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 7 unmodified lines
660	if (t)	644	if (t)
661	shndx_data = elf_newdata(t);	645	shndx_data = elf_newdata(t);
662		646
663	buf = calloc(1, entsize);	647	buf = calloc(num, entsize);
664	if (!buf) {	648	if (!buf) {
665	WARN("malloc");	649	WARN("malloc");
666	return -1;	650	return -1;
667	}	651	}
668		652
669	symtab_data->d_buf = buf;	653	symtab_data->d_buf = buf;
670	symtab_data->d_size = entsize;	654	symtab_data->d_size = num * entsize;
671	symtab_data->d_align = 1;	655	symtab_data->d_align = 1;
672	symtab_data->d_type = ELF_T_SYM;	656	symtab_data->d_type = ELF_T_SYM;
673		657
674	symtab->sh.sh_size += entsize;
675	symtab->changed = true;	658	symtab->changed = true;
		659	symtab->truncate = true;
676		660
677	if (t) {	661	if (t) {
678	shndx_data->d_buf = &sym->sec->idx;	662	buf = calloc(num, sizeof(Elf32_Word));
679	shndx_data->d_size = sizeof(Elf32_Word);	663	if (!buf) {
		664	WARN("malloc");
		665	return -1;
		666	}
		667
		668	shndx_data->d_buf = buf;
		669	shndx_data->d_size = num * sizeof(Elf32_Word);
680	shndx_data->d_align = sizeof(Elf32_Word);	670	shndx_data->d_align = sizeof(Elf32_Word);
681	shndx_data->d_type = ELF_T_WORD;	671	shndx_data->d_type = ELF_T_WORD;
682		672
683	symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
684	symtab_shndx->changed = true;	673	symtab_shndx->changed = true;
		674	symtab_shndx->truncate = true;
685	}	675	}
686		676
687	break;	677	break;
@@ -730,11 +720,11 @@ static int elf_update_symbol(struct elf elf, struct section symtab, · 42 unmodified lines
730	}	720	}
731		721
732	static struct symbol *	722	static struct symbol *
733	elf_create_section_symbol(struct elf elf, struct section sec)	723	__elf_create_symbol(struct elf elf, struct symbol sym)
734	{	724	{
735	struct section symtab, symtab_shndx;	725	struct section symtab, symtab_shndx;
736	Elf32_Word first_non_local, new_idx;	726	Elf32_Word first_non_local, new_idx;
737	struct symbol sym, old;	727	struct symbol *old;
738		728
739	symtab = find_section_by_name(elf, ".symtab");	729	symtab = find_section_by_name(elf, ".symtab");
740	if (symtab) {	730	if (symtab) {
@@ -744,27 +734,16 @@ elf_create_section_symbol(struct elf elf, struct section sec) · 3 unmodified lines
744	return NULL;	734	return NULL;
745	}	735	}
746		736
747	sym = calloc(1, sizeof(*sym));	737	new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
748	if (!sym) {
749	perror("malloc");
750	return NULL;
751	}
752
753	sym->name = sec->name;
754	sym->sec = sec;
755		738
756	// st_name 0	739	if (GELF_ST_BIND(sym->sym.st_info) != STB_LOCAL)
757	sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);	740	goto non_local;
758	// st_other 0
759	// st_value 0
760	// st_size 0
761		741
762	/*	742	/*
763	* Move the first global symbol, as per sh_info, into a new, higher	743	* Move the first global symbol, as per sh_info, into a new, higher
764	* symbol index. This fees up a spot for a new local symbol.	744	* symbol index. This fees up a spot for a new local symbol.
765	*/	745	*/
766	first_non_local = symtab->sh.sh_info;	746	first_non_local = symtab->sh.sh_info;
767	new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize;
768	old = find_symbol_by_index(elf, first_non_local);	747	old = find_symbol_by_index(elf, first_non_local);
769	if (old) {	748	if (old) {
770	old->idx = new_idx;	749	old->idx = new_idx;
@@ -782,18 +761,82 @@ elf_create_section_symbol(struct elf elf, struct section sec) · 11 unmodified lines
782	new_idx = first_non_local;	761	new_idx = first_non_local;
783	}	762	}
784		763
		764	/*
		765	* Either way, we will add a LOCAL symbol.
		766	*/
		767	symtab->sh.sh_info += 1;
		768
		769	non_local:
785	sym->idx = new_idx;	770	sym->idx = new_idx;
786	if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) {	771	if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) {
787	WARN("elf_update_symbol");	772	WARN("elf_update_symbol");
788	return NULL;	773	return NULL;
789	}	774	}
790		775
791	/*	776	symtab->sh.sh_size += symtab->sh.sh_entsize;
792	* Either way, we added a LOCAL symbol.	777	symtab->changed = true;
793	*/
794	symtab->sh.sh_info += 1;
795		778
796	elf_add_symbol(elf, sym);	779	if (symtab_shndx) {
		780	symtab_shndx->sh.sh_size += sizeof(Elf32_Word);
		781	symtab_shndx->changed = true;
		782	}
		783
		784	return sym;
		785	}
		786
		787	static struct symbol *
		788	elf_create_section_symbol(struct elf elf, struct section sec)
		789	{
		790	struct symbol sym = calloc(1, sizeof(sym));
		791
		792	if (!sym) {
		793	perror("malloc");
		794	return NULL;
		795	}
		796
		797	sym->name = sec->name;
		798	sym->sec = sec;
		799
		800	// st_name 0
		801	sym->sym.st_info = GELF_ST_INFO(STB_LOCAL, STT_SECTION);
		802	// st_other 0
		803	// st_value 0
		804	// st_size 0
		805
		806	sym = __elf_create_symbol(elf, sym);
		807	if (sym)
		808	elf_add_symbol(elf, sym);
		809
		810	return sym;
		811	}
		812
		813	static int elf_add_string(struct elf elf, struct section strtab, char *str);
		814
		815	struct symbol *
		816	elf_create_prefix_symbol(struct elf elf, struct symbol orig, long size)
		817	{
		818	struct symbol sym = calloc(1, sizeof(sym));
		819	size_t namelen = strlen(orig->name) + sizeof("__pfx_");
		820	char *name = malloc(namelen);
		821
		822	if (!sym \|\| !name) {
		823	perror("malloc");
		824	return NULL;
		825	}
		826
		827	snprintf(name, namelen, "__pfx_%s", orig->name);
		828
		829	sym->name = name;
		830	sym->sec = orig->sec;
		831
		832	sym->sym.st_name = elf_add_string(elf, NULL, name);
		833	sym->sym.st_info = orig->sym.st_info;
		834	sym->sym.st_value = orig->sym.st_value - size;
		835	sym->sym.st_size = size;
		836
		837	sym = __elf_create_symbol(elf, sym);
		838	if (sym)
		839	elf_add_symbol(elf, sym);
797		840
798	return sym;	841	return sym;
799	}	842	}
@@ -850,11 +893,12 @@ static int read_rela_reloc(struct section sec, int i, struct reloc reloc, unsi · 50 unmodified lines
850		893
851	static int read_relocs(struct elf *elf)	894	static int read_relocs(struct elf *elf)
852	{	895	{
		896	unsigned long nr_reloc, max_reloc = 0, tot_reloc = 0;
853	struct section *sec;	897	struct section *sec;
854	struct reloc *reloc;	898	struct reloc *reloc;
855	int i;
856	unsigned int symndx;	899	unsigned int symndx;
857	unsigned long nr_reloc, max_reloc = 0, tot_reloc = 0;	900	struct symbol *sym;
		901	int i;
858		902
859	if (!elf_alloc_hash(reloc, elf->text_size / 16))	903	if (!elf_alloc_hash(reloc, elf->text_size / 16))
860	return -1;	904	return -1;
@@ -895,13 +939,14 @@ static int read_relocs(struct elf *elf) · 34 unmodified lines
895		939
896	reloc->sec = sec;	940	reloc->sec = sec;
897	reloc->idx = i;	941	reloc->idx = i;
898	reloc->sym = find_symbol_by_index(elf, symndx);	942	reloc->sym = sym = find_symbol_by_index(elf, symndx);
899	if (!reloc->sym) {	943	if (!reloc->sym) {
900	WARN("can't find reloc entry symbol %d for %s",	944	WARN("can't find reloc entry symbol %d for %s",
901	symndx, sec->name);	945	symndx, sec->name);
902	return -1;	946	return -1;
903	}	947	}
904		948
		949	list_add_tail(&reloc->sym_reloc_entry, &sym->reloc_list);
905	list_add_tail(&reloc->list, &sec->reloc_list);	950	list_add_tail(&reloc->list, &sec->reloc_list);
906	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));	951	elf_hash_add(reloc, &reloc->hash, reloc_hash(reloc));
907		952
@@ -1285,6 +1330,60 @@ int elf_write_reloc(struct elf elf, struct reloc reloc) · 377 unmodified lines
1285	return 0;	1330	return 0;
1286	}	1331	}
1287		1332
		1333	/*
		1334	* When Elf_Scn::sh_size is smaller than the combined Elf_Data::d_size
		1335	* do you:
		1336	*
		1337	* A) adhere to the section header and truncate the data, or
		1338	* B) ignore the section header and write out all the data you've got?
		1339	*
		1340	* Yes, libelf sucks and we need to manually truncate if we over-allocate data.
		1341	*/
		1342	static int elf_truncate_section(struct elf elf, struct section sec)
		1343	{
		1344	u64 size = sec->sh.sh_size;
		1345	bool truncated = false;
		1346	Elf_Data *data = NULL;
		1347	Elf_Scn *s;
		1348
		1349	s = elf_getscn(elf->elf, sec->idx);
		1350	if (!s) {
		1351	WARN_ELF("elf_getscn");
		1352	return -1;
		1353	}
		1354
		1355	for (;;) {
		1356	/* get next data descriptor for the relevant section */
		1357	data = elf_getdata(s, data);
		1358
		1359	if (!data) {
		1360	if (size) {
		1361	WARN("end of section data but non-zero size left\n");
		1362	return -1;
		1363	}
		1364	return 0;
		1365	}
		1366
		1367	if (truncated) {
		1368	/* when we remove symbols */
		1369	WARN("truncated; but more data\n");
		1370	return -1;
		1371	}
		1372
		1373	if (!data->d_size) {
		1374	WARN("zero size data");
		1375	return -1;
		1376	}
		1377
		1378	if (data->d_size > size) {
		1379	truncated = true;
		1380	data->d_size = size;
		1381	}
		1382
		1383	size -= data->d_size;
		1384	}
		1385	}
		1386
1288	int elf_write(struct elf *elf)	1387	int elf_write(struct elf *elf)
1289	{	1388	{
1290	struct section *sec;	1389	struct section *sec;
@@ -1295,6 +1394,9 @@ int elf_write(struct elf *elf) · 4 unmodified lines
1295		1394
1296	/* Update changed relocation sections and section headers: */	1395	/* Update changed relocation sections and section headers: */
1297	list_for_each_entry(sec, &elf->sections, list) {	1396	list_for_each_entry(sec, &elf->sections, list) {
		1397	if (sec->truncate)
		1398	elf_truncate_section(elf, sec);
		1399
1298	if (sec->changed) {	1400	if (sec->changed) {
1299	s = elf_getscn(elf->elf, sec->idx);	1401	s = elf_getscn(elf->elf, sec->idx);
1300	if (!s) {	1402	if (!s) {

tools/objtool/include/objtool/arch.h+2−0

@@ -93,4 +93,6 @@ bool arch_is_rethunk(struct symbol *sym); · 92 unmodified lines
93		93
94	int arch_rewrite_retpolines(struct objtool_file *file);	94	int arch_rewrite_retpolines(struct objtool_file *file);
95		95
		96	bool arch_pc_relative_reloc(struct reloc *reloc);
		97
96	#endif /* _ARCH_H */	98	#endif /* _ARCH_H */

tools/objtool/include/objtool/builtin.h+3−0

 	bool dump_orc;
 	bool hack_jump_label;
 	bool hack_noinstr;
+	bool hack_skylake;
 	bool ibt;
 	bool mcount;
 	bool noinstr;
 	bool stackval;
 	bool static_call;
 	bool uaccess;
+	int prefix;
+	bool cfi;
 	/* options: */
 	bool backtrace;

tools/objtool/include/objtool/check.h+11−1

 	struct reloc *jump_table;
 	struct reloc *reloc;
 	struct list_head alts;
-	struct symbol *func;
+	struct symbol *sym;
 	struct list_head stack_ops;
 	struct cfi_state *cfi;
 };
+static inline struct symbol *insn_func(struct instruction *insn)
+{
+	struct symbol *sym = insn->sym;
+	if (sym && sym->type != STT_FUNC)
+		sym = NULL;
+	return sym;
+}
 #define VISITED_BRANCH		0x01
 #define VISITED_BRANCH_UACCESS	0x02
 #define VISITED_BRANCH_MASK	0x03

tools/objtool/include/objtool/elf.h+7−2

 	struct hlist_node hash;
 	struct hlist_node name_hash;
 	GElf_Shdr sh;
-	struct rb_root symbol_tree;
+	struct rb_root_cached symbol_tree;
 	struct list_head symbol_list;
 	struct list_head reloc_list;
 	struct section *base, *reloc;
 	Elf_Data *data;
 	char *name;
 	int idx;
-	bool changed, text, rodata, noinstr;
+	bool changed, text, rodata, noinstr, init, truncate;
 };
 struct symbol {
 	unsigned char bind, type;
 	unsigned long offset;
 	unsigned int len;
+	unsigned long __subtree_last;
 	struct symbol *pfunc, *cfunc, *alias;
 	u8 uaccess_safe      : 1;
 	u8 static_call_tramp : 1;
 	u8 fentry            : 1;
 	u8 profiling_func    : 1;
 	struct list_head pv_target;
+	struct list_head reloc_list;
 };
 struct reloc {
 	};
 	struct section *sec;
 	struct symbol *sym;
+	struct list_head sym_reloc_entry;
 	unsigned long offset;
 	unsigned int type;
 	s64 addend;
 struct elf *elf_open_read(const char *name, int flags);
 struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
+struct symbol *elf_create_prefix_symbol(struct elf *elf, struct symbol *orig, long size);
 int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset,
 		  unsigned int type, struct symbol *sym, s64 addend);
 int elf_add_reloc_to_insn(struct elf *elf, struct section *sec,

tools/objtool/include/objtool/objtool.h+1−0

@@ -28,6 +28,7 @@ struct objtool_file { · 27 unmodified lines
28	struct list_head static_call_list;	28	struct list_head static_call_list;
29	struct list_head mcount_loc_list;	29	struct list_head mcount_loc_list;
30	struct list_head endbr_list;	30	struct list_head endbr_list;
		31	struct list_head call_list;
31	bool ignore_unreachables, hints, rodata;	32	bool ignore_unreachables, hints, rodata;
32		33
33	unsigned int nr_endbr;	34	unsigned int nr_endbr;

tools/objtool/objtool.c+1−0

@@ -106,6 +106,7 @@ struct objtool_file objtool_open_read(const char _objname) · 105 unmodified lines
106	INIT_LIST_HEAD(&file.static_call_list);	106	INIT_LIST_HEAD(&file.static_call_list);
107	INIT_LIST_HEAD(&file.mcount_loc_list);	107	INIT_LIST_HEAD(&file.mcount_loc_list);
108	INIT_LIST_HEAD(&file.endbr_list);	108	INIT_LIST_HEAD(&file.endbr_list);
		109	INIT_LIST_HEAD(&file.call_list);
109	file.ignore_unreachables = opts.no_unreachable;	110	file.ignore_unreachables = opts.no_unreachable;
110	file.hints = false;	111	file.hints = false;
111		112