x86: Prepare inline-asm for straight-line-speculation

Replace all ret/retq instructions with ASM_RET in preparation of making it more than a single instruction.

Peter Zijlstra committed 4 years ago

Commit b17c2ba1 parent f94909c11 files · +19 −14Browse files

Unified · Split

11 files

arch/x86/include/asm/linkage.h +4 −0
arch/x86/include/asm/paravirt.h +1 −1
arch/x86/include/asm/qspinlock_paravirt.h +2 −2
arch/x86/kernel/alternative.c +1 −1
arch/x86/kernel/kprobes/core.c +1 −1
arch/x86/kernel/paravirt.c +2 −2
arch/x86/kvm/emulate.c +2 −2
arch/x86/lib/error-inject.c +2 −1
samples/ftrace/ftrace-direct-modify.c +2 −2
samples/ftrace/ftrace-direct-too.c +1 −1
samples/ftrace/ftrace-direct.c +1 −1

arch/x86/include/asm/linkage.h+4−0

@@ -18,6 +18,10 @@ · 17 unmodified lines
18	18	#define __ALIGN_STR __stringify(__ALIGN)
19	19	#endif
20	20
	21	+#else /* __ASSEMBLY__ */
	22	+
	23	+#define ASM_RET "ret\n\t"
	24	+
21	25	#endif /* __ASSEMBLY__ */
22	26
23	27	#endif /* _ASM_X86_LINKAGE_H */

arch/x86/include/asm/paravirt.h+1−1

@@ -671,7 +671,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); · 670 unmodified lines
671	671	"call " #func ";" \
672	672	PV_RESTORE_ALL_CALLER_REGS \
673	673	FRAME_END \
674		− "ret;" \
	674	+ ASM_RET \
675	675	".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \
676	676	".popsection")
677	677

arch/x86/include/asm/qspinlock_paravirt.h+2−2

@@ -48,7 +48,7 @@ asm (".pushsection .text;" · 47 unmodified lines
48	48	"jne .slowpath;"
49	49	"pop %rdx;"
50	50	FRAME_END
51		− "ret;"
	51	+ ASM_RET
52	52	".slowpath: "
53	53	"push %rsi;"
54	54	"movzbl %al,%esi;"
@@ -56,7 +56,7 @@ asm (".pushsection .text;" · 1 unmodified line
56	56	"pop %rsi;"
57	57	"pop %rdx;"
58	58	FRAME_END
59		− "ret;"
	59	+ ASM_RET
60	60	".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
61	61	".popsection");
62	62

arch/x86/kernel/alternative.c+1−1

@@ -714,7 +714,7 @@ asm ( · 713 unmodified lines
714	714	" .type int3_magic, @function\n"
715	715	"int3_magic:\n"
716	716	" movl $1, (%" _ASM_ARG1 ")\n"
717		−" ret\n"
	717	+ ASM_RET
718	718	" .size int3_magic, .-int3_magic\n"
719	719	" .popsection\n"
720	720	);

arch/x86/kernel/kprobes/core.c+1−1

@@ -1051,7 +1051,7 @@ asm( · 1050 unmodified lines
1051	1051	" addl $4, %esp\n"
1052	1052	" popfl\n"
1053	1053	#endif
1054		− " ret\n"
	1054	+ ASM_RET
1055	1055	".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"
1056	1056	);
1057	1057	NOKPROBE_SYMBOL(__kretprobe_trampoline);

arch/x86/kernel/paravirt.c+2−2

@@ -41,7 +41,7 @@ extern void _paravirt_nop(void); · 40 unmodified lines
41	41	asm (".pushsection .entry.text, \"ax\"\n"
42	42	".global _paravirt_nop\n"
43	43	"_paravirt_nop:\n\t"
44		− "ret\n\t"
	44	+ ASM_RET
45	45	".size _paravirt_nop, . - _paravirt_nop\n\t"
46	46	".type _paravirt_nop, @function\n\t"
47	47	".popsection");
@@ -51,7 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n" · 3 unmodified lines
51	51	".global paravirt_ret0\n"
52	52	"paravirt_ret0:\n\t"
53	53	"xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
54		− "ret\n\t"
	54	+ ASM_RET
55	55	".size paravirt_ret0, . - paravirt_ret0\n\t"
56	56	".type paravirt_ret0, @function\n\t"
57	57	".popsection");

arch/x86/kvm/emulate.c+2−2

@@ -315,7 +315,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 314 unmodified lines
315	315	__FOP_FUNC(#name)
316	316
317	317	#define __FOP_RET(name) \
318		− "ret \n\t" \
	318	+ ASM_RET \
319	319	".size " name ", .-" name "\n\t"
320	320
321	321	#define FOP_RET(name) \
@@ -435,7 +435,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 113 unmodified lines
435	435	__FOP_RET(#op)
436	436
437	437	asm(".pushsection .fixup, \"ax\"\n"
438		− "kvm_fastop_exception: xor %esi, %esi; ret\n"
	438	+ "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
439	439	".popsection");
440	440
441	441	FOP_START(setcc)

arch/x86/lib/error-inject.c+2−1

@@ -1,5 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2
	3	+#include <linux/linkage.h>
3	4	#include <linux/error-injection.h>
4	5	#include <linux/kprobes.h>
5	6
@@ -10,7 +11,7 @@ asm( · 4 unmodified lines
10	11	".type just_return_func, @function\n"
11	12	".globl just_return_func\n"
12	13	"just_return_func:\n"
13		− " ret\n"
	14	+ ASM_RET
14	15	".size just_return_func, .-just_return_func\n"
15	16	);
16	17

samples/ftrace/ftrace-direct-modify.c+2−2

@@ -31,7 +31,7 @@ asm ( · 30 unmodified lines
31	31	" call my_direct_func1\n"
32	32	" leave\n"
33	33	" .size my_tramp1, .-my_tramp1\n"
34		−" ret\n"
	34	+ ASM_RET
35	35	" .type my_tramp2, @function\n"
36	36	" .globl my_tramp2\n"
37	37	" my_tramp2:"
@@ -39,7 +39,7 @@ asm ( · 1 unmodified line
39	39	" movq %rsp, %rbp\n"
40	40	" call my_direct_func2\n"
41	41	" leave\n"
42		−" ret\n"
	42	+ ASM_RET
43	43	" .size my_tramp2, .-my_tramp2\n"
44	44	" .popsection\n"
45	45	);

samples/ftrace/ftrace-direct-too.c+1−1

@@ -31,7 +31,7 @@ asm ( · 30 unmodified lines
31	31	" popq %rsi\n"
32	32	" popq %rdi\n"
33	33	" leave\n"
34		−" ret\n"
	34	+ ASM_RET
35	35	" .size my_tramp, .-my_tramp\n"
36	36	" .popsection\n"
37	37	);

samples/ftrace/ftrace-direct.c+1−1

@@ -25,7 +25,7 @@ asm ( · 24 unmodified lines
25	25	" call my_direct_func\n"
26	26	" popq %rdi\n"
27	27	" leave\n"
28		−" ret\n"
	28	+ ASM_RET
29	29	" .size my_tramp, .-my_tramp\n"
30	30	" .popsection\n"
31	31	);