History

x86: Prepare inline-asm for straight-line-speculation

Replace all ret/retq instructions with ASM_RET in preparation of making it more than a single instruction.

Peter Zijlstra committed 4 years ago

Commit b17c2ba1 parent f94909c11 files · +19 −14Browse files

Unified · Split

11 files
  1. arch/x86/include/asm/linkage.h +4 −0
  2. arch/x86/include/asm/paravirt.h +1 −1
  3. arch/x86/include/asm/qspinlock_paravirt.h +2 −2
  4. arch/x86/kernel/alternative.c +1 −1
  5. arch/x86/kernel/kprobes/core.c +1 −1
  6. arch/x86/kernel/paravirt.c +2 −2
  7. arch/x86/kvm/emulate.c +2 −2
  8. arch/x86/lib/error-inject.c +2 −1
  9. samples/ftrace/ftrace-direct-modify.c +2 −2
  10. samples/ftrace/ftrace-direct-too.c +1 −1
  11. samples/ftrace/ftrace-direct.c +1 −1
arch/x86/include/asm/linkage.h+4−0
@@ -18,6 +18,10 @@ · 17 unmodified lines
18#define __ALIGN_STR __stringify(__ALIGN)18#define __ALIGN_STR __stringify(__ALIGN)
19#endif19#endif
20 20
21#else /* __ASSEMBLY__ */
22
23#define ASM_RET "ret\n\t"
24
21#endif /* __ASSEMBLY__ */25#endif /* __ASSEMBLY__ */
22 26
23#endif /* _ASM_X86_LINKAGE_H */27#endif /* _ASM_X86_LINKAGE_H */
arch/x86/include/asm/paravirt.h+1−1
@@ -671,7 +671,7 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu); · 670 unmodified lines
671 "call " #func ";" \671 "call " #func ";" \
672 PV_RESTORE_ALL_CALLER_REGS \672 PV_RESTORE_ALL_CALLER_REGS \
673 FRAME_END \673 FRAME_END \
674 "ret;" \674 ASM_RET \
675 ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \675 ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \
676 ".popsection")676 ".popsection")
677 677
arch/x86/include/asm/qspinlock_paravirt.h+2−2
@@ -48,7 +48,7 @@ asm (".pushsection .text;" · 47 unmodified lines
48 "jne .slowpath;"48 "jne .slowpath;"
49 "pop %rdx;"49 "pop %rdx;"
50 FRAME_END50 FRAME_END
51 "ret;"51 ASM_RET
52 ".slowpath: "52 ".slowpath: "
53 "push %rsi;"53 "push %rsi;"
54 "movzbl %al,%esi;"54 "movzbl %al,%esi;"
@@ -56,7 +56,7 @@ asm (".pushsection .text;" · 1 unmodified line
56 "pop %rsi;"56 "pop %rsi;"
57 "pop %rdx;"57 "pop %rdx;"
58 FRAME_END58 FRAME_END
59 "ret;"59 ASM_RET
60 ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"60 ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
61 ".popsection");61 ".popsection");
62 62
arch/x86/kernel/alternative.c+1−1
@@ -714,7 +714,7 @@ asm ( · 713 unmodified lines
714" .type int3_magic, @function\n"714" .type int3_magic, @function\n"
715"int3_magic:\n"715"int3_magic:\n"
716" movl $1, (%" _ASM_ARG1 ")\n"716" movl $1, (%" _ASM_ARG1 ")\n"
717" ret\n"717 ASM_RET
718" .size int3_magic, .-int3_magic\n"718" .size int3_magic, .-int3_magic\n"
719" .popsection\n"719" .popsection\n"
720);720);
arch/x86/kernel/kprobes/core.c+1−1
@@ -1051,7 +1051,7 @@ asm( · 1050 unmodified lines
1051 " addl $4, %esp\n"1051 " addl $4, %esp\n"
1052 " popfl\n"1052 " popfl\n"
1053#endif1053#endif
1054 " ret\n"1054 ASM_RET
1055 ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"1055 ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n"
1056);1056);
1057NOKPROBE_SYMBOL(__kretprobe_trampoline);1057NOKPROBE_SYMBOL(__kretprobe_trampoline);
arch/x86/kernel/paravirt.c+2−2
@@ -41,7 +41,7 @@ extern void _paravirt_nop(void); · 40 unmodified lines
41asm (".pushsection .entry.text, \"ax\"\n"41asm (".pushsection .entry.text, \"ax\"\n"
42 ".global _paravirt_nop\n"42 ".global _paravirt_nop\n"
43 "_paravirt_nop:\n\t"43 "_paravirt_nop:\n\t"
44 "ret\n\t"44 ASM_RET
45 ".size _paravirt_nop, . - _paravirt_nop\n\t"45 ".size _paravirt_nop, . - _paravirt_nop\n\t"
46 ".type _paravirt_nop, @function\n\t"46 ".type _paravirt_nop, @function\n\t"
47 ".popsection");47 ".popsection");
@@ -51,7 +51,7 @@ asm (".pushsection .entry.text, \"ax\"\n" · 3 unmodified lines
51 ".global paravirt_ret0\n"51 ".global paravirt_ret0\n"
52 "paravirt_ret0:\n\t"52 "paravirt_ret0:\n\t"
53 "xor %" _ASM_AX ", %" _ASM_AX ";\n\t"53 "xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
54 "ret\n\t"54 ASM_RET
55 ".size paravirt_ret0, . - paravirt_ret0\n\t"55 ".size paravirt_ret0, . - paravirt_ret0\n\t"
56 ".type paravirt_ret0, @function\n\t"56 ".type paravirt_ret0, @function\n\t"
57 ".popsection");57 ".popsection");
arch/x86/kvm/emulate.c+2−2
@@ -315,7 +315,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 314 unmodified lines
315 __FOP_FUNC(#name)315 __FOP_FUNC(#name)
316 316
317#define __FOP_RET(name) \317#define __FOP_RET(name) \
318 "ret \n\t" \318 ASM_RET \
319 ".size " name ", .-" name "\n\t"319 ".size " name ", .-" name "\n\t"
320 320
321#define FOP_RET(name) \321#define FOP_RET(name) \
@@ -435,7 +435,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop); · 113 unmodified lines
435 __FOP_RET(#op)435 __FOP_RET(#op)
436 436
437asm(".pushsection .fixup, \"ax\"\n"437asm(".pushsection .fixup, \"ax\"\n"
438 "kvm_fastop_exception: xor %esi, %esi; ret\n"438 "kvm_fastop_exception: xor %esi, %esi; " ASM_RET
439 ".popsection");439 ".popsection");
440 440
441FOP_START(setcc)441FOP_START(setcc)
arch/x86/lib/error-inject.c+2−1
@@ -1,5 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2 2
3#include <linux/linkage.h>
3#include <linux/error-injection.h>4#include <linux/error-injection.h>
4#include <linux/kprobes.h>5#include <linux/kprobes.h>
5 6
@@ -10,7 +11,7 @@ asm( · 4 unmodified lines
10 ".type just_return_func, @function\n"11 ".type just_return_func, @function\n"
11 ".globl just_return_func\n"12 ".globl just_return_func\n"
12 "just_return_func:\n"13 "just_return_func:\n"
13 " ret\n"14 ASM_RET
14 ".size just_return_func, .-just_return_func\n"15 ".size just_return_func, .-just_return_func\n"
15);16);
16 17
samples/ftrace/ftrace-direct-modify.c+2−2
@@ -31,7 +31,7 @@ asm ( · 30 unmodified lines
31" call my_direct_func1\n"31" call my_direct_func1\n"
32" leave\n"32" leave\n"
33" .size my_tramp1, .-my_tramp1\n"33" .size my_tramp1, .-my_tramp1\n"
34" ret\n"34 ASM_RET
35" .type my_tramp2, @function\n"35" .type my_tramp2, @function\n"
36" .globl my_tramp2\n"36" .globl my_tramp2\n"
37" my_tramp2:"37" my_tramp2:"
@@ -39,7 +39,7 @@ asm ( · 1 unmodified line
39" movq %rsp, %rbp\n"39" movq %rsp, %rbp\n"
40" call my_direct_func2\n"40" call my_direct_func2\n"
41" leave\n"41" leave\n"
42" ret\n"42 ASM_RET
43" .size my_tramp2, .-my_tramp2\n"43" .size my_tramp2, .-my_tramp2\n"
44" .popsection\n"44" .popsection\n"
45);45);
samples/ftrace/ftrace-direct-too.c+1−1
@@ -31,7 +31,7 @@ asm ( · 30 unmodified lines
31" popq %rsi\n"31" popq %rsi\n"
32" popq %rdi\n"32" popq %rdi\n"
33" leave\n"33" leave\n"
34" ret\n"34 ASM_RET
35" .size my_tramp, .-my_tramp\n"35" .size my_tramp, .-my_tramp\n"
36" .popsection\n"36" .popsection\n"
37);37);
samples/ftrace/ftrace-direct.c+1−1
@@ -25,7 +25,7 @@ asm ( · 24 unmodified lines
25" call my_direct_func\n"25" call my_direct_func\n"
26" popq %rdi\n"26" popq %rdi\n"
27" leave\n"27" leave\n"
28" ret\n"28 ASM_RET
29" .size my_tramp, .-my_tramp\n"29" .size my_tramp, .-my_tramp\n"
30" .popsection\n"30" .popsection\n"
31);31);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t