History

x86,ftrace: Fix ftrace_regs_caller() unwind

The ftrace_regs_caller() trampoline does something 'funny' when there is a direct-caller present. In that case it stuffs the 'direct-caller' address on the return stack and then exits the function. This then results in 'returning' to the direct-caller with the exact registers we came in with -- an indirect tail-call without using a register. This however (rightfully) confuses objtool because the function shares a few instruction in order to have a single exit path, but the stack layout is different for them, depending through which path we came there. This is currently cludged by forcing the stack state to the non-direct case, but this generates actively wrong (ORC) unwind information for the direct case, leading to potential broken unwinds. Fix this issue by fully separating the exit paths. This results in having to poke a second RET into the trampoline copy, see ftrace_regs_caller_ret. This brings us to a second objtool problem, in order for it to perceive the 'jmp ftrace_epilogue' as a function exit, it needs to be recognised as a tail call. In order to make that happen, ftrace_epilogue needs to be the start of an STT_FUNC, so re-arrange code to make this so. Finally, a third issue is that objtool requires functions to exit with the same stack layout they started with, which is obviously violated in the direct case, employ the new HINT_RET_OFFSET to tell objtool this is an expected exception. Together, this results in generating correct ORC unwind information for the ftrace_regs_caller() function and it's trampoline copies.

Peter Zijlstra committed 6 years ago

Commit 02987391 parent e25eea82 files · +25 −19Browse files

Unified · Split

2 files
  1. arch/x86/kernel/ftrace.c +10 −2
  2. arch/x86/kernel/ftrace_64.S +15 −17
arch/x86/kernel/ftrace.c+10−2
@@ -282,7 +282,8 @@ static inline void tramp_free(void *tramp) { } · 281 unmodified lines
282282
283283 /* Defined as markers to the end of the ftrace default trampolines */
284284 extern void ftrace_regs_caller_end(void);
285−extern void ftrace_epilogue(void);
285+extern void ftrace_regs_caller_ret(void);
286+extern void ftrace_caller_end(void);
286287 extern void ftrace_caller_op_ptr(void);
287288 extern void ftrace_regs_caller_op_ptr(void);
288289
@@ -334,7 +335,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) · 45 unmodified lines
334335 call_offset = (unsigned long)ftrace_regs_call;
335336 } else {
336337 start_offset = (unsigned long)ftrace_caller;
337− end_offset = (unsigned long)ftrace_epilogue;
338+ end_offset = (unsigned long)ftrace_caller_end;
338339 op_offset = (unsigned long)ftrace_caller_op_ptr;
339340 call_offset = (unsigned long)ftrace_call;
340341 }
@@ -366,6 +367,13 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) · 25 unmodified lines
366367 if (WARN_ON(ret < 0))
367368 goto fail;
368369
370+ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
371+ ip = trampoline + (ftrace_regs_caller_ret - ftrace_regs_caller);
372+ ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
373+ if (WARN_ON(ret < 0))
374+ goto fail;
375+ }
376+
369377 /*
370378 * The address of the ftrace_ops that is used for this trampoline
371379 * is stored at the end of the trampoline. This will be used to
arch/x86/kernel/ftrace_64.S+15−17
@@ -157,8 +157,12 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) · 156 unmodified lines
157157 * think twice before adding any new code or changing the
158158 * layout here.
159159 */
160−SYM_INNER_LABEL(ftrace_epilogue, SYM_L_GLOBAL)
160+SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
161161
162+ jmp ftrace_epilogue
163+SYM_FUNC_END(ftrace_caller);
164+
165+SYM_FUNC_START(ftrace_epilogue)
162166 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
163167 SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
164168 jmp ftrace_stub
@@ -170,14 +174,12 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) · 5 unmodified lines
170174 */
171175 SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
172176 retq
173−SYM_FUNC_END(ftrace_caller)
177+SYM_FUNC_END(ftrace_epilogue)
174178
175179 SYM_FUNC_START(ftrace_regs_caller)
176180 /* Save the current flags before any operations that can change them */
177181 pushfq
178182
179− UNWIND_HINT_SAVE
180−
181183 /* added 8 bytes to save flags */
182184 save_mcount_regs 8
183185 /* save_mcount_regs fills in first two parameters */
@@ -233,7 +235,10 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 49 unmodified lines
233235 movq ORIG_RAX(%rsp), %rax
234236 movq %rax, MCOUNT_REG_SIZE-8(%rsp)
235237
236− /* If ORIG_RAX is anything but zero, make this a call to that */
238+ /*
239+ * If ORIG_RAX is anything but zero, make this a call to that.
240+ * See arch_ftrace_set_direct_caller().
241+ */
237242 movq ORIG_RAX(%rsp), %rax
238243 cmpq $0, %rax
239244 je 1f
@@ -244,20 +249,14 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 4 unmodified lines
244249 movq %rax, MCOUNT_REG_SIZE(%rsp)
245250
246251 restore_mcount_regs 8
252+ /* Restore flags */
253+ popfq
247254
248− jmp 2f
255+SYM_INNER_LABEL(ftrace_regs_caller_ret, SYM_L_GLOBAL);
256+ UNWIND_HINT_RET_OFFSET
257+ jmp ftrace_epilogue
249258
250259 1: restore_mcount_regs
251−
252−
253−2:
254− /*
255− * The stack layout is nondetermistic here, depending on which path was
256− * taken. This confuses objtool and ORC, rightfully so. For now,
257− * pretend the stack always looks like the non-direct case.
258− */
259− UNWIND_HINT_RESTORE
260−
261260 /* Restore flags */
262261 popfq
263262
@@ -268,7 +267,6 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 4 unmodified lines
268267 * to the return.
269268 */
270269 SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
271−
272270 jmp ftrace_epilogue
273271
274272 SYM_FUNC_END(ftrace_regs_caller)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t