History

x86,ftrace: Fix ftrace_regs_caller() unwind

The ftrace_regs_caller() trampoline does something 'funny' when there is a direct-caller present. In that case it stuffs the 'direct-caller' address on the return stack and then exits the function. This then results in 'returning' to the direct-caller with the exact registers we came in with -- an indirect tail-call without using a register. This however (rightfully) confuses objtool because the function shares a few instruction in order to have a single exit path, but the stack layout is different for them, depending through which path we came there. This is currently cludged by forcing the stack state to the non-direct case, but this generates actively wrong (ORC) unwind information for the direct case, leading to potential broken unwinds. Fix this issue by fully separating the exit paths. This results in having to poke a second RET into the trampoline copy, see ftrace_regs_caller_ret. This brings us to a second objtool problem, in order for it to perceive the 'jmp ftrace_epilogue' as a function exit, it needs to be recognised as a tail call. In order to make that happen, ftrace_epilogue needs to be the start of an STT_FUNC, so re-arrange code to make this so. Finally, a third issue is that objtool requires functions to exit with the same stack layout they started with, which is obviously violated in the direct case, employ the new HINT_RET_OFFSET to tell objtool this is an expected exception. Together, this results in generating correct ORC unwind information for the ftrace_regs_caller() function and it's trampoline copies.

Peter Zijlstra committed 6 years ago

Commit 02987391 parent e25eea82 files · +25 −19Browse files

Unified · Split

2 files
  1. arch/x86/kernel/ftrace.c +10 −2
  2. arch/x86/kernel/ftrace_64.S +15 −17
arch/x86/kernel/ftrace.c+10−2
@@ -282,7 +282,8 @@ static inline void tramp_free(void *tramp) { } · 281 unmodified lines
282 282
283/* Defined as markers to the end of the ftrace default trampolines */283/* Defined as markers to the end of the ftrace default trampolines */
284extern void ftrace_regs_caller_end(void);284extern void ftrace_regs_caller_end(void);
285extern void ftrace_epilogue(void);285extern void ftrace_regs_caller_ret(void);
286extern void ftrace_caller_end(void);
286extern void ftrace_caller_op_ptr(void);287extern void ftrace_caller_op_ptr(void);
287extern void ftrace_regs_caller_op_ptr(void);288extern void ftrace_regs_caller_op_ptr(void);
288 289
@@ -334,7 +335,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) · 45 unmodified lines
334 call_offset = (unsigned long)ftrace_regs_call;335 call_offset = (unsigned long)ftrace_regs_call;
335 } else {336 } else {
336 start_offset = (unsigned long)ftrace_caller;337 start_offset = (unsigned long)ftrace_caller;
337 end_offset = (unsigned long)ftrace_epilogue;338 end_offset = (unsigned long)ftrace_caller_end;
338 op_offset = (unsigned long)ftrace_caller_op_ptr;339 op_offset = (unsigned long)ftrace_caller_op_ptr;
339 call_offset = (unsigned long)ftrace_call;340 call_offset = (unsigned long)ftrace_call;
340 }341 }
@@ -366,6 +367,13 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) · 25 unmodified lines
366 if (WARN_ON(ret < 0))367 if (WARN_ON(ret < 0))
367 goto fail;368 goto fail;
368 369
370 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
371 ip = trampoline + (ftrace_regs_caller_ret - ftrace_regs_caller);
372 ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
373 if (WARN_ON(ret < 0))
374 goto fail;
375 }
376
369 /*377 /*
370 * The address of the ftrace_ops that is used for this trampoline378 * The address of the ftrace_ops that is used for this trampoline
371 * is stored at the end of the trampoline. This will be used to379 * is stored at the end of the trampoline. This will be used to
arch/x86/kernel/ftrace_64.S+15−17
@@ -157,8 +157,12 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) · 156 unmodified lines
157 * think twice before adding any new code or changing the157 * think twice before adding any new code or changing the
158 * layout here.158 * layout here.
159 */159 */
160SYM_INNER_LABEL(ftrace_epilogue, SYM_L_GLOBAL)160SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
161 161
162 jmp ftrace_epilogue
163SYM_FUNC_END(ftrace_caller);
164
165SYM_FUNC_START(ftrace_epilogue)
162#ifdef CONFIG_FUNCTION_GRAPH_TRACER166#ifdef CONFIG_FUNCTION_GRAPH_TRACER
163SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)167SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
164 jmp ftrace_stub168 jmp ftrace_stub
@@ -170,14 +174,12 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) · 5 unmodified lines
170 */174 */
171SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)175SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
172 retq176 retq
173SYM_FUNC_END(ftrace_caller)177SYM_FUNC_END(ftrace_epilogue)
174 178
175SYM_FUNC_START(ftrace_regs_caller)179SYM_FUNC_START(ftrace_regs_caller)
176 /* Save the current flags before any operations that can change them */180 /* Save the current flags before any operations that can change them */
177 pushfq181 pushfq
178 182
179 UNWIND_HINT_SAVE
180
181 /* added 8 bytes to save flags */183 /* added 8 bytes to save flags */
182 save_mcount_regs 8184 save_mcount_regs 8
183 /* save_mcount_regs fills in first two parameters */185 /* save_mcount_regs fills in first two parameters */
@@ -233,7 +235,10 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 49 unmodified lines
233 movq ORIG_RAX(%rsp), %rax235 movq ORIG_RAX(%rsp), %rax
234 movq %rax, MCOUNT_REG_SIZE-8(%rsp)236 movq %rax, MCOUNT_REG_SIZE-8(%rsp)
235 237
236 /* If ORIG_RAX is anything but zero, make this a call to that */238 /*
239 * If ORIG_RAX is anything but zero, make this a call to that.
240 * See arch_ftrace_set_direct_caller().
241 */
237 movq ORIG_RAX(%rsp), %rax242 movq ORIG_RAX(%rsp), %rax
238 cmpq $0, %rax243 cmpq $0, %rax
239 je 1f244 je 1f
@@ -244,20 +249,14 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 4 unmodified lines
244 movq %rax, MCOUNT_REG_SIZE(%rsp)249 movq %rax, MCOUNT_REG_SIZE(%rsp)
245 250
246 restore_mcount_regs 8251 restore_mcount_regs 8
252 /* Restore flags */
253 popfq
247 254
248 jmp 2f255SYM_INNER_LABEL(ftrace_regs_caller_ret, SYM_L_GLOBAL);
256 UNWIND_HINT_RET_OFFSET
257 jmp ftrace_epilogue
249 258
2501: restore_mcount_regs2591: restore_mcount_regs
251
252
2532:
254 /*
255 * The stack layout is nondetermistic here, depending on which path was
256 * taken. This confuses objtool and ORC, rightfully so. For now,
257 * pretend the stack always looks like the non-direct case.
258 */
259 UNWIND_HINT_RESTORE
260
261 /* Restore flags */260 /* Restore flags */
262 popfq261 popfq
263 262
@@ -268,7 +267,6 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL) · 4 unmodified lines
268 * to the return.267 * to the return.
269 */268 */
270SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)269SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
271
272 jmp ftrace_epilogue270 jmp ftrace_epilogue
273 271
274SYM_FUNC_END(ftrace_regs_caller)272SYM_FUNC_END(ftrace_regs_caller)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t