Add support for deferred userspace unwind to perf. Where perf currently relies on in-place stack unwinding; from NMI context and all that. This moves the userspace part of the unwind to right before the return-to-userspace. This has two distinct benefits, the biggest is that it moves the unwind to a faultable context. It becomes possible to fault in debug info (.eh_frame, SFrame etc.) that might not otherwise be readily available. And secondly, it de-duplicates the user callchain where multiple samples happen during the same kernel entry. To facilitate this the perf interface is extended with a new record type: PERF_RECORD_CALLCHAIN_DEFERRED and two new attribute flags: perf_event_attr::defer_callchain - to request the user unwind be deferred perf_event_attr::defer_output - to request PERF_RECORD_CALLCHAIN_DEFERRED records The existing PERF_RECORD_SAMPLE callchain section gets a new context type: PERF_CONTEXT_USER_DEFERRED After which will come a single entry, denoting the 'cookie' of the deferred callchain that should be attached here, matching the 'cookie' field of the above mentioned PERF_RECORD_CALLCHAIN_DEFERRED. The 'defer_callchain' flag is expected on all events with PERF_SAMPLE_CALLCHAIN. The 'defer_output' flag is expect on the event responsible for collecting side-band events (like mmap, comm etc.). Setting 'defer_output' on multiple events will get you duplicated PERF_RECORD_CALLCHAIN_DEFERRED records. Based on earlier patches by Josh and Steven.
Unified · Split
| @@ -1720,7 +1720,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p · 1719 unmodified lines | |||
| 1720 | extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs); | 1720 | extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs); |
| 1721 | extern struct perf_callchain_entry * | 1721 | extern struct perf_callchain_entry * |
| 1722 | get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, | 1722 | get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, |
| 1723 | u32 max_stack, bool crosstask, bool add_mark); | 1723 | u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie); |
| 1724 | extern int get_callchain_buffers(int max_stack); | 1724 | extern int get_callchain_buffers(int max_stack); |
| 1725 | extern void put_callchain_buffers(void); | 1725 | extern void put_callchain_buffers(void); |
| 1726 | extern struct perf_callchain_entry *get_callchain_entry(int *rctx); | 1726 | extern struct perf_callchain_entry *get_callchain_entry(int *rctx); |
| @@ -6,18 +6,6 @@ · 5 unmodified lines | |||
| 6 | #include <linux/unwind_user.h> | 6 | #include <linux/unwind_user.h> |
| 7 | #include <linux/unwind_deferred_types.h> | 7 | #include <linux/unwind_deferred_types.h> |
| 8 | 8 | ||
| 9 | struct unwind_work; | ||
| 10 | |||
| 11 | typedef void (*unwind_callback_t)(struct unwind_work *work, | ||
| 12 | struct unwind_stacktrace *trace, | ||
| 13 | u64 cookie); | ||
| 14 | |||
| 15 | struct unwind_work { | ||
| 16 | struct list_head list; | ||
| 17 | unwind_callback_t func; | ||
| 18 | int bit; | ||
| 19 | }; | ||
| 20 | |||
| 21 | #ifdef CONFIG_UNWIND_USER | 9 | #ifdef CONFIG_UNWIND_USER |
| 22 | 10 | ||
| 23 | enum { | 11 | enum { |
| @@ -39,4 +39,17 @@ struct unwind_task_info { · 38 unmodified lines | |||
| 39 | union unwind_task_id id; | 39 | union unwind_task_id id; |
| 40 | }; | 40 | }; |
| 41 | 41 | ||
| 42 | struct unwind_work; | ||
| 43 | struct unwind_stacktrace; | ||
| 44 | |||
| 45 | typedef void (*unwind_callback_t)(struct unwind_work *work, | ||
| 46 | struct unwind_stacktrace *trace, | ||
| 47 | u64 cookie); | ||
| 48 | |||
| 49 | struct unwind_work { | ||
| 50 | struct list_head list; | ||
| 51 | unwind_callback_t func; | ||
| 52 | int bit; | ||
| 53 | }; | ||
| 54 | |||
| 42 | #endif /* _LINUX_UNWIND_USER_DEFERRED_TYPES_H */ | 55 | #endif /* _LINUX_UNWIND_USER_DEFERRED_TYPES_H */ |
| @@ -463,7 +463,9 @@ struct perf_event_attr { · 462 unmodified lines | |||
| 463 | inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ | 463 | inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ |
| 464 | remove_on_exec : 1, /* event is removed from task on exec */ | 464 | remove_on_exec : 1, /* event is removed from task on exec */ |
| 465 | sigtrap : 1, /* send synchronous SIGTRAP on event */ | 465 | sigtrap : 1, /* send synchronous SIGTRAP on event */ |
| 466 | __reserved_1 : 26; | 466 | defer_callchain: 1, /* request PERF_RECORD_CALLCHAIN_DEFERRED records */ |
| 467 | defer_output : 1, /* output PERF_RECORD_CALLCHAIN_DEFERRED records */ | ||
| 468 | __reserved_1 : 24; | ||
| 467 | 469 | ||
| 468 | union { | 470 | union { |
| 469 | __u32 wakeup_events; /* wake up every n events */ | 471 | __u32 wakeup_events; /* wake up every n events */ |
| @@ -1239,6 +1241,22 @@ enum perf_event_type { · 769 unmodified lines | |||
| 1239 | */ | 1241 | */ |
| 1240 | PERF_RECORD_AUX_OUTPUT_HW_ID = 21, | 1242 | PERF_RECORD_AUX_OUTPUT_HW_ID = 21, |
| 1241 | 1243 | ||
| 1244 | /* | ||
| 1245 | * This user callchain capture was deferred until shortly before | ||
| 1246 | * returning to user space. Previous samples would have kernel | ||
| 1247 | * callchains only and they need to be stitched with this to make full | ||
| 1248 | * callchains. | ||
| 1249 | * | ||
| 1250 | * struct { | ||
| 1251 | * struct perf_event_header header; | ||
| 1252 | * u64 cookie; | ||
| 1253 | * u64 nr; | ||
| 1254 | * u64 ips[nr]; | ||
| 1255 | * struct sample_id sample_id; | ||
| 1256 | * }; | ||
| 1257 | */ | ||
| 1258 | PERF_RECORD_CALLCHAIN_DEFERRED = 22, | ||
| 1259 | |||
| 1242 | PERF_RECORD_MAX, /* non-ABI */ | 1260 | PERF_RECORD_MAX, /* non-ABI */ |
| 1243 | }; | 1261 | }; |
| 1244 | 1262 | ||
| @@ -1269,6 +1287,7 @@ enum perf_callchain_context { · 24 unmodified lines | |||
| 1269 | PERF_CONTEXT_HV = (__u64)-32, | 1287 | PERF_CONTEXT_HV = (__u64)-32, |
| 1270 | PERF_CONTEXT_KERNEL = (__u64)-128, | 1288 | PERF_CONTEXT_KERNEL = (__u64)-128, |
| 1271 | PERF_CONTEXT_USER = (__u64)-512, | 1289 | PERF_CONTEXT_USER = (__u64)-512, |
| 1290 | PERF_CONTEXT_USER_DEFERRED = (__u64)-640, | ||
| 1272 | 1291 | ||
| 1273 | PERF_CONTEXT_GUEST = (__u64)-2048, | 1292 | PERF_CONTEXT_GUEST = (__u64)-2048, |
| 1274 | PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, | 1293 | PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, |
| @@ -315,7 +315,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 314 unmodified lines | |||
| 315 | max_depth = sysctl_perf_event_max_stack; | 315 | max_depth = sysctl_perf_event_max_stack; |
| 316 | 316 | ||
| 317 | trace = get_perf_callchain(regs, kernel, user, max_depth, | 317 | trace = get_perf_callchain(regs, kernel, user, max_depth, |
| 318 | false, false); | 318 | false, false, 0); |
| 319 | 319 | ||
| 320 | if (unlikely(!trace)) | 320 | if (unlikely(!trace)) |
| 321 | /* couldn't fetch the stack trace */ | 321 | /* couldn't fetch the stack trace */ |
| @@ -452,7 +452,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · 130 unmodified lines | |||
| 452 | trace = get_callchain_entry_for_task(task, max_depth); | 452 | trace = get_callchain_entry_for_task(task, max_depth); |
| 453 | else | 453 | else |
| 454 | trace = get_perf_callchain(regs, kernel, user, max_depth, | 454 | trace = get_perf_callchain(regs, kernel, user, max_depth, |
| 455 | crosstask, false); | 455 | crosstask, false, 0); |
| 456 | 456 | ||
| 457 | if (unlikely(!trace) || trace->nr < skip) { | 457 | if (unlikely(!trace) || trace->nr < skip) { |
| 458 | if (may_fault) | 458 | if (may_fault) |
| @@ -218,7 +218,7 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr · 217 unmodified lines | |||
| 218 | 218 | ||
| 219 | struct perf_callchain_entry * | 219 | struct perf_callchain_entry * |
| 220 | get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, | 220 | get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, |
| 221 | u32 max_stack, bool crosstask, bool add_mark) | 221 | u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie) |
| 222 | { | 222 | { |
| 223 | struct perf_callchain_entry *entry; | 223 | struct perf_callchain_entry *entry; |
| 224 | struct perf_callchain_entry_ctx ctx; | 224 | struct perf_callchain_entry_ctx ctx; |
| @@ -251,6 +251,18 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, · 26 unmodified lines | |||
| 251 | regs = task_pt_regs(current); | 251 | regs = task_pt_regs(current); |
| 252 | } | 252 | } |
| 253 | 253 | ||
| 254 | if (defer_cookie) { | ||
| 255 | /* | ||
| 256 | * Foretell the coming of PERF_RECORD_CALLCHAIN_DEFERRED | ||
| 257 | * which can be stitched to this one, and add | ||
| 258 | * the cookie after it (it will be cut off when the | ||
| 259 | * user stack is copied to the callchain). | ||
| 260 | */ | ||
| 261 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED); | ||
| 262 | perf_callchain_store_context(&ctx, defer_cookie); | ||
| 263 | goto exit_put; | ||
| 264 | } | ||
| 265 | |||
| 254 | if (add_mark) | 266 | if (add_mark) |
| 255 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); | 267 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); |
| 256 | 268 | ||
| @@ -56,6 +56,7 @@ · 55 unmodified lines | |||
| 56 | #include <linux/buildid.h> | 56 | #include <linux/buildid.h> |
| 57 | #include <linux/task_work.h> | 57 | #include <linux/task_work.h> |
| 58 | #include <linux/percpu-rwsem.h> | 58 | #include <linux/percpu-rwsem.h> |
| 59 | #include <linux/unwind_deferred.h> | ||
| 59 | 60 | ||
| 60 | #include "internal.h" | 61 | #include "internal.h" |
| 61 | 62 | ||
| @@ -8200,6 +8201,8 @@ static u64 perf_get_page_size(unsigned long addr) · 8138 unmodified lines | |||
| 8200 | 8201 | ||
| 8201 | static struct perf_callchain_entry __empty_callchain = { .nr = 0, }; | 8202 | static struct perf_callchain_entry __empty_callchain = { .nr = 0, }; |
| 8202 | 8203 | ||
| 8204 | static struct unwind_work perf_unwind_work; | ||
| 8205 | |||
| 8203 | struct perf_callchain_entry * | 8206 | struct perf_callchain_entry * |
| 8204 | perf_callchain(struct perf_event *event, struct pt_regs *regs) | 8207 | perf_callchain(struct perf_event *event, struct pt_regs *regs) |
| 8205 | { | 8208 | { |
| @@ -8208,8 +8211,11 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) · 2 unmodified lines | |||
| 8208 | !(current->flags & (PF_KTHREAD | PF_USER_WORKER)); | 8211 | !(current->flags & (PF_KTHREAD | PF_USER_WORKER)); |
| 8209 | /* Disallow cross-task user callchains. */ | 8212 | /* Disallow cross-task user callchains. */ |
| 8210 | bool crosstask = event->ctx->task && event->ctx->task != current; | 8213 | bool crosstask = event->ctx->task && event->ctx->task != current; |
| 8214 | bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user && | ||
| 8215 | event->attr.defer_callchain; | ||
| 8211 | const u32 max_stack = event->attr.sample_max_stack; | 8216 | const u32 max_stack = event->attr.sample_max_stack; |
| 8212 | struct perf_callchain_entry *callchain; | 8217 | struct perf_callchain_entry *callchain; |
| 8218 | u64 defer_cookie; | ||
| 8213 | 8219 | ||
| 8214 | if (!current->mm) | 8220 | if (!current->mm) |
| 8215 | user = false; | 8221 | user = false; |
| @@ -8217,8 +8223,13 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs) · 1 unmodified line | |||
| 8217 | if (!kernel && !user) | 8223 | if (!kernel && !user) |
| 8218 | return &__empty_callchain; | 8224 | return &__empty_callchain; |
| 8219 | 8225 | ||
| 8220 | callchain = get_perf_callchain(regs, kernel, user, | 8226 | if (!(user && defer_user && !crosstask && |
| 8221 | max_stack, crosstask, true); | 8227 | unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0)) |
| 8228 | defer_cookie = 0; | ||
| 8229 | |||
| 8230 | callchain = get_perf_callchain(regs, kernel, user, max_stack, | ||
| 8231 | crosstask, true, defer_cookie); | ||
| 8232 | |||
| 8222 | return callchain ?: &__empty_callchain; | 8233 | return callchain ?: &__empty_callchain; |
| 8223 | } | 8234 | } |
| 8224 | 8235 | ||
| @@ -10003,6 +10014,66 @@ void perf_event_bpf_event(struct bpf_prog *prog, · 1778 unmodified lines | |||
| 10003 | perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL); | 10014 | perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL); |
| 10004 | } | 10015 | } |
| 10005 | 10016 | ||
| 10017 | struct perf_callchain_deferred_event { | ||
| 10018 | struct unwind_stacktrace *trace; | ||
| 10019 | struct { | ||
| 10020 | struct perf_event_header header; | ||
| 10021 | u64 cookie; | ||
| 10022 | u64 nr; | ||
| 10023 | u64 ips[]; | ||
| 10024 | } event; | ||
| 10025 | }; | ||
| 10026 | |||
| 10027 | static void perf_callchain_deferred_output(struct perf_event *event, void *data) | ||
| 10028 | { | ||
| 10029 | struct perf_callchain_deferred_event *deferred_event = data; | ||
| 10030 | struct perf_output_handle handle; | ||
| 10031 | struct perf_sample_data sample; | ||
| 10032 | int ret, size = deferred_event->event.header.size; | ||
| 10033 | |||
| 10034 | if (!event->attr.defer_output) | ||
| 10035 | return; | ||
| 10036 | |||
| 10037 | /* XXX do we really need sample_id_all for this ??? */ | ||
| 10038 | perf_event_header__init_id(&deferred_event->event.header, &sample, event); | ||
| 10039 | |||
| 10040 | ret = perf_output_begin(&handle, &sample, event, | ||
| 10041 | deferred_event->event.header.size); | ||
| 10042 | if (ret) | ||
| 10043 | goto out; | ||
| 10044 | |||
| 10045 | perf_output_put(&handle, deferred_event->event); | ||
| 10046 | for (int i = 0; i < deferred_event->trace->nr; i++) { | ||
| 10047 | u64 entry = deferred_event->trace->entries[i]; | ||
| 10048 | perf_output_put(&handle, entry); | ||
| 10049 | } | ||
| 10050 | perf_event__output_id_sample(event, &handle, &sample); | ||
| 10051 | |||
| 10052 | perf_output_end(&handle); | ||
| 10053 | out: | ||
| 10054 | deferred_event->event.header.size = size; | ||
| 10055 | } | ||
| 10056 | |||
| 10057 | static void perf_unwind_deferred_callback(struct unwind_work *work, | ||
| 10058 | struct unwind_stacktrace *trace, u64 cookie) | ||
| 10059 | { | ||
| 10060 | struct perf_callchain_deferred_event deferred_event = { | ||
| 10061 | .trace = trace, | ||
| 10062 | .event = { | ||
| 10063 | .header = { | ||
| 10064 | .type = PERF_RECORD_CALLCHAIN_DEFERRED, | ||
| 10065 | .misc = PERF_RECORD_MISC_USER, | ||
| 10066 | .size = sizeof(deferred_event.event) + | ||
| 10067 | (trace->nr * sizeof(u64)), | ||
| 10068 | }, | ||
| 10069 | .cookie = cookie, | ||
| 10070 | .nr = trace->nr, | ||
| 10071 | }, | ||
| 10072 | }; | ||
| 10073 | |||
| 10074 | perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL); | ||
| 10075 | } | ||
| 10076 | |||
| 10006 | struct perf_text_poke_event { | 10077 | struct perf_text_poke_event { |
| 10007 | const void *old_bytes; | 10078 | const void *old_bytes; |
| 10008 | const void *new_bytes; | 10079 | const void *new_bytes; |
| @@ -14799,6 +14870,9 @@ void __init perf_event_init(void) · 4790 unmodified lines | |||
| 14799 | 14870 | ||
| 14800 | idr_init(&pmu_idr); | 14871 | idr_init(&pmu_idr); |
| 14801 | 14872 | ||
| 14873 | unwind_deferred_init(&perf_unwind_work, | ||
| 14874 | perf_unwind_deferred_callback); | ||
| 14875 | |||
| 14802 | perf_event_init_all_cpus(); | 14876 | perf_event_init_all_cpus(); |
| 14803 | init_srcu_struct(&pmus_srcu); | 14877 | init_srcu_struct(&pmus_srcu); |
| 14804 | perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); | 14878 | perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); |
| @@ -463,7 +463,9 @@ struct perf_event_attr { · 462 unmodified lines | |||
| 463 | inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ | 463 | inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */ |
| 464 | remove_on_exec : 1, /* event is removed from task on exec */ | 464 | remove_on_exec : 1, /* event is removed from task on exec */ |
| 465 | sigtrap : 1, /* send synchronous SIGTRAP on event */ | 465 | sigtrap : 1, /* send synchronous SIGTRAP on event */ |
| 466 | __reserved_1 : 26; | 466 | defer_callchain: 1, /* request PERF_RECORD_CALLCHAIN_DEFERRED records */ |
| 467 | defer_output : 1, /* output PERF_RECORD_CALLCHAIN_DEFERRED records */ | ||
| 468 | __reserved_1 : 24; | ||
| 467 | 469 | ||
| 468 | union { | 470 | union { |
| 469 | __u32 wakeup_events; /* wake up every n events */ | 471 | __u32 wakeup_events; /* wake up every n events */ |
| @@ -1239,6 +1241,22 @@ enum perf_event_type { · 769 unmodified lines | |||
| 1239 | */ | 1241 | */ |
| 1240 | PERF_RECORD_AUX_OUTPUT_HW_ID = 21, | 1242 | PERF_RECORD_AUX_OUTPUT_HW_ID = 21, |
| 1241 | 1243 | ||
| 1244 | /* | ||
| 1245 | * This user callchain capture was deferred until shortly before | ||
| 1246 | * returning to user space. Previous samples would have kernel | ||
| 1247 | * callchains only and they need to be stitched with this to make full | ||
| 1248 | * callchains. | ||
| 1249 | * | ||
| 1250 | * struct { | ||
| 1251 | * struct perf_event_header header; | ||
| 1252 | * u64 cookie; | ||
| 1253 | * u64 nr; | ||
| 1254 | * u64 ips[nr]; | ||
| 1255 | * struct sample_id sample_id; | ||
| 1256 | * }; | ||
| 1257 | */ | ||
| 1258 | PERF_RECORD_CALLCHAIN_DEFERRED = 22, | ||
| 1259 | |||
| 1242 | PERF_RECORD_MAX, /* non-ABI */ | 1260 | PERF_RECORD_MAX, /* non-ABI */ |
| 1243 | }; | 1261 | }; |
| 1244 | 1262 | ||
| @@ -1269,6 +1287,7 @@ enum perf_callchain_context { · 24 unmodified lines | |||
| 1269 | PERF_CONTEXT_HV = (__u64)-32, | 1287 | PERF_CONTEXT_HV = (__u64)-32, |
| 1270 | PERF_CONTEXT_KERNEL = (__u64)-128, | 1288 | PERF_CONTEXT_KERNEL = (__u64)-128, |
| 1271 | PERF_CONTEXT_USER = (__u64)-512, | 1289 | PERF_CONTEXT_USER = (__u64)-512, |
| 1290 | PERF_CONTEXT_USER_DEFERRED = (__u64)-640, | ||
| 1272 | 1291 | ||
| 1273 | PERF_CONTEXT_GUEST = (__u64)-2048, | 1292 | PERF_CONTEXT_GUEST = (__u64)-2048, |
| 1274 | PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, | 1293 | PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176, |