perf: Support deferred user unwind

Add support for deferred userspace unwind to perf. Where perf currently relies on in-place stack unwinding; from NMI context and all that. This moves the userspace part of the unwind to right before the return-to-userspace. This has two distinct benefits, the biggest is that it moves the unwind to a faultable context. It becomes possible to fault in debug info (.eh_frame, SFrame etc.) that might not otherwise be readily available. And secondly, it de-duplicates the user callchain where multiple samples happen during the same kernel entry. To facilitate this the perf interface is extended with a new record type: PERF_RECORD_CALLCHAIN_DEFERRED and two new attribute flags: perf_event_attr::defer_callchain - to request the user unwind be deferred perf_event_attr::defer_output - to request PERF_RECORD_CALLCHAIN_DEFERRED records The existing PERF_RECORD_SAMPLE callchain section gets a new context type: PERF_CONTEXT_USER_DEFERRED After which will come a single entry, denoting the 'cookie' of the deferred callchain that should be attached here, matching the 'cookie' field of the above mentioned PERF_RECORD_CALLCHAIN_DEFERRED. The 'defer_callchain' flag is expected on all events with PERF_SAMPLE_CALLCHAIN. The 'defer_output' flag is expect on the event responsible for collecting side-band events (like mmap, comm etc.). Setting 'defer_output' on multiple events will get you duplicated PERF_RECORD_CALLCHAIN_DEFERRED records. Based on earlier patches by Josh and Steven.

Peter Zijlstra committed 1 year ago

Commit c69993e1 parent ae258848 files · +145 −20Browse files

Unified · Split

8 files

include/linux/perf_event.h +1 −1
include/linux/unwind_deferred.h +0 −12
include/linux/unwind_deferred_types.h +13 −0
include/uapi/linux/perf_event.h +20 −1
kernel/bpf/stackmap.c +2 −2
kernel/events/callchain.c +13 −1
kernel/events/core.c +76 −2
tools/include/uapi/linux/perf_event.h +20 −1

include/linux/perf_event.h+1−1

@@ -1720,7 +1720,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p · 1719 unmodified lines
1720	1720	extern void perf_callchain_kernel(struct perf_callchain_entry_ctx entry, struct pt_regs regs);
1721	1721	extern struct perf_callchain_entry *
1722	1722	get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
1723		− u32 max_stack, bool crosstask, bool add_mark);
	1723	+ u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie);
1724	1724	extern int get_callchain_buffers(int max_stack);
1725	1725	extern void put_callchain_buffers(void);
1726	1726	extern struct perf_callchain_entry get_callchain_entry(int rctx);

include/linux/unwind_deferred.h+0−12

@@ -6,18 +6,6 @@ · 5 unmodified lines
6	6	#include <linux/unwind_user.h>
7	7	#include <linux/unwind_deferred_types.h>
8	8
9		−struct unwind_work;
10		−
11		−typedef void (unwind_callback_t)(struct unwind_work work,
12		− struct unwind_stacktrace *trace,
13		− u64 cookie);
14		−
15		−struct unwind_work {
16		− struct list_head list;
17		− unwind_callback_t func;
18		− int bit;
19		−};
20		−
21	9	#ifdef CONFIG_UNWIND_USER
22	10
23	11	enum {

include/linux/unwind_deferred_types.h+13−0

@@ -39,4 +39,17 @@ struct unwind_task_info { · 38 unmodified lines
39	39	union unwind_task_id id;
40	40	};
41	41
	42	+struct unwind_work;
	43	+struct unwind_stacktrace;
	44	+
	45	+typedef void (unwind_callback_t)(struct unwind_work work,
	46	+ struct unwind_stacktrace *trace,
	47	+ u64 cookie);
	48	+
	49	+struct unwind_work {
	50	+ struct list_head list;
	51	+ unwind_callback_t func;
	52	+ int bit;
	53	+};
	54	+
42	55	#endif /* _LINUX_UNWIND_USER_DEFERRED_TYPES_H */

include/uapi/linux/perf_event.h+20−1

@@ -463,7 +463,9 @@ struct perf_event_attr { · 462 unmodified lines
463	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
464	464	remove_on_exec : 1, /* event is removed from task on exec */
465	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
466		− __reserved_1 : 26;
	466	+ defer_callchain: 1, /* request PERF_RECORD_CALLCHAIN_DEFERRED records */
	467	+ defer_output : 1, /* output PERF_RECORD_CALLCHAIN_DEFERRED records */
	468	+ __reserved_1 : 24;
467	469
468	470	union {
469	471	__u32 wakeup_events; /* wake up every n events */
@@ -1239,6 +1241,22 @@ enum perf_event_type { · 769 unmodified lines
1239	1241	*/
1240	1242	PERF_RECORD_AUX_OUTPUT_HW_ID = 21,
1241	1243
	1244	+ /*
	1245	+ * This user callchain capture was deferred until shortly before
	1246	+ * returning to user space. Previous samples would have kernel
	1247	+ * callchains only and they need to be stitched with this to make full
	1248	+ * callchains.
	1249	+ *
	1250	+ * struct {
	1251	+ * struct perf_event_header header;
	1252	+ * u64 cookie;
	1253	+ * u64 nr;
	1254	+ * u64 ips[nr];
	1255	+ * struct sample_id sample_id;
	1256	+ * };
	1257	+ */
	1258	+ PERF_RECORD_CALLCHAIN_DEFERRED = 22,
	1259	+
1242	1260	PERF_RECORD_MAX, /* non-ABI */
1243	1261	};
1244	1262
@@ -1269,6 +1287,7 @@ enum perf_callchain_context { · 24 unmodified lines
1269	1287	PERF_CONTEXT_HV = (__u64)-32,
1270	1288	PERF_CONTEXT_KERNEL = (__u64)-128,
1271	1289	PERF_CONTEXT_USER = (__u64)-512,
	1290	+ PERF_CONTEXT_USER_DEFERRED = (__u64)-640,
1272	1291
1273	1292	PERF_CONTEXT_GUEST = (__u64)-2048,
1274	1293	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,

kernel/bpf/stackmap.c+2−2

@@ -315,7 +315,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs , regs, struct bpf_map , map, · 314 unmodified lines
315	315	max_depth = sysctl_perf_event_max_stack;
316	316
317	317	trace = get_perf_callchain(regs, kernel, user, max_depth,
318		− false, false);
	318	+ false, false, 0);
319	319
320	320	if (unlikely(!trace))
321	321	/* couldn't fetch the stack trace */
@@ -452,7 +452,7 @@ static long __bpf_get_stack(struct pt_regs regs, struct task_struct task, · 130 unmodified lines
452	452	trace = get_callchain_entry_for_task(task, max_depth);
453	453	else
454	454	trace = get_perf_callchain(regs, kernel, user, max_depth,
455		− crosstask, false);
	455	+ crosstask, false, 0);
456	456
457	457	if (unlikely(!trace) \|\| trace->nr < skip) {
458	458	if (may_fault)

kernel/events/callchain.c+13−1

@@ -218,7 +218,7 @@ static void fixup_uretprobe_trampoline_entries(struct perf_callchain_entry *entr · 217 unmodified lines
218	218
219	219	struct perf_callchain_entry *
220	220	get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
221		− u32 max_stack, bool crosstask, bool add_mark)
	221	+ u32 max_stack, bool crosstask, bool add_mark, u64 defer_cookie)
222	222	{
223	223	struct perf_callchain_entry *entry;
224	224	struct perf_callchain_entry_ctx ctx;
@@ -251,6 +251,18 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, · 26 unmodified lines
251	251	regs = task_pt_regs(current);
252	252	}
253	253
	254	+ if (defer_cookie) {
	255	+ /*
	256	+ * Foretell the coming of PERF_RECORD_CALLCHAIN_DEFERRED
	257	+ * which can be stitched to this one, and add
	258	+ * the cookie after it (it will be cut off when the
	259	+ * user stack is copied to the callchain).
	260	+ */
	261	+ perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED);
	262	+ perf_callchain_store_context(&ctx, defer_cookie);
	263	+ goto exit_put;
	264	+ }
	265	+
254	266	if (add_mark)
255	267	perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
256	268

kernel/events/core.c+76−2

@@ -56,6 +56,7 @@ · 55 unmodified lines
56	56	#include <linux/buildid.h>
57	57	#include <linux/task_work.h>
58	58	#include <linux/percpu-rwsem.h>
	59	+#include <linux/unwind_deferred.h>
59	60
60	61	#include "internal.h"
61	62
@@ -8200,6 +8201,8 @@ static u64 perf_get_page_size(unsigned long addr) · 8138 unmodified lines
8200	8201
8201	8202	static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
8202	8203
	8204	+static struct unwind_work perf_unwind_work;
	8205	+
8203	8206	struct perf_callchain_entry *
8204	8207	perf_callchain(struct perf_event event, struct pt_regs regs)
8205	8208	{
@@ -8208,8 +8211,11 @@ perf_callchain(struct perf_event event, struct pt_regs regs) · 2 unmodified lines
8208	8211	!(current->flags & (PF_KTHREAD \| PF_USER_WORKER));
8209	8212	/* Disallow cross-task user callchains. */
8210	8213	bool crosstask = event->ctx->task && event->ctx->task != current;
	8214	+ bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user &&
	8215	+ event->attr.defer_callchain;
8211	8216	const u32 max_stack = event->attr.sample_max_stack;
8212	8217	struct perf_callchain_entry *callchain;
	8218	+ u64 defer_cookie;
8213	8219
8214	8220	if (!current->mm)
8215	8221	user = false;
@@ -8217,8 +8223,13 @@ perf_callchain(struct perf_event event, struct pt_regs regs) · 1 unmodified line
8217	8223	if (!kernel && !user)
8218	8224	return &__empty_callchain;
8219	8225
8220		− callchain = get_perf_callchain(regs, kernel, user,
8221		− max_stack, crosstask, true);
	8226	+ if (!(user && defer_user && !crosstask &&
	8227	+ unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0))
	8228	+ defer_cookie = 0;
	8229	+
	8230	+ callchain = get_perf_callchain(regs, kernel, user, max_stack,
	8231	+ crosstask, true, defer_cookie);
	8232	+
8222	8233	return callchain ?: &__empty_callchain;
8223	8234	}
8224	8235
@@ -10003,6 +10014,66 @@ void perf_event_bpf_event(struct bpf_prog *prog, · 1778 unmodified lines
10003	10014	perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL);
10004	10015	}
10005	10016
	10017	+struct perf_callchain_deferred_event {
	10018	+ struct unwind_stacktrace *trace;
	10019	+ struct {
	10020	+ struct perf_event_header header;
	10021	+ u64 cookie;
	10022	+ u64 nr;
	10023	+ u64 ips[];
	10024	+ } event;
	10025	+};
	10026	+
	10027	+static void perf_callchain_deferred_output(struct perf_event event, void data)
	10028	+{
	10029	+ struct perf_callchain_deferred_event *deferred_event = data;
	10030	+ struct perf_output_handle handle;
	10031	+ struct perf_sample_data sample;
	10032	+ int ret, size = deferred_event->event.header.size;
	10033	+
	10034	+ if (!event->attr.defer_output)
	10035	+ return;
	10036	+
	10037	+ /* XXX do we really need sample_id_all for this ??? */
	10038	+ perf_event_header__init_id(&deferred_event->event.header, &sample, event);
	10039	+
	10040	+ ret = perf_output_begin(&handle, &sample, event,
	10041	+ deferred_event->event.header.size);
	10042	+ if (ret)
	10043	+ goto out;
	10044	+
	10045	+ perf_output_put(&handle, deferred_event->event);
	10046	+ for (int i = 0; i < deferred_event->trace->nr; i++) {
	10047	+ u64 entry = deferred_event->trace->entries[i];
	10048	+ perf_output_put(&handle, entry);
	10049	+ }
	10050	+ perf_event__output_id_sample(event, &handle, &sample);
	10051	+
	10052	+ perf_output_end(&handle);
	10053	+out:
	10054	+ deferred_event->event.header.size = size;
	10055	+}
	10056	+
	10057	+static void perf_unwind_deferred_callback(struct unwind_work *work,
	10058	+ struct unwind_stacktrace *trace, u64 cookie)
	10059	+{
	10060	+ struct perf_callchain_deferred_event deferred_event = {
	10061	+ .trace = trace,
	10062	+ .event = {
	10063	+ .header = {
	10064	+ .type = PERF_RECORD_CALLCHAIN_DEFERRED,
	10065	+ .misc = PERF_RECORD_MISC_USER,
	10066	+ .size = sizeof(deferred_event.event) +
	10067	+ (trace->nr * sizeof(u64)),
	10068	+ },
	10069	+ .cookie = cookie,
	10070	+ .nr = trace->nr,
	10071	+ },
	10072	+ };
	10073	+
	10074	+ perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
	10075	+}
	10076	+
10006	10077	struct perf_text_poke_event {
10007	10078	const void *old_bytes;
10008	10079	const void *new_bytes;
@@ -14799,6 +14870,9 @@ void __init perf_event_init(void) · 4790 unmodified lines
14799	14870
14800	14871	idr_init(&pmu_idr);
14801	14872
	14873	+ unwind_deferred_init(&perf_unwind_work,
	14874	+ perf_unwind_deferred_callback);
	14875	+
14802	14876	perf_event_init_all_cpus();
14803	14877	init_srcu_struct(&pmus_srcu);
14804	14878	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);

tools/include/uapi/linux/perf_event.h+20−1

@@ -463,7 +463,9 @@ struct perf_event_attr { · 462 unmodified lines
463	463	inherit_thread : 1, /* children only inherit if cloned with CLONE_THREAD */
464	464	remove_on_exec : 1, /* event is removed from task on exec */
465	465	sigtrap : 1, /* send synchronous SIGTRAP on event */
466		− __reserved_1 : 26;
	466	+ defer_callchain: 1, /* request PERF_RECORD_CALLCHAIN_DEFERRED records */
	467	+ defer_output : 1, /* output PERF_RECORD_CALLCHAIN_DEFERRED records */
	468	+ __reserved_1 : 24;
467	469
468	470	union {
469	471	__u32 wakeup_events; /* wake up every n events */
@@ -1239,6 +1241,22 @@ enum perf_event_type { · 769 unmodified lines
1239	1241	*/
1240	1242	PERF_RECORD_AUX_OUTPUT_HW_ID = 21,
1241	1243
	1244	+ /*
	1245	+ * This user callchain capture was deferred until shortly before
	1246	+ * returning to user space. Previous samples would have kernel
	1247	+ * callchains only and they need to be stitched with this to make full
	1248	+ * callchains.
	1249	+ *
	1250	+ * struct {
	1251	+ * struct perf_event_header header;
	1252	+ * u64 cookie;
	1253	+ * u64 nr;
	1254	+ * u64 ips[nr];
	1255	+ * struct sample_id sample_id;
	1256	+ * };
	1257	+ */
	1258	+ PERF_RECORD_CALLCHAIN_DEFERRED = 22,
	1259	+
1242	1260	PERF_RECORD_MAX, /* non-ABI */
1243	1261	};
1244	1262
@@ -1269,6 +1287,7 @@ enum perf_callchain_context { · 24 unmodified lines
1269	1287	PERF_CONTEXT_HV = (__u64)-32,
1270	1288	PERF_CONTEXT_KERNEL = (__u64)-128,
1271	1289	PERF_CONTEXT_USER = (__u64)-512,
	1290	+ PERF_CONTEXT_USER_DEFERRED = (__u64)-640,
1272	1291
1273	1292	PERF_CONTEXT_GUEST = (__u64)-2048,
1274	1293	PERF_CONTEXT_GUEST_KERNEL = (__u64)-2176,

perf: Support deferred user unwind

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code