perf core: Allow setting up max frame stack depth via sysctl

The default remains 127, which is good for most cases, and not even hit most of the time, but then for some cases, as reported by Brendan, 1024+ deep frames are appearing on the radar for things like groovy, ruby. And in some workloads putting a _lower_ cap on this may make sense. One that is per event still needs to be put in place tho. The new file is: # cat /proc/sys/kernel/perf_event_max_stack 127 Chaging it: # echo 256 > /proc/sys/kernel/perf_event_max_stack # cat /proc/sys/kernel/perf_event_max_stack 256 But as soon as there is some event using callchains we get: # echo 512 > /proc/sys/kernel/perf_event_max_stack -bash: echo: write error: Device or resource busy # Because we only allocate the callchain percpu data structures when there is a user, which allows for changing the max easily, its just a matter of having no callchain users at that point.

Arnaldo Carvalho de Melo committed 10 years ago

Commit c5dfd781 parent c2a218c13 files · +84 −23Browse files

Unified · Split

13 files

Documentation/sysctl/kernel.txt +14 −0
arch/arm/kernel/perf_callchain.c +1 −1
arch/arm64/kernel/perf_callchain.c +2 −2
arch/metag/kernel/perf_callchain.c +1 −1
arch/mips/kernel/perf_event.c +2 −2
arch/powerpc/perf/callchain.c +2 −2
arch/sparc/kernel/perf_event.c +3 −3
arch/x86/events/core.c +2 −2
arch/xtensa/kernel/perf_event.c +2 −2
include/linux/perf_event.h +6 −2
kernel/bpf/stackmap.c +4 −4
kernel/events/callchain.c +33 −2
kernel/sysctl.c +12 −0

Documentation/sysctl/kernel.txt+14−0

@@ -60,6 +60,7 @@ show up in /proc/sys/kernel: · 59 unmodified lines
60	60	- panic_on_warn
61	61	- perf_cpu_time_max_percent
62	62	- perf_event_paranoid
	63	+- perf_event_max_stack
63	64	- pid_max
64	65	- powersave-nap [ PPC only ]
65	66	- printk
@@ -654,6 +655,19 @@ users (without CAP_SYS_ADMIN). The default value is 1. · 588 unmodified lines
654	655
655	656	==============================================================
656	657
	658	+perf_event_max_stack:
	659	+
	660	+Controls maximum number of stack frames to copy for (attr.sample_type &
	661	+PERF_SAMPLE_CALLCHAIN) configured events, for instance, when using
	662	+'perf record -g' or 'perf trace --call-graph fp'.
	663	+
	664	+This can only be done when no events are in use that have callchains
	665	+enabled, otherwise writing to this file will return -EBUSY.
	666	+
	667	+The default value is 127.
	668	+
	669	+==============================================================
	670	+
657	671	pid_max:
658	672
659	673	PID allocation wrap value. When the kernel's next PID value

arch/arm/kernel/perf_callchain.c+1−1

@@ -75,7 +75,7 @@ perf_callchain_user(struct perf_callchain_entry entry, struct pt_regs regs) · 74 unmodified lines
75	75
76	76	tail = (struct frame_tail __user *)regs->ARM_fp - 1;
77	77
78		− while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
	78	+ while ((entry->nr < sysctl_perf_event_max_stack) &&
79	79	tail && !((unsigned long)tail & 0x3))
80	80	tail = user_backtrace(tail, entry);
81	81	}

arch/arm64/kernel/perf_callchain.c+2−2

@@ -122,7 +122,7 @@ void perf_callchain_user(struct perf_callchain_entry *entry, · 121 unmodified lines
122	122
123	123	tail = (struct frame_tail __user *)regs->regs[29];
124	124
125		− while (entry->nr < PERF_MAX_STACK_DEPTH &&
	125	+ while (entry->nr < sysctl_perf_event_max_stack &&
126	126	tail && !((unsigned long)tail & 0xf))
127	127	tail = user_backtrace(tail, entry);
128	128	} else {
@@ -132,7 +132,7 @@ void perf_callchain_user(struct perf_callchain_entry *entry, · 3 unmodified lines
132	132
133	133	tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
134	134
135		− while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
	135	+ while ((entry->nr < sysctl_perf_event_max_stack) &&
136	136	tail && !((unsigned long)tail & 0x3))
137	137	tail = compat_user_backtrace(tail, entry);
138	138	#endif

arch/metag/kernel/perf_callchain.c+1−1

@@ -65,7 +65,7 @@ perf_callchain_user(struct perf_callchain_entry entry, struct pt_regs regs) · 64 unmodified lines
65	65
66	66	--frame;
67	67
68		− while ((entry->nr < PERF_MAX_STACK_DEPTH) && frame)
	68	+ while ((entry->nr < sysctl_perf_event_max_stack) && frame)
69	69	frame = user_backtrace(frame, entry);
70	70	}
71	71

arch/mips/kernel/perf_event.c+2−2

@@ -35,7 +35,7 @@ static void save_raw_perf_callchain(struct perf_callchain_entry *entry, · 34 unmodified lines
35	35	addr = *sp++;
36	36	if (__kernel_text_address(addr)) {
37	37	perf_callchain_store(entry, addr);
38		− if (entry->nr >= PERF_MAX_STACK_DEPTH)
	38	+ if (entry->nr >= sysctl_perf_event_max_stack)
39	39	break;
40	40	}
41	41	}
@@ -59,7 +59,7 @@ void perf_callchain_kernel(struct perf_callchain_entry *entry, · 17 unmodified lines
59	59	}
60	60	do {
61	61	perf_callchain_store(entry, pc);
62		− if (entry->nr >= PERF_MAX_STACK_DEPTH)
	62	+ if (entry->nr >= sysctl_perf_event_max_stack)
63	63	break;
64	64	pc = unwind_stack(current, &sp, pc, &ra);
65	65	} while (pc);

arch/powerpc/perf/callchain.c+2−2

@@ -247,7 +247,7 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, · 246 unmodified lines
247	247	sp = regs->gpr[1];
248	248	perf_callchain_store(entry, next_ip);
249	249
250		− while (entry->nr < PERF_MAX_STACK_DEPTH) {
	250	+ while (entry->nr < sysctl_perf_event_max_stack) {
251	251	fp = (unsigned long __user *) sp;
252	252	if (!valid_user_sp(sp, 1) \|\| read_user_stack_64(fp, &next_sp))
253	253	return;
@@ -453,7 +453,7 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, · 199 unmodified lines
453	453	sp = regs->gpr[1];
454	454	perf_callchain_store(entry, next_ip);
455	455
456		− while (entry->nr < PERF_MAX_STACK_DEPTH) {
	456	+ while (entry->nr < sysctl_perf_event_max_stack) {
457	457	fp = (unsigned int __user *) (unsigned long) sp;
458	458	if (!valid_user_sp(sp, 0) \|\| read_user_stack_32(fp, &next_sp))
459	459	return;

arch/sparc/kernel/perf_event.c+3−3

@@ -1756,7 +1756,7 @@ void perf_callchain_kernel(struct perf_callchain_entry *entry, · 1755 unmodified lines
1756	1756	}
1757	1757	}
1758	1758	#endif
1759		− } while (entry->nr < PERF_MAX_STACK_DEPTH);
	1759	+ } while (entry->nr < sysctl_perf_event_max_stack);
1760	1760	}
1761	1761
1762	1762	static inline int
@@ -1790,7 +1790,7 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, · 27 unmodified lines
1790	1790	pc = sf.callers_pc;
1791	1791	ufp = (unsigned long)sf.fp + STACK_BIAS;
1792	1792	perf_callchain_store(entry, pc);
1793		− } while (entry->nr < PERF_MAX_STACK_DEPTH);
	1793	+ } while (entry->nr < sysctl_perf_event_max_stack);
1794	1794	}
1795	1795
1796	1796	static void perf_callchain_user_32(struct perf_callchain_entry *entry,
@@ -1822,7 +1822,7 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, · 25 unmodified lines
1822	1822	ufp = (unsigned long)sf.fp;
1823	1823	}
1824	1824	perf_callchain_store(entry, pc);
1825		− } while (entry->nr < PERF_MAX_STACK_DEPTH);
	1825	+ } while (entry->nr < sysctl_perf_event_max_stack);
1826	1826	}
1827	1827
1828	1828	void

arch/x86/events/core.c+2−2

@@ -2277,7 +2277,7 @@ perf_callchain_user32(struct pt_regs regs, struct perf_callchain_entry entry) · 2276 unmodified lines
2277	2277
2278	2278	fp = compat_ptr(ss_base + regs->bp);
2279	2279	pagefault_disable();
2280		− while (entry->nr < PERF_MAX_STACK_DEPTH) {
	2280	+ while (entry->nr < sysctl_perf_event_max_stack) {
2281	2281	unsigned long bytes;
2282	2282	frame.next_frame = 0;
2283	2283	frame.return_address = 0;
@@ -2337,7 +2337,7 @@ perf_callchain_user(struct perf_callchain_entry entry, struct pt_regs regs) · 53 unmodified lines
2337	2337	return;
2338	2338
2339	2339	pagefault_disable();
2340		− while (entry->nr < PERF_MAX_STACK_DEPTH) {
	2340	+ while (entry->nr < sysctl_perf_event_max_stack) {
2341	2341	unsigned long bytes;
2342	2342	frame.next_frame = NULL;
2343	2343	frame.return_address = 0;

arch/xtensa/kernel/perf_event.c+2−2

@@ -332,14 +332,14 @@ static int callchain_trace(struct stackframe frame, void data) · 331 unmodified lines
332	332	void perf_callchain_kernel(struct perf_callchain_entry *entry,
333	333	struct pt_regs *regs)
334	334	{
335		− xtensa_backtrace_kernel(regs, PERF_MAX_STACK_DEPTH,
	335	+ xtensa_backtrace_kernel(regs, sysctl_perf_event_max_stack,
336	336	callchain_trace, NULL, entry);
337	337	}
338	338
339	339	void perf_callchain_user(struct perf_callchain_entry *entry,
340	340	struct pt_regs *regs)
341	341	{
342		− xtensa_backtrace_user(regs, PERF_MAX_STACK_DEPTH,
	342	+ xtensa_backtrace_user(regs, sysctl_perf_event_max_stack,
343	343	callchain_trace, entry);
344	344	}
345	345

include/linux/perf_event.h+6−2

@@ -58,7 +58,7 @@ struct perf_guest_info_callbacks { · 57 unmodified lines
58	58
59	59	struct perf_callchain_entry {
60	60	__u64 nr;
61		− __u64 ip[PERF_MAX_STACK_DEPTH];
	61	+ __u64 ip[0]; /* /proc/sys/kernel/perf_event_max_stack */
62	62	};
63	63
64	64	struct perf_raw_record {
@@ -993,9 +993,11 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, · 928 unmodified lines
993	993	extern int get_callchain_buffers(void);
994	994	extern void put_callchain_buffers(void);
995	995
	996	+extern int sysctl_perf_event_max_stack;
	997	+
996	998	static inline int perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
997	999	{
998		− if (entry->nr < PERF_MAX_STACK_DEPTH) {
	1000	+ if (entry->nr < sysctl_perf_event_max_stack) {
999	1001	entry->ip[entry->nr++] = ip;
1000	1002	return 0;
1001	1003	} else {
@@ -1017,6 +1019,8 @@ extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, · 15 unmodified lines
1017	1019	void __user buffer, size_t lenp,
1018	1020	loff_t *ppos);
1019	1021
	1022	+int perf_event_max_stack_handler(struct ctl_table *table, int write,
	1023	+ void __user buffer, size_t lenp, loff_t *ppos);
1020	1024
1021	1025	static inline bool perf_paranoid_tracepoint_raw(void)
1022	1026	{

kernel/bpf/stackmap.c+4−4

@@ -66,7 +66,7 @@ static struct bpf_map stack_map_alloc(union bpf_attr attr) · 65 unmodified lines
66	66	/* check sanity of attributes */
67	67	if (attr->max_entries == 0 \|\| attr->key_size != 4 \|\|
68	68	value_size < 8 \|\| value_size % 8 \|\|
69		− value_size / 8 > PERF_MAX_STACK_DEPTH)
	69	+ value_size / 8 > sysctl_perf_event_max_stack)
70	70	return ERR_PTR(-EINVAL);
71	71
72	72	/* hash table size must be power of 2 */
@@ -124,8 +124,8 @@ static u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5) · 51 unmodified lines
124	124	struct perf_callchain_entry *trace;
125	125	struct stack_map_bucket bucket, new_bucket, *old_bucket;
126	126	u32 max_depth = map->value_size / 8;
127		− /* stack_map_alloc() checks that max_depth <= PERF_MAX_STACK_DEPTH */
128		− u32 init_nr = PERF_MAX_STACK_DEPTH - max_depth;
	127	+ /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
	128	+ u32 init_nr = sysctl_perf_event_max_stack - max_depth;
129	129	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
130	130	u32 hash, id, trace_nr, trace_len;
131	131	bool user = flags & BPF_F_USER_STACK;
@@ -143,7 +143,7 @@ static u64 bpf_get_stackid(u64 r1, u64 r2, u64 flags, u64 r4, u64 r5) · 11 unmodified lines
143	143	return -EFAULT;
144	144
145	145	/* get_perf_callchain() guarantees that trace->nr >= init_nr
146		− * and trace-nr <= PERF_MAX_STACK_DEPTH, so trace_nr <= max_depth
	146	+ * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
147	147	*/
148	148	trace_nr = trace->nr - init_nr;
149	149

kernel/events/callchain.c+33−2

@@ -18,6 +18,14 @@ struct callchain_cpus_entries { · 17 unmodified lines
18	18	struct perf_callchain_entry *cpu_entries[0];
19	19	};
20	20
	21	+int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
	22	+
	23	+static inline size_t perf_callchain_entry__sizeof(void)
	24	+{
	25	+ return (sizeof(struct perf_callchain_entry) +
	26	+ sizeof(__u64) * sysctl_perf_event_max_stack);
	27	+}
	28	+
21	29	static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
22	30	static atomic_t nr_callchain_events;
23	31	static DEFINE_MUTEX(callchain_mutex);
@@ -73,7 +81,7 @@ static int alloc_callchain_buffers(void) · 49 unmodified lines
73	81	if (!entries)
74	82	return -ENOMEM;
75	83
76		− size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
	84	+ size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
77	85
78	86	for_each_possible_cpu(cpu) {
79	87	entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
@@ -147,7 +155,8 @@ static struct perf_callchain_entry get_callchain_entry(int rctx) · 67 unmodified lines
147	155
148	156	cpu = smp_processor_id();
149	157
150		− return &entries->cpu_entries[cpu][*rctx];
	158	+ return (((void *)entries->cpu_entries[cpu]) +
	159	+ (rctx perf_callchain_entry__sizeof()));
151	160	}
152	161
153	162	static void
@@ -215,3 +224,25 @@ exit_put: · 61 unmodified lines
215	224
216	225	return entry;
217	226	}
	227	+
	228	+int perf_event_max_stack_handler(struct ctl_table *table, int write,
	229	+ void __user buffer, size_t lenp, loff_t *ppos)
	230	+{
	231	+ int new_value = sysctl_perf_event_max_stack, ret;
	232	+ struct ctl_table new_table = *table;
	233	+
	234	+ new_table.data = &new_value;
	235	+ ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
	236	+ if (ret \|\| !write)
	237	+ return ret;
	238	+
	239	+ mutex_lock(&callchain_mutex);
	240	+ if (atomic_read(&nr_callchain_events))
	241	+ ret = -EBUSY;
	242	+ else
	243	+ sysctl_perf_event_max_stack = new_value;
	244	+
	245	+ mutex_unlock(&callchain_mutex);
	246	+
	247	+ return ret;
	248	+}

kernel/sysctl.c+12−0

@@ -130,6 +130,9 @@ static int one_thousand = 1000; · 129 unmodified lines
130	130	#ifdef CONFIG_PRINTK
131	131	static int ten_thousand = 10000;
132	132	#endif
	133	+#ifdef CONFIG_PERF_EVENTS
	134	+static int six_hundred_forty_kb = 640 * 1024;
	135	+#endif
133	136
134	137	/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
135	138	static unsigned long dirty_bytes_min = 2 * PAGE_SIZE;
@@ -1144,6 +1147,15 @@ static struct ctl_table kern_table[] = { · 1008 unmodified lines
1144	1147	.extra1 = &zero,
1145	1148	.extra2 = &one_hundred,
1146	1149	},
	1150	+ {
	1151	+ .procname = "perf_event_max_stack",
	1152	+ .data = NULL, /* filled in by handler */
	1153	+ .maxlen = sizeof(sysctl_perf_event_max_stack),
	1154	+ .mode = 0644,
	1155	+ .proc_handler = perf_event_max_stack_handler,
	1156	+ .extra1 = &zero,
	1157	+ .extra2 = &six_hundred_forty_kb,
	1158	+ },
1147	1159	#endif
1148	1160	#ifdef CONFIG_KMEMCHECK
1149	1161	{