History

perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

Improves performance.

Peter Zijlstra committed 16 years ago

Commit b7e2ece1 parent 1c34bde5 files · +47 −72Browse files

Unified · Split

5 files
  1. include/linux/ftrace_event.h +3 −6
  2. include/trace/ftrace.h +7 −10
  3. kernel/trace/trace_event_perf.c +29 −44
  4. kernel/trace/trace_kprobe.c +4 −6
  5. kernel/trace/trace_syscalls.c +4 −6
include/linux/ftrace_event.h+3−6
@@ -197,20 +197,17 @@ extern void perf_trace_disable(int event_id); · 196 unmodified lines
197197 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
198198 char *filter_str);
199199 extern void ftrace_profile_free_filter(struct perf_event *event);
200−extern void *
201−perf_trace_buf_prepare(int size, unsigned short type, int *rctxp,
202− unsigned long *irq_flags);
200+extern void *perf_trace_buf_prepare(int size, unsigned short type,
201+ struct pt_regs *regs, int *rctxp);
203202
204203 static inline void
205204 perf_trace_buf_submit(void *raw_data, int size, int rctx, u64 addr,
206− u64 count, unsigned long irq_flags, struct pt_regs *regs,
207− void *event)
205+ u64 count, struct pt_regs *regs, void *event)
208206 {
209207 struct trace_entry *entry = raw_data;
210208
211209 perf_tp_event(entry->type, addr, count, raw_data, size, regs, event);
212210 perf_swevent_put_recursion_context(rctx);
213− local_irq_restore(irq_flags);
214211 }
215212 #endif
216213
include/trace/ftrace.h+7−10
@@ -768,7 +768,6 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \ · 767 unmodified lines
768768 struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\
769769 struct ftrace_raw_##call *entry; \
770770 u64 __addr = 0, __count = 1; \
771− unsigned long irq_flags; \
772771 int __entry_size; \
773772 int __data_size; \
774773 int rctx; \
@@ -781,17 +780,18 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \ · 6 unmodified lines
781780 if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE, \
782781 "profile buffer not large enough")) \
783782 return; \
783+ \
784784 entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare( \
785− __entry_size, event_call->id, &rctx, &irq_flags); \
785+ __entry_size, event_call->id, __regs, &rctx); \
786786 if (!entry) \
787787 return; \
788+ \
788789 tstruct \
789790 \
790791 { assign; } \
791792 \
792793 perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \
793− __count, irq_flags, __regs, \
794− event_call->perf_data); \
794+ __count, __regs, event_call->perf_data); \
795795 }
796796
797797 #undef DEFINE_EVENT
@@ -799,13 +799,10 @@ perf_trace_templ_##call(struct ftrace_event_call *event_call, \ · 1 unmodified line
799799 static notrace void perf_trace_##call(proto) \
800800 { \
801801 struct ftrace_event_call *event_call = &event_##call; \
802− struct pt_regs *__regs = &get_cpu_var(perf_trace_regs); \
803− \
804− perf_fetch_caller_regs(__regs, 1); \
805− \
806− perf_trace_templ_##template(event_call, __regs, args); \
802+ struct pt_regs __regs; \
807803 \
808− put_cpu_var(perf_trace_regs); \
804+ perf_fetch_caller_regs(&__regs, 1); \
805+ perf_trace_templ_##template(event_call, &__regs, args); \
809806 }
810807
811808 #undef DEFINE_EVENT_PRINT
kernel/trace/trace_event_perf.c+29−44
@@ -9,13 +9,9 @@ · 8 unmodified lines
99 #include <linux/kprobes.h>
1010 #include "trace.h"
1111
12−DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
13−EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
14−
1512 EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
1613
17−static char *perf_trace_buf;
18−static char *perf_trace_buf_nmi;
14+static char *perf_trace_buf[4];
1915
2016 /*
2117 * Force it to be aligned to unsigned long to avoid misaligned accesses
@@ -29,7 +25,6 @@ static int total_ref_count; · 7 unmodified lines
2925
3026 static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
3127 {
32− char *buf;
3328 int ret = -ENOMEM;
3429
3530 if (event->perf_refcount++ > 0) {
@@ -38,17 +33,16 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 2 unmodified lines
3833 }
3934
4035 if (!total_ref_count) {
41− buf = (char *)alloc_percpu(perf_trace_t);
42− if (!buf)
43− goto fail_buf;
44−
45− rcu_assign_pointer(perf_trace_buf, buf);
36+ char *buf;
37+ int i;
4638
47− buf = (char *)alloc_percpu(perf_trace_t);
48− if (!buf)
49− goto fail_buf_nmi;
39+ for (i = 0; i < 4; i++) {
40+ buf = (char *)alloc_percpu(perf_trace_t);
41+ if (!buf)
42+ goto fail_buf;
5043
51− rcu_assign_pointer(perf_trace_buf_nmi, buf);
44+ rcu_assign_pointer(perf_trace_buf[i], buf);
45+ }
5246 }
5347
5448 ret = event->perf_event_enable(event);
@@ -58,14 +52,15 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 3 unmodified lines
5852 return 0;
5953 }
6054
61−fail_buf_nmi:
55+fail_buf:
6256 if (!total_ref_count) {
63− free_percpu(perf_trace_buf_nmi);
64− free_percpu(perf_trace_buf);
65− perf_trace_buf_nmi = NULL;
66− perf_trace_buf = NULL;
57+ int i;
58+
59+ for (i = 0; i < 4; i++) {
60+ free_percpu(perf_trace_buf[i]);
61+ perf_trace_buf[i] = NULL;
62+ }
6763 }
68−fail_buf:
6964 event->perf_refcount--;
7065
7166 return ret;
@@ -91,19 +86,19 @@ int perf_trace_enable(int event_id, void *data) · 19 unmodified lines
9186
9287 static void perf_trace_event_disable(struct ftrace_event_call *event)
9388 {
94− char *buf, *nmi_buf;
95−
9689 if (--event->perf_refcount > 0)
9790 return;
9891
9992 event->perf_event_disable(event);
10093
10194 if (!--total_ref_count) {
102− buf = perf_trace_buf;
103− rcu_assign_pointer(perf_trace_buf, NULL);
95+ char *buf[4];
96+ int i;
10497
105− nmi_buf = perf_trace_buf_nmi;
106− rcu_assign_pointer(perf_trace_buf_nmi, NULL);
98+ for (i = 0; i < 4; i++) {
99+ buf[i] = perf_trace_buf[i];
100+ rcu_assign_pointer(perf_trace_buf[i], NULL);
101+ }
107102
108103 /*
109104 * Ensure every events in profiling have finished before
@@ -111,8 +106,8 @@ static void perf_trace_event_disable(struct ftrace_event_call *event) · 1 unmodified line
111106 */
112107 synchronize_sched();
113108
114− free_percpu(buf);
115− free_percpu(nmi_buf);
109+ for (i = 0; i < 4; i++)
110+ free_percpu(buf[i]);
116111 }
117112 }
118113
@@ -132,47 +127,37 @@ void perf_trace_disable(int event_id) · 13 unmodified lines
132127 }
133128
134129 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
135− int *rctxp, unsigned long *irq_flags)
130+ struct pt_regs *regs, int *rctxp)
136131 {
137132 struct trace_entry *entry;
138133 char *trace_buf, *raw_data;
139− int pc, cpu;
134+ int pc;
140135
141136 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
142137
143138 pc = preempt_count();
144139
145− /* Protect the per cpu buffer, begin the rcu read side */
146− local_irq_save(*irq_flags);
147−
148140 *rctxp = perf_swevent_get_recursion_context();
149141 if (*rctxp < 0)
150142 goto err_recursion;
151143
152− cpu = smp_processor_id();
153−
154− if (in_nmi())
155− trace_buf = rcu_dereference_sched(perf_trace_buf_nmi);
156− else
157− trace_buf = rcu_dereference_sched(perf_trace_buf);
158−
144+ trace_buf = rcu_dereference_sched(perf_trace_buf[*rctxp]);
159145 if (!trace_buf)
160146 goto err;
161147
162− raw_data = per_cpu_ptr(trace_buf, cpu);
148+ raw_data = per_cpu_ptr(trace_buf, smp_processor_id());
163149
164150 /* zero the dead bytes from align to not leak stack to user */
165151 memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
166152
167153 entry = (struct trace_entry *)raw_data;
168− tracing_generic_entry_update(entry, *irq_flags, pc);
154+ tracing_generic_entry_update(entry, regs->flags, pc);
169155 entry->type = type;
170156
171157 return raw_data;
172158 err:
173159 perf_swevent_put_recursion_context(*rctxp);
174160 err_recursion:
175− local_irq_restore(*irq_flags);
176161 return NULL;
177162 }
178163 EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
kernel/trace/trace_kprobe.c+4−6
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 1342 unmodified lines
13431343 struct kprobe_trace_entry_head *entry;
13441344 u8 *data;
13451345 int size, __size, i;
1346− unsigned long irq_flags;
13471346 int rctx;
13481347
13491348 __size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 3 unmodified lines
13531352 "profile buffer not large enough"))
13541353 return;
13551354
1356− entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
1355+ entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
13571356 if (!entry)
13581357 return;
13591358
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 2 unmodified lines
13621361 for (i = 0; i < tp->nr_args; i++)
13631362 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
13641363
1365− perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
1364+ perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
13661365 }
13671366
13681367 /* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 5 unmodified lines
13741373 struct kretprobe_trace_entry_head *entry;
13751374 u8 *data;
13761375 int size, __size, i;
1377− unsigned long irq_flags;
13781376 int rctx;
13791377
13801378 __size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 3 unmodified lines
13841382 "profile buffer not large enough"))
13851383 return;
13861384
1387− entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
1385+ entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
13881386 if (!entry)
13891387 return;
13901388
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 4 unmodified lines
13951393 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
13961394
13971395 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
1398− irq_flags, regs, call->perf_data);
1396+ regs, call->perf_data);
13991397 }
14001398
14011399 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c+4−6
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 437 unmodified lines
438438 {
439439 struct syscall_metadata *sys_data;
440440 struct syscall_trace_enter *rec;
441− unsigned long flags;
442441 int syscall_nr;
443442 int rctx;
444443 int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 16 unmodified lines
461460 return;
462461
463462 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
464− sys_data->enter_event->id, &rctx, &flags);
463+ sys_data->enter_event->id, regs, &rctx);
465464 if (!rec)
466465 return;
467466
468467 rec->nr = syscall_nr;
469468 syscall_get_arguments(current, regs, 0, sys_data->nb_args,
470469 (unsigned long *)&rec->args);
471− perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
470+ perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
472471 sys_data->enter_event->perf_data);
473472 }
474473
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 36 unmodified lines
511510 {
512511 struct syscall_metadata *sys_data;
513512 struct syscall_trace_exit *rec;
514− unsigned long flags;
515513 int syscall_nr;
516514 int rctx;
517515 int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 19 unmodified lines
537535 return;
538536
539537 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
540− sys_data->exit_event->id, &rctx, &flags);
538+ sys_data->exit_event->id, regs, &rctx);
541539 if (!rec)
542540 return;
543541
544542 rec->nr = syscall_nr;
545543 rec->ret = syscall_get_return_value(current, regs);
546544
547− perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
545+ perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
548546 sys_data->exit_event->perf_data);
549547 }
550548