History

perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

Improves performance.

Peter Zijlstra committed 16 years ago

Commit b7e2ece1 parent 1c34bde5 files · +47 −72Browse files

Showing the changes to kernel/trace (3 of 5 files). Show all

Unified · Split

3 files
  1. kernel/trace/trace_event_perf.c +29 −44
  2. kernel/trace/trace_kprobe.c +4 −6
  3. kernel/trace/trace_syscalls.c +4 −6
kernel/trace/trace_event_perf.c+29−44
@@ -9,13 +9,9 @@ · 8 unmodified lines
99 #include <linux/kprobes.h>
1010 #include "trace.h"
1111
12−DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
13−EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
14−
1512 EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
1613
17−static char *perf_trace_buf;
18−static char *perf_trace_buf_nmi;
14+static char *perf_trace_buf[4];
1915
2016 /*
2117 * Force it to be aligned to unsigned long to avoid misaligned accesses
@@ -29,7 +25,6 @@ static int total_ref_count; · 7 unmodified lines
2925
3026 static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
3127 {
32− char *buf;
3328 int ret = -ENOMEM;
3429
3530 if (event->perf_refcount++ > 0) {
@@ -38,17 +33,16 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 2 unmodified lines
3833 }
3934
4035 if (!total_ref_count) {
41− buf = (char *)alloc_percpu(perf_trace_t);
42− if (!buf)
43− goto fail_buf;
44−
45− rcu_assign_pointer(perf_trace_buf, buf);
36+ char *buf;
37+ int i;
4638
47− buf = (char *)alloc_percpu(perf_trace_t);
48− if (!buf)
49− goto fail_buf_nmi;
39+ for (i = 0; i < 4; i++) {
40+ buf = (char *)alloc_percpu(perf_trace_t);
41+ if (!buf)
42+ goto fail_buf;
5043
51− rcu_assign_pointer(perf_trace_buf_nmi, buf);
44+ rcu_assign_pointer(perf_trace_buf[i], buf);
45+ }
5246 }
5347
5448 ret = event->perf_event_enable(event);
@@ -58,14 +52,15 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 3 unmodified lines
5852 return 0;
5953 }
6054
61−fail_buf_nmi:
55+fail_buf:
6256 if (!total_ref_count) {
63− free_percpu(perf_trace_buf_nmi);
64− free_percpu(perf_trace_buf);
65− perf_trace_buf_nmi = NULL;
66− perf_trace_buf = NULL;
57+ int i;
58+
59+ for (i = 0; i < 4; i++) {
60+ free_percpu(perf_trace_buf[i]);
61+ perf_trace_buf[i] = NULL;
62+ }
6763 }
68−fail_buf:
6964 event->perf_refcount--;
7065
7166 return ret;
@@ -91,19 +86,19 @@ int perf_trace_enable(int event_id, void *data) · 19 unmodified lines
9186
9287 static void perf_trace_event_disable(struct ftrace_event_call *event)
9388 {
94− char *buf, *nmi_buf;
95−
9689 if (--event->perf_refcount > 0)
9790 return;
9891
9992 event->perf_event_disable(event);
10093
10194 if (!--total_ref_count) {
102− buf = perf_trace_buf;
103− rcu_assign_pointer(perf_trace_buf, NULL);
95+ char *buf[4];
96+ int i;
10497
105− nmi_buf = perf_trace_buf_nmi;
106− rcu_assign_pointer(perf_trace_buf_nmi, NULL);
98+ for (i = 0; i < 4; i++) {
99+ buf[i] = perf_trace_buf[i];
100+ rcu_assign_pointer(perf_trace_buf[i], NULL);
101+ }
107102
108103 /*
109104 * Ensure every events in profiling have finished before
@@ -111,8 +106,8 @@ static void perf_trace_event_disable(struct ftrace_event_call *event) · 1 unmodified line
111106 */
112107 synchronize_sched();
113108
114− free_percpu(buf);
115− free_percpu(nmi_buf);
109+ for (i = 0; i < 4; i++)
110+ free_percpu(buf[i]);
116111 }
117112 }
118113
@@ -132,47 +127,37 @@ void perf_trace_disable(int event_id) · 13 unmodified lines
132127 }
133128
134129 __kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
135− int *rctxp, unsigned long *irq_flags)
130+ struct pt_regs *regs, int *rctxp)
136131 {
137132 struct trace_entry *entry;
138133 char *trace_buf, *raw_data;
139− int pc, cpu;
134+ int pc;
140135
141136 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
142137
143138 pc = preempt_count();
144139
145− /* Protect the per cpu buffer, begin the rcu read side */
146− local_irq_save(*irq_flags);
147−
148140 *rctxp = perf_swevent_get_recursion_context();
149141 if (*rctxp < 0)
150142 goto err_recursion;
151143
152− cpu = smp_processor_id();
153−
154− if (in_nmi())
155− trace_buf = rcu_dereference_sched(perf_trace_buf_nmi);
156− else
157− trace_buf = rcu_dereference_sched(perf_trace_buf);
158−
144+ trace_buf = rcu_dereference_sched(perf_trace_buf[*rctxp]);
159145 if (!trace_buf)
160146 goto err;
161147
162− raw_data = per_cpu_ptr(trace_buf, cpu);
148+ raw_data = per_cpu_ptr(trace_buf, smp_processor_id());
163149
164150 /* zero the dead bytes from align to not leak stack to user */
165151 memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
166152
167153 entry = (struct trace_entry *)raw_data;
168− tracing_generic_entry_update(entry, *irq_flags, pc);
154+ tracing_generic_entry_update(entry, regs->flags, pc);
169155 entry->type = type;
170156
171157 return raw_data;
172158 err:
173159 perf_swevent_put_recursion_context(*rctxp);
174160 err_recursion:
175− local_irq_restore(*irq_flags);
176161 return NULL;
177162 }
178163 EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
kernel/trace/trace_kprobe.c+4−6
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 1342 unmodified lines
13431343 struct kprobe_trace_entry_head *entry;
13441344 u8 *data;
13451345 int size, __size, i;
1346− unsigned long irq_flags;
13471346 int rctx;
13481347
13491348 __size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 3 unmodified lines
13531352 "profile buffer not large enough"))
13541353 return;
13551354
1356− entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
1355+ entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
13571356 if (!entry)
13581357 return;
13591358
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 2 unmodified lines
13621361 for (i = 0; i < tp->nr_args; i++)
13631362 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
13641363
1365− perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);
1364+ perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
13661365 }
13671366
13681367 /* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 5 unmodified lines
13741373 struct kretprobe_trace_entry_head *entry;
13751374 u8 *data;
13761375 int size, __size, i;
1377− unsigned long irq_flags;
13781376 int rctx;
13791377
13801378 __size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 3 unmodified lines
13841382 "profile buffer not large enough"))
13851383 return;
13861384
1387− entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);
1385+ entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
13881386 if (!entry)
13891387 return;
13901388
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 4 unmodified lines
13951393 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
13961394
13971395 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
1398− irq_flags, regs, call->perf_data);
1396+ regs, call->perf_data);
13991397 }
14001398
14011399 static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c+4−6
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 437 unmodified lines
438438 {
439439 struct syscall_metadata *sys_data;
440440 struct syscall_trace_enter *rec;
441− unsigned long flags;
442441 int syscall_nr;
443442 int rctx;
444443 int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 16 unmodified lines
461460 return;
462461
463462 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
464− sys_data->enter_event->id, &rctx, &flags);
463+ sys_data->enter_event->id, regs, &rctx);
465464 if (!rec)
466465 return;
467466
468467 rec->nr = syscall_nr;
469468 syscall_get_arguments(current, regs, 0, sys_data->nb_args,
470469 (unsigned long *)&rec->args);
471− perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
470+ perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
472471 sys_data->enter_event->perf_data);
473472 }
474473
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 36 unmodified lines
511510 {
512511 struct syscall_metadata *sys_data;
513512 struct syscall_trace_exit *rec;
514− unsigned long flags;
515513 int syscall_nr;
516514 int rctx;
517515 int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 19 unmodified lines
537535 return;
538536
539537 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
540− sys_data->exit_event->id, &rctx, &flags);
538+ sys_data->exit_event->id, regs, &rctx);
541539 if (!rec)
542540 return;
543541
544542 rec->nr = syscall_nr;
545543 rec->ret = syscall_get_return_value(current, regs);
546544
547− perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,
545+ perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
548546 sys_data->exit_event->perf_data);
549547 }
550548