History

perf, trace: Optimize tracepoints by removing IRQ-disable from perf/tracepoint interaction

Improves performance.

Peter Zijlstra committed 16 years ago

Commit b7e2ece1 parent 1c34bde5 files · +47 −72Browse files

Showing the changes to kernel/trace (3 of 5 files). Show all

Unified · Split

3 files
  1. kernel/trace/trace_event_perf.c +29 −44
  2. kernel/trace/trace_kprobe.c +4 −6
  3. kernel/trace/trace_syscalls.c +4 −6
kernel/trace/trace_event_perf.c+29−44
@@ -9,13 +9,9 @@ · 8 unmodified lines
9#include <linux/kprobes.h>9#include <linux/kprobes.h>
10#include "trace.h"10#include "trace.h"
11 11
12DEFINE_PER_CPU(struct pt_regs, perf_trace_regs);
13EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs);
14
15EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);12EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs);
16 13
17static char *perf_trace_buf;14static char *perf_trace_buf[4];
18static char *perf_trace_buf_nmi;
19 15
20/*16/*
21 * Force it to be aligned to unsigned long to avoid misaligned accesses17 * Force it to be aligned to unsigned long to avoid misaligned accesses
@@ -29,7 +25,6 @@ static int total_ref_count; · 7 unmodified lines
29 25
30static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)26static int perf_trace_event_enable(struct ftrace_event_call *event, void *data)
31{27{
32 char *buf;
33 int ret = -ENOMEM;28 int ret = -ENOMEM;
34 29
35 if (event->perf_refcount++ > 0) {30 if (event->perf_refcount++ > 0) {
@@ -38,17 +33,16 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 2 unmodified lines
38 }33 }
39 34
40 if (!total_ref_count) {35 if (!total_ref_count) {
41 buf = (char *)alloc_percpu(perf_trace_t);36 char *buf;
42 if (!buf)37 int i;
43 goto fail_buf;
44
45 rcu_assign_pointer(perf_trace_buf, buf);
46 38
47 buf = (char *)alloc_percpu(perf_trace_t);39 for (i = 0; i < 4; i++) {
48 if (!buf)40 buf = (char *)alloc_percpu(perf_trace_t);
49 goto fail_buf_nmi;41 if (!buf)
42 goto fail_buf;
50 43
51 rcu_assign_pointer(perf_trace_buf_nmi, buf);44 rcu_assign_pointer(perf_trace_buf[i], buf);
45 }
52 }46 }
53 47
54 ret = event->perf_event_enable(event);48 ret = event->perf_event_enable(event);
@@ -58,14 +52,15 @@ static int perf_trace_event_enable(struct ftrace_event_call *event, void *data) · 3 unmodified lines
58 return 0;52 return 0;
59 }53 }
60 54
61fail_buf_nmi:55fail_buf:
62 if (!total_ref_count) {56 if (!total_ref_count) {
63 free_percpu(perf_trace_buf_nmi);57 int i;
64 free_percpu(perf_trace_buf);58
65 perf_trace_buf_nmi = NULL;59 for (i = 0; i < 4; i++) {
66 perf_trace_buf = NULL;60 free_percpu(perf_trace_buf[i]);
61 perf_trace_buf[i] = NULL;
62 }
67 }63 }
68fail_buf:
69 event->perf_refcount--;64 event->perf_refcount--;
70 65
71 return ret;66 return ret;
@@ -91,19 +86,19 @@ int perf_trace_enable(int event_id, void *data) · 19 unmodified lines
91 86
92static void perf_trace_event_disable(struct ftrace_event_call *event)87static void perf_trace_event_disable(struct ftrace_event_call *event)
93{88{
94 char *buf, *nmi_buf;
95
96 if (--event->perf_refcount > 0)89 if (--event->perf_refcount > 0)
97 return;90 return;
98 91
99 event->perf_event_disable(event);92 event->perf_event_disable(event);
100 93
101 if (!--total_ref_count) {94 if (!--total_ref_count) {
102 buf = perf_trace_buf;95 char *buf[4];
103 rcu_assign_pointer(perf_trace_buf, NULL);96 int i;
104 97
105 nmi_buf = perf_trace_buf_nmi;98 for (i = 0; i < 4; i++) {
106 rcu_assign_pointer(perf_trace_buf_nmi, NULL);99 buf[i] = perf_trace_buf[i];
100 rcu_assign_pointer(perf_trace_buf[i], NULL);
101 }
107 102
108 /*103 /*
109 * Ensure every events in profiling have finished before104 * Ensure every events in profiling have finished before
@@ -111,8 +106,8 @@ static void perf_trace_event_disable(struct ftrace_event_call *event) · 1 unmodified line
111 */106 */
112 synchronize_sched();107 synchronize_sched();
113 108
114 free_percpu(buf);109 for (i = 0; i < 4; i++)
115 free_percpu(nmi_buf);110 free_percpu(buf[i]);
116 }111 }
117}112}
118 113
@@ -132,47 +127,37 @@ void perf_trace_disable(int event_id) · 13 unmodified lines
132}127}
133 128
134__kprobes void *perf_trace_buf_prepare(int size, unsigned short type,129__kprobes void *perf_trace_buf_prepare(int size, unsigned short type,
135 int *rctxp, unsigned long *irq_flags)130 struct pt_regs *regs, int *rctxp)
136{131{
137 struct trace_entry *entry;132 struct trace_entry *entry;
138 char *trace_buf, *raw_data;133 char *trace_buf, *raw_data;
139 int pc, cpu;134 int pc;
140 135
141 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));136 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
142 137
143 pc = preempt_count();138 pc = preempt_count();
144 139
145 /* Protect the per cpu buffer, begin the rcu read side */
146 local_irq_save(*irq_flags);
147
148 *rctxp = perf_swevent_get_recursion_context();140 *rctxp = perf_swevent_get_recursion_context();
149 if (*rctxp < 0)141 if (*rctxp < 0)
150 goto err_recursion;142 goto err_recursion;
151 143
152 cpu = smp_processor_id();144 trace_buf = rcu_dereference_sched(perf_trace_buf[*rctxp]);
153
154 if (in_nmi())
155 trace_buf = rcu_dereference_sched(perf_trace_buf_nmi);
156 else
157 trace_buf = rcu_dereference_sched(perf_trace_buf);
158
159 if (!trace_buf)145 if (!trace_buf)
160 goto err;146 goto err;
161 147
162 raw_data = per_cpu_ptr(trace_buf, cpu);148 raw_data = per_cpu_ptr(trace_buf, smp_processor_id());
163 149
164 /* zero the dead bytes from align to not leak stack to user */150 /* zero the dead bytes from align to not leak stack to user */
165 memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));151 memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
166 152
167 entry = (struct trace_entry *)raw_data;153 entry = (struct trace_entry *)raw_data;
168 tracing_generic_entry_update(entry, *irq_flags, pc);154 tracing_generic_entry_update(entry, regs->flags, pc);
169 entry->type = type;155 entry->type = type;
170 156
171 return raw_data;157 return raw_data;
172err:158err:
173 perf_swevent_put_recursion_context(*rctxp);159 perf_swevent_put_recursion_context(*rctxp);
174err_recursion:160err_recursion:
175 local_irq_restore(*irq_flags);
176 return NULL;161 return NULL;
177}162}
178EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);163EXPORT_SYMBOL_GPL(perf_trace_buf_prepare);
kernel/trace/trace_kprobe.c+4−6
@@ -1343,7 +1343,6 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 1342 unmodified lines
1343 struct kprobe_trace_entry_head *entry;1343 struct kprobe_trace_entry_head *entry;
1344 u8 *data;1344 u8 *data;
1345 int size, __size, i;1345 int size, __size, i;
1346 unsigned long irq_flags;
1347 int rctx;1346 int rctx;
1348 1347
1349 __size = sizeof(*entry) + tp->size;1348 __size = sizeof(*entry) + tp->size;
@@ -1353,7 +1352,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 3 unmodified lines
1353 "profile buffer not large enough"))1352 "profile buffer not large enough"))
1354 return;1353 return;
1355 1354
1356 entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);1355 entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
1357 if (!entry)1356 if (!entry)
1358 return;1357 return;
1359 1358
@@ -1362,7 +1361,7 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, · 2 unmodified lines
1362 for (i = 0; i < tp->nr_args; i++)1361 for (i = 0; i < tp->nr_args; i++)
1363 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);1362 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
1364 1363
1365 perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs, call->perf_data);1364 perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, call->perf_data);
1366}1365}
1367 1366
1368/* Kretprobe profile handler */1367/* Kretprobe profile handler */
@@ -1374,7 +1373,6 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 5 unmodified lines
1374 struct kretprobe_trace_entry_head *entry;1373 struct kretprobe_trace_entry_head *entry;
1375 u8 *data;1374 u8 *data;
1376 int size, __size, i;1375 int size, __size, i;
1377 unsigned long irq_flags;
1378 int rctx;1376 int rctx;
1379 1377
1380 __size = sizeof(*entry) + tp->size;1378 __size = sizeof(*entry) + tp->size;
@@ -1384,7 +1382,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 3 unmodified lines
1384 "profile buffer not large enough"))1382 "profile buffer not large enough"))
1385 return;1383 return;
1386 1384
1387 entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags);1385 entry = perf_trace_buf_prepare(size, call->id, regs, &rctx);
1388 if (!entry)1386 if (!entry)
1389 return;1387 return;
1390 1388
@@ -1395,7 +1393,7 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, · 4 unmodified lines
1395 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);1393 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset);
1396 1394
1397 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,1395 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1,
1398 irq_flags, regs, call->perf_data);1396 regs, call->perf_data);
1399}1397}
1400 1398
1401static int probe_perf_enable(struct ftrace_event_call *call)1399static int probe_perf_enable(struct ftrace_event_call *call)
kernel/trace/trace_syscalls.c+4−6
@@ -438,7 +438,6 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 437 unmodified lines
438{438{
439 struct syscall_metadata *sys_data;439 struct syscall_metadata *sys_data;
440 struct syscall_trace_enter *rec;440 struct syscall_trace_enter *rec;
441 unsigned long flags;
442 int syscall_nr;441 int syscall_nr;
443 int rctx;442 int rctx;
444 int size;443 int size;
@@ -461,14 +460,14 @@ static void perf_syscall_enter(struct pt_regs *regs, long id) · 16 unmodified lines
461 return;460 return;
462 461
463 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,462 rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size,
464 sys_data->enter_event->id, &rctx, &flags);463 sys_data->enter_event->id, regs, &rctx);
465 if (!rec)464 if (!rec)
466 return;465 return;
467 466
468 rec->nr = syscall_nr;467 rec->nr = syscall_nr;
469 syscall_get_arguments(current, regs, 0, sys_data->nb_args,468 syscall_get_arguments(current, regs, 0, sys_data->nb_args,
470 (unsigned long *)&rec->args);469 (unsigned long *)&rec->args);
471 perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,470 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
472 sys_data->enter_event->perf_data);471 sys_data->enter_event->perf_data);
473}472}
474 473
@@ -511,7 +510,6 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 36 unmodified lines
511{510{
512 struct syscall_metadata *sys_data;511 struct syscall_metadata *sys_data;
513 struct syscall_trace_exit *rec;512 struct syscall_trace_exit *rec;
514 unsigned long flags;
515 int syscall_nr;513 int syscall_nr;
516 int rctx;514 int rctx;
517 int size;515 int size;
@@ -537,14 +535,14 @@ static void perf_syscall_exit(struct pt_regs *regs, long ret) · 19 unmodified lines
537 return;535 return;
538 536
539 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,537 rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size,
540 sys_data->exit_event->id, &rctx, &flags);538 sys_data->exit_event->id, regs, &rctx);
541 if (!rec)539 if (!rec)
542 return;540 return;
543 541
544 rec->nr = syscall_nr;542 rec->nr = syscall_nr;
545 rec->ret = syscall_get_return_value(current, regs);543 rec->ret = syscall_get_return_value(current, regs);
546 544
547 perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs,545 perf_trace_buf_submit(rec, size, rctx, 0, 1, regs,
548 sys_data->exit_event->perf_data);546 sys_data->exit_event->perf_data);
549}547}
550 548