// SPDX-License-Identifier: GPL-2.0/* * * Function graph tracer. * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> * Mostly borrowed from function tracer which * is Copyright (c) Steven Rostedt <srostedt@redhat.com> * */#include <linux/uaccess.h>#include <linux/ftrace.h>#include <linux/interrupt.h>#include <linux/slab.h>#include <linux/fs.h> #include "trace.h"#include "trace_output.h" /* When set, irq functions will be ignored */static int ftrace_graph_skip_irqs; struct fgraph_cpu_data { pid_t last_pid; int depth; int depth_irq; int ignore; unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH];}; struct fgraph_data { struct fgraph_cpu_data __percpu *cpu_data; /* Place to preserve last processed entry. */ union { struct ftrace_graph_ent_entry ent; struct fgraph_retaddr_ent_entry rent; } ent; struct ftrace_graph_ret_entry ret; int failed; int cpu;}; #define TRACE_GRAPH_INDENT 2 unsigned int fgraph_max_depth; static struct tracer_opt trace_opts[] = { /* Display overruns? (for self-debug purpose) */ { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, /* Display CPU ? */ { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, /* Display Overhead ? */ { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, /* Display proc name/pid */ { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, /* Display duration of execution */ { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, /* Display absolute time of an entry */ { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, /* Display interrupts */ { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, /* Display function name after trailing } */ { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) },#ifdef CONFIG_FUNCTION_GRAPH_RETVAL /* Display function return value ? */ { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, /* Display function return value in hexadecimal format ? */ { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) },#endif#ifdef CONFIG_FUNCTION_GRAPH_RETADDR /* Display function return address ? */ { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) },#endif /* Include sleep time (scheduled out) between entry and return */ { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, #ifdef CONFIG_FUNCTION_PROFILER /* Include time within nested functions */ { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) },#endif { } /* Empty entry */}; static struct tracer_flags tracer_flags = { /* Don't display overruns, proc, or tail by default */ .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, .opts = trace_opts}; static bool tracer_flags_is_set(u32 flags){ return (tracer_flags.val & flags) == flags;} /* * DURATION column is being also used to display IRQ signs, * following values are used by print_graph_irq and others * to fill in space into DURATION column. */enum { FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT,}; static voidprint_graph_duration(struct trace_array *tr, unsigned long long duration, struct trace_seq *s, u32 flags); int __trace_graph_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx){ struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct ftrace_graph_ent_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, sizeof(*entry), trace_ctx); if (!event) return 0; entry = ring_buffer_event_data(event); entry->graph_ent = *trace; trace_buffer_unlock_commit_nostack(buffer, event); return 1;} #ifdef CONFIG_FUNCTION_GRAPH_RETADDRint __trace_graph_retaddr_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx, unsigned long retaddr){ struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct fgraph_retaddr_ent_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, sizeof(*entry), trace_ctx); if (!event) return 0; entry = ring_buffer_event_data(event); entry->graph_ent.func = trace->func; entry->graph_ent.depth = trace->depth; entry->graph_ent.retaddr = retaddr; trace_buffer_unlock_commit_nostack(buffer, event); return 1;}#elseint __trace_graph_retaddr_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx, unsigned long retaddr){ return 1;}#endif static inline int ftrace_graph_ignore_irqs(void){ if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) return 0; return in_hardirq();} struct fgraph_times { unsigned long long calltime; unsigned long long sleeptime; /* may be optional! */}; int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs){ unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; struct fgraph_times *ftimes; unsigned int trace_ctx; long disabled; int ret = 0; int cpu; if (*task_var & TRACE_GRAPH_NOTRACE) return 0; /* * Do not trace a function if it's filtered by set_graph_notrace. * Make the index of ret stack negative to indicate that it should * ignore further functions. But it needs its own ret stack entry * to recover the original index in order to continue tracing after * returning from the function. */ if (ftrace_graph_notrace_addr(trace->func)) { *task_var |= TRACE_GRAPH_NOTRACE; /* * Need to return 1 to have the return called * that will clear the NOTRACE bit. */ return 1; } if (!ftrace_trace_task(tr)) return 0; if (ftrace_graph_ignore_func(gops, trace)) return 0; if (ftrace_graph_ignore_irqs()) return 0; if (fgraph_sleep_time) { /* Only need to record the calltime */ ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); } else { ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); if (ftimes) ftimes->sleeptime = current->ftrace_sleeptime; } if (!ftimes) return 0; ftimes->calltime = trace_clock_local(); /* * Stop here if tracing_threshold is set. We only write function return * events to the ring buffer. */ if (tracing_thresh) return 1; preempt_disable_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_read(&data->disabled); if (likely(!disabled)) { trace_ctx = tracing_gen_ctx(); if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { unsigned long retaddr = ftrace_graph_top_ret_addr(current); ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); } else { ret = __trace_graph_entry(tr, trace, trace_ctx); } } preempt_enable_notrace(); return ret;} static void__trace_graph_function(struct trace_array *tr, unsigned long ip, unsigned int trace_ctx){ u64 time = trace_clock_local(); struct ftrace_graph_ent ent = { .func = ip, .depth = 0, }; struct ftrace_graph_ret ret = { .func = ip, .depth = 0, }; __trace_graph_entry(tr, &ent, trace_ctx); __trace_graph_return(tr, &ret, trace_ctx, time, time);} voidtrace_graph_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, unsigned int trace_ctx){ __trace_graph_function(tr, ip, trace_ctx);} void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, unsigned int trace_ctx, u64 calltime, u64 rettime){ struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct ftrace_graph_ret_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, sizeof(*entry), trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); entry->ret = *trace; entry->calltime = calltime; entry->rettime = rettime; trace_buffer_unlock_commit_nostack(buffer, event);} static void handle_nosleeptime(struct ftrace_graph_ret *trace, struct fgraph_times *ftimes, int size){ if (fgraph_sleep_time || size < sizeof(*ftimes)) return; ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime;} void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs){ unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; struct fgraph_times *ftimes; unsigned int trace_ctx; u64 calltime, rettime; long disabled; int size; int cpu; rettime = trace_clock_local(); ftrace_graph_addr_finish(gops, trace); if (*task_var & TRACE_GRAPH_NOTRACE) { *task_var &= ~TRACE_GRAPH_NOTRACE; return; } ftimes = fgraph_retrieve_data(gops->idx, &size); if (!ftimes) return; handle_nosleeptime(trace, ftimes, size); calltime = ftimes->calltime; preempt_disable_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_read(&data->disabled); if (likely(!disabled)) { trace_ctx = tracing_gen_ctx(); __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); } preempt_enable_notrace();} static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs){ struct fgraph_times *ftimes; int size; ftrace_graph_addr_finish(gops, trace); if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); return; } ftimes = fgraph_retrieve_data(gops->idx, &size); if (!ftimes) return; handle_nosleeptime(trace, ftimes, size); if (tracing_thresh && (trace_clock_local() - ftimes->calltime < tracing_thresh)) return; else trace_graph_return(trace, gops, fregs);} static struct fgraph_ops funcgraph_ops = { .entryfunc = &trace_graph_entry, .retfunc = &trace_graph_return,}; int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops){ struct fgraph_ops *gops; gops = kzalloc(sizeof(*gops), GFP_KERNEL); if (!gops) return -ENOMEM; gops->entryfunc = &trace_graph_entry; gops->retfunc = &trace_graph_return; tr->gops = gops; gops->private = tr; fgraph_init_ops(&gops->ops, ops); return 0;} void free_fgraph_ops(struct trace_array *tr){ kfree(tr->gops);} __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops){ tr->gops = &funcgraph_ops; funcgraph_ops.private = tr; fgraph_init_ops(&tr->gops->ops, ops);} static int graph_trace_init(struct trace_array *tr){ int ret; tr->gops->entryfunc = trace_graph_entry; if (tracing_thresh) tr->gops->retfunc = trace_graph_thresh_return; else tr->gops->retfunc = trace_graph_return; /* Make gops functions are visible before we start tracing */ smp_mb(); ret = register_ftrace_graph(tr->gops); if (ret) return ret; tracing_start_cmdline_record(); return 0;} static void graph_trace_reset(struct trace_array *tr){ tracing_stop_cmdline_record(); unregister_ftrace_graph(tr->gops);} static int graph_trace_update_thresh(struct trace_array *tr){ graph_trace_reset(tr); return graph_trace_init(tr);} static int max_bytes_for_cpu; static void print_graph_cpu(struct trace_seq *s, int cpu){ /* * Start with a space character - to make it stand out * to the right a bit when trace output is pasted into * email: */ trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu);} #define TRACE_GRAPH_PROCINFO_LENGTH 14 static void print_graph_proc(struct trace_seq *s, pid_t pid){ char comm[TASK_COMM_LEN]; /* sign + log10(MAX_INT) + '\0' */ char pid_str[11]; int spaces = 0; int len; int i; trace_find_cmdline(pid, comm); comm[7] = '\0'; sprintf(pid_str, "%d", pid); /* 1 stands for the "-" character */ len = strlen(comm) + strlen(pid_str) + 1; if (len < TRACE_GRAPH_PROCINFO_LENGTH) spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; /* First spaces to align center */ for (i = 0; i < spaces / 2; i++) trace_seq_putc(s, ' '); trace_seq_printf(s, "%s-%s", comm, pid_str); /* Last spaces to align center */ for (i = 0; i < spaces - (spaces / 2); i++) trace_seq_putc(s, ' ');} static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry){ trace_seq_putc(s, ' '); trace_print_lat_fmt(s, entry); trace_seq_puts(s, " | ");Lines 1–500 of 1649. Load more
// SPDX-License-Identifier: GPL-2.0 /* * * Function graph tracer. * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> * Mostly borrowed from function tracer which * is Copyright (c) Steven Rostedt <srostedt@redhat.com> * */ #include <linux/uaccess.h> #include <linux/ftrace.h> #include <linux/interrupt.h> #include <linux/slab.h> #include <linux/fs.h> #include "trace.h" #include "trace_output.h" /* When set, irq functions will be ignored */ static int ftrace_graph_skip_irqs; struct fgraph_cpu_data { pid_t last_pid; int depth; int depth_irq; int ignore; unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; }; struct fgraph_data { struct fgraph_cpu_data __percpu *cpu_data; /* Place to preserve last processed entry. */ union { struct ftrace_graph_ent_entry ent; struct fgraph_retaddr_ent_entry rent; } ent; struct ftrace_graph_ret_entry ret; int failed; int cpu; }; #define TRACE_GRAPH_INDENT 2 unsigned int fgraph_max_depth; static struct tracer_opt trace_opts[] = { /* Display overruns? (for self-debug purpose) */ { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, /* Display CPU ? */ { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, /* Display Overhead ? */ { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, /* Display proc name/pid */ { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, /* Display duration of execution */ { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) }, /* Display absolute time of an entry */ { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) }, /* Display interrupts */ { TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) }, /* Display function name after trailing } */ { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, #ifdef CONFIG_FUNCTION_GRAPH_RETVAL /* Display function return value ? */ { TRACER_OPT(funcgraph-retval, TRACE_GRAPH_PRINT_RETVAL) }, /* Display function return value in hexadecimal format ? */ { TRACER_OPT(funcgraph-retval-hex, TRACE_GRAPH_PRINT_RETVAL_HEX) }, #endif #ifdef CONFIG_FUNCTION_GRAPH_RETADDR /* Display function return address ? */ { TRACER_OPT(funcgraph-retaddr, TRACE_GRAPH_PRINT_RETADDR) }, #endif /* Include sleep time (scheduled out) between entry and return */ { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, #ifdef CONFIG_FUNCTION_PROFILER /* Include time within nested functions */ { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, #endif { } /* Empty entry */ }; static struct tracer_flags tracer_flags = { /* Don't display overruns, proc, or tail by default */ .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD | TRACE_GRAPH_PRINT_DURATION | TRACE_GRAPH_PRINT_IRQS | TRACE_GRAPH_SLEEP_TIME | TRACE_GRAPH_GRAPH_TIME, .opts = trace_opts }; static bool tracer_flags_is_set(u32 flags) { return (tracer_flags.val & flags) == flags; } /* * DURATION column is being also used to display IRQ signs, * following values are used by print_graph_irq and others * to fill in space into DURATION column. */ enum { FLAGS_FILL_FULL = 1 << TRACE_GRAPH_PRINT_FILL_SHIFT, FLAGS_FILL_START = 2 << TRACE_GRAPH_PRINT_FILL_SHIFT, FLAGS_FILL_END = 3 << TRACE_GRAPH_PRINT_FILL_SHIFT, }; static void print_graph_duration(struct trace_array *tr, unsigned long long duration, struct trace_seq *s, u32 flags); int __trace_graph_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx) { struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct ftrace_graph_ent_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, sizeof(*entry), trace_ctx); if (!event) return 0; entry = ring_buffer_event_data(event); entry->graph_ent = *trace; trace_buffer_unlock_commit_nostack(buffer, event); return 1; } #ifdef CONFIG_FUNCTION_GRAPH_RETADDR int __trace_graph_retaddr_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx, unsigned long retaddr) { struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct fgraph_retaddr_ent_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RETADDR_ENT, sizeof(*entry), trace_ctx); if (!event) return 0; entry = ring_buffer_event_data(event); entry->graph_ent.func = trace->func; entry->graph_ent.depth = trace->depth; entry->graph_ent.retaddr = retaddr; trace_buffer_unlock_commit_nostack(buffer, event); return 1; } #else int __trace_graph_retaddr_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, unsigned int trace_ctx, unsigned long retaddr) { return 1; } #endif static inline int ftrace_graph_ignore_irqs(void) { if (!ftrace_graph_skip_irqs || trace_recursion_test(TRACE_IRQ_BIT)) return 0; return in_hardirq(); } struct fgraph_times { unsigned long long calltime; unsigned long long sleeptime; /* may be optional! */ }; int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; struct fgraph_times *ftimes; unsigned int trace_ctx; long disabled; int ret = 0; int cpu; if (*task_var & TRACE_GRAPH_NOTRACE) return 0; /* * Do not trace a function if it's filtered by set_graph_notrace. * Make the index of ret stack negative to indicate that it should * ignore further functions. But it needs its own ret stack entry * to recover the original index in order to continue tracing after * returning from the function. */ if (ftrace_graph_notrace_addr(trace->func)) { *task_var |= TRACE_GRAPH_NOTRACE; /* * Need to return 1 to have the return called * that will clear the NOTRACE bit. */ return 1; } if (!ftrace_trace_task(tr)) return 0; if (ftrace_graph_ignore_func(gops, trace)) return 0; if (ftrace_graph_ignore_irqs()) return 0; if (fgraph_sleep_time) { /* Only need to record the calltime */ ftimes = fgraph_reserve_data(gops->idx, sizeof(ftimes->calltime)); } else { ftimes = fgraph_reserve_data(gops->idx, sizeof(*ftimes)); if (ftimes) ftimes->sleeptime = current->ftrace_sleeptime; } if (!ftimes) return 0; ftimes->calltime = trace_clock_local(); /* * Stop here if tracing_threshold is set. We only write function return * events to the ring buffer. */ if (tracing_thresh) return 1; preempt_disable_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_read(&data->disabled); if (likely(!disabled)) { trace_ctx = tracing_gen_ctx(); if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && tracer_flags_is_set(TRACE_GRAPH_PRINT_RETADDR)) { unsigned long retaddr = ftrace_graph_top_ret_addr(current); ret = __trace_graph_retaddr_entry(tr, trace, trace_ctx, retaddr); } else { ret = __trace_graph_entry(tr, trace, trace_ctx); } } preempt_enable_notrace(); return ret; } static void __trace_graph_function(struct trace_array *tr, unsigned long ip, unsigned int trace_ctx) { u64 time = trace_clock_local(); struct ftrace_graph_ent ent = { .func = ip, .depth = 0, }; struct ftrace_graph_ret ret = { .func = ip, .depth = 0, }; __trace_graph_entry(tr, &ent, trace_ctx); __trace_graph_return(tr, &ret, trace_ctx, time, time); } void trace_graph_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, unsigned int trace_ctx) { __trace_graph_function(tr, ip, trace_ctx); } void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, unsigned int trace_ctx, u64 calltime, u64 rettime) { struct ring_buffer_event *event; struct trace_buffer *buffer = tr->array_buffer.buffer; struct ftrace_graph_ret_entry *entry; event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, sizeof(*entry), trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); entry->ret = *trace; entry->calltime = calltime; entry->rettime = rettime; trace_buffer_unlock_commit_nostack(buffer, event); } static void handle_nosleeptime(struct ftrace_graph_ret *trace, struct fgraph_times *ftimes, int size) { if (fgraph_sleep_time || size < sizeof(*ftimes)) return; ftimes->calltime += current->ftrace_sleeptime - ftimes->sleeptime; } void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { unsigned long *task_var = fgraph_get_task_var(gops); struct trace_array *tr = gops->private; struct trace_array_cpu *data; struct fgraph_times *ftimes; unsigned int trace_ctx; u64 calltime, rettime; long disabled; int size; int cpu; rettime = trace_clock_local(); ftrace_graph_addr_finish(gops, trace); if (*task_var & TRACE_GRAPH_NOTRACE) { *task_var &= ~TRACE_GRAPH_NOTRACE; return; } ftimes = fgraph_retrieve_data(gops->idx, &size); if (!ftimes) return; handle_nosleeptime(trace, ftimes, size); calltime = ftimes->calltime; preempt_disable_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_read(&data->disabled); if (likely(!disabled)) { trace_ctx = tracing_gen_ctx(); __trace_graph_return(tr, trace, trace_ctx, calltime, rettime); } preempt_enable_notrace(); } static void trace_graph_thresh_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops, struct ftrace_regs *fregs) { struct fgraph_times *ftimes; int size; ftrace_graph_addr_finish(gops, trace); if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); return; } ftimes = fgraph_retrieve_data(gops->idx, &size); if (!ftimes) return; handle_nosleeptime(trace, ftimes, size); if (tracing_thresh && (trace_clock_local() - ftimes->calltime < tracing_thresh)) return; else trace_graph_return(trace, gops, fregs); } static struct fgraph_ops funcgraph_ops = { .entryfunc = &trace_graph_entry, .retfunc = &trace_graph_return, }; int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) { struct fgraph_ops *gops; gops = kzalloc(sizeof(*gops), GFP_KERNEL); if (!gops) return -ENOMEM; gops->entryfunc = &trace_graph_entry; gops->retfunc = &trace_graph_return; tr->gops = gops; gops->private = tr; fgraph_init_ops(&gops->ops, ops); return 0; } void free_fgraph_ops(struct trace_array *tr) { kfree(tr->gops); } __init void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops) { tr->gops = &funcgraph_ops; funcgraph_ops.private = tr; fgraph_init_ops(&tr->gops->ops, ops); } static int graph_trace_init(struct trace_array *tr) { int ret; tr->gops->entryfunc = trace_graph_entry; if (tracing_thresh) tr->gops->retfunc = trace_graph_thresh_return; else tr->gops->retfunc = trace_graph_return; /* Make gops functions are visible before we start tracing */ smp_mb(); ret = register_ftrace_graph(tr->gops); if (ret) return ret; tracing_start_cmdline_record(); return 0; } static void graph_trace_reset(struct trace_array *tr) { tracing_stop_cmdline_record(); unregister_ftrace_graph(tr->gops); } static int graph_trace_update_thresh(struct trace_array *tr) { graph_trace_reset(tr); return graph_trace_init(tr); } static int max_bytes_for_cpu; static void print_graph_cpu(struct trace_seq *s, int cpu) { /* * Start with a space character - to make it stand out * to the right a bit when trace output is pasted into * email: */ trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu); } #define TRACE_GRAPH_PROCINFO_LENGTH 14 static void print_graph_proc(struct trace_seq *s, pid_t pid) { char comm[TASK_COMM_LEN]; /* sign + log10(MAX_INT) + '\0' */ char pid_str[11]; int spaces = 0; int len; int i; trace_find_cmdline(pid, comm); comm[7] = '\0'; sprintf(pid_str, "%d", pid); /* 1 stands for the "-" character */ len = strlen(comm) + strlen(pid_str) + 1; if (len < TRACE_GRAPH_PROCINFO_LENGTH) spaces = TRACE_GRAPH_PROCINFO_LENGTH - len; /* First spaces to align center */ for (i = 0; i < spaces / 2; i++) trace_seq_putc(s, ' '); trace_seq_printf(s, "%s-%s", comm, pid_str); /* Last spaces to align center */ for (i = 0; i < spaces - (spaces / 2); i++) trace_seq_putc(s, ' '); } static void print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry) { trace_seq_putc(s, ' '); trace_print_lat_fmt(s, entry); trace_seq_puts(s, " | ");