/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 William Lee Irwin III */ #include <linux/stop_machine.h>#include <linux/clocksource.h>#include <linux/kallsyms.h>#include <linux/seq_file.h>#include <linux/suspend.h>#include <linux/debugfs.h>#include <linux/hardirq.h>#include <linux/kthread.h>#include <linux/uaccess.h>#include <linux/ftrace.h>#include <linux/sysctl.h>#include <linux/slab.h>#include <linux/ctype.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/ftrace.h>#include <asm/setup.h> #include "trace_output.h"#include "trace_stat.h" #define FTRACE_WARN_ON(cond) \ do { \ if (WARN_ON(cond)) \ ftrace_kill(); \ } while (0) #define FTRACE_WARN_ON_ONCE(cond) \ do { \ if (WARN_ON_ONCE(cond)) \ ftrace_kill(); \ } while (0) /* hash bits for specific function selection */#define FTRACE_HASH_BITS 7#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) /* ftrace_enabled is a method to turn ftrace on or off */int ftrace_enabled __read_mostly;static int last_ftrace_enabled; /* Quick disabling of function tracer. */int function_trace_stop; /* List for set_ftrace_pid's pids. */LIST_HEAD(ftrace_pids);struct ftrace_pid { struct list_head list; struct pid *pid;}; /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops ftrace_list_end __read_mostly ={ .func = ftrace_stub,}; static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; /* * Traverse the ftrace_list, invoking all entries. The reason that we * can use rcu_dereference_raw() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism. The rcu_dereference_raw() calls are needed to handle * concurrent insertions into the ftrace_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */static void ftrace_list_func(unsigned long ip, unsigned long parent_ip){ struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ while (op != &ftrace_list_end) { op->func(ip, parent_ip); op = rcu_dereference_raw(op->next); /*see above*/ };} static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip){ if (!test_tsk_trace_trace(current)) return; ftrace_pid_function(ip, parent_ip);} static void set_ftrace_pid_function(ftrace_func_t func){ /* do not set ftrace_pid_function to itself! */ if (func != ftrace_pid_func) ftrace_pid_function = func;} /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing. There may be lag */void clear_ftrace_function(void){ ftrace_trace_function = ftrace_stub; __ftrace_trace_function = ftrace_stub; ftrace_pid_function = ftrace_stub;} #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST/* * For those archs that do not test ftrace_trace_stop in their * mcount call site, we need to do it from C. */static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip){ if (function_trace_stop) return; __ftrace_trace_function(ip, parent_ip);}#endif static int __register_ftrace_function(struct ftrace_ops *ops){ ops->next = ftrace_list; /* * We are entering ops into the ftrace_list but another * CPU might be walking that list. We need to make sure * the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ rcu_assign_pointer(ftrace_list, ops); if (ftrace_enabled) { ftrace_func_t func; if (ops->next == &ftrace_list_end) func = ops->func; else func = ftrace_list_func; if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } /* * For one func, simply call it directly. * For more than one func, call the chain. */#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func;#else __ftrace_trace_function = func; ftrace_trace_function = ftrace_test_stop_func;#endif } return 0;} static int __unregister_ftrace_function(struct ftrace_ops *ops){ struct ftrace_ops **p; /* * If we are removing the last function, then simply point * to the ftrace_stub. */ if (ftrace_list == ops && ops->next == &ftrace_list_end) { ftrace_trace_function = ftrace_stub; ftrace_list = &ftrace_list_end; return 0; } for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) if (*p == ops) break; if (*p != ops) return -1; *p = (*p)->next; if (ftrace_enabled) { /* If we only have one func left, then call that directly */ if (ftrace_list->next == &ftrace_list_end) { ftrace_func_t func = ftrace_list->func; if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; }#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func;#else __ftrace_trace_function = func;#endif } } return 0;} static void ftrace_update_pid_func(void){ ftrace_func_t func; if (ftrace_trace_function == ftrace_stub) return; #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST func = ftrace_trace_function;#else func = __ftrace_trace_function;#endif if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } else { if (func == ftrace_pid_func) func = ftrace_pid_function; } #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func;#else __ftrace_trace_function = func;#endif} #ifdef CONFIG_FUNCTION_PROFILERstruct ftrace_profile { struct hlist_node node; unsigned long ip; unsigned long counter;#ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared;#endif}; struct ftrace_profile_page { struct ftrace_profile_page *next; unsigned long index; struct ftrace_profile records[];}; struct ftrace_profile_stat { atomic_t disabled; struct hlist_head *hash; struct ftrace_profile_page *pages; struct ftrace_profile_page *start; struct tracer_stat stat;}; #define PROFILE_RECORDS_SIZE \ (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) #define PROFILES_PER_PAGE \ (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) static int ftrace_profile_bits __read_mostly;static int ftrace_profile_enabled __read_mostly; /* ftrace_profile_lock - synchronize the enable and disable of the profiler */static DEFINE_MUTEX(ftrace_profile_lock); static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ static void *function_stat_next(void *v, int idx){ struct ftrace_profile *rec = v; struct ftrace_profile_page *pg; pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); again: if (idx != 0) rec++; if ((void *)rec >= (void *)&pg->records[pg->index]) { pg = pg->next; if (!pg) return NULL; rec = &pg->records[0]; if (!rec->counter) goto again; } return rec;} static void *function_stat_start(struct tracer_stat *trace){ struct ftrace_profile_stat *stat = container_of(trace, struct ftrace_profile_stat, stat); if (!stat || !stat->start) return NULL; return function_stat_next(&stat->start->records[0], 0);} #ifdef CONFIG_FUNCTION_GRAPH_TRACER/* function graph compares on total time */static int function_stat_cmp(void *p1, void *p2){ struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; if (a->time < b->time) return -1; if (a->time > b->time) return 1; else return 0;}#else/* not function graph compares against hits */static int function_stat_cmp(void *p1, void *p2){ struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; if (a->counter < b->counter) return -1; if (a->counter > b->counter) return 1; else return 0;}#endif static int function_stat_headers(struct seq_file *m){#ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_printf(m, " Function " "Hit Time Avg s^2\n" " -------- " "--- ---- --- ---\n");#else seq_printf(m, " Function Hit\n" " -------- ---\n");#endif return 0;} static int function_stat_show(struct seq_file *m, void *v){ struct ftrace_profile *rec = v; char str[KSYM_SYMBOL_LEN]; int ret = 0;#ifdef CONFIG_FUNCTION_GRAPH_TRACER static struct trace_seq s; unsigned long long avg; unsigned long long stddev;#endif mutex_lock(&ftrace_profile_lock); /* we raced with function_profile_reset() */ if (unlikely(rec->counter == 0)) { ret = -EBUSY; goto out; } kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); seq_printf(m, " %-30.30s %10lu", str, rec->counter); #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_printf(m, " "); avg = rec->time; do_div(avg, rec->counter); /* Sample standard deviation (s^2) */ if (rec->counter <= 1) stddev = 0; else { stddev = rec->time_squared - rec->counter * avg * avg; /* * Divide only 1000 for ns^2 -> us^2 conversion. * trace_print_graph_duration will divide 1000 again. */ do_div(stddev, (rec->counter - 1) * 1000); } trace_seq_init(&s); trace_print_graph_duration(rec->time, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(avg, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(stddev, &s); trace_print_seq(m, &s);#endif seq_putc(m, '\n');out: mutex_unlock(&ftrace_profile_lock); return ret;} static void ftrace_profile_reset(struct ftrace_profile_stat *stat){ struct ftrace_profile_page *pg; pg = stat->pages = stat->start; while (pg) { memset(pg->records, 0, PROFILE_RECORDS_SIZE); pg->index = 0; pg = pg->next; } memset(stat->hash, 0, FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));} int ftrace_profile_pages_init(struct ftrace_profile_stat *stat){ struct ftrace_profile_page *pg; int functions; int pages; int i; /* If we already allocated, do nothing */ if (stat->pages) return 0; stat->pages = (void *)get_zeroed_page(GFP_KERNEL); if (!stat->pages) return -ENOMEM; #ifdef CONFIG_DYNAMIC_FTRACE functions = ftrace_update_tot_cnt;#else /* * We do not know the number of functions that exist because * dynamic tracing is what counts them. With past experience * we have around 20K functions. That should be more than enough. * It is highly unlikely we will execute every function in * the kernel. */ functions = 20000;#endif pg = stat->start = stat->pages; pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); for (i = 0; i < pages; i++) { pg->next = (void *)get_zeroed_page(GFP_KERNEL); if (!pg->next) goto out_free; pg = pg->next; } return 0; out_free: pg = stat->start; while (pg) { unsigned long tmp = (unsigned long)pg; pg = pg->next; free_page(tmp); } free_page((unsigned long)stat->pages); stat->pages = NULL;Lines 1–500 of 3497. Load more
/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 William Lee Irwin III */ #include <linux/stop_machine.h> #include <linux/clocksource.h> #include <linux/kallsyms.h> #include <linux/seq_file.h> #include <linux/suspend.h> #include <linux/debugfs.h> #include <linux/hardirq.h> #include <linux/kthread.h> #include <linux/uaccess.h> #include <linux/ftrace.h> #include <linux/sysctl.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/list.h> #include <linux/hash.h> #include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/ftrace.h> #include <asm/setup.h> #include "trace_output.h" #include "trace_stat.h" #define FTRACE_WARN_ON(cond) \ do { \ if (WARN_ON(cond)) \ ftrace_kill(); \ } while (0) #define FTRACE_WARN_ON_ONCE(cond) \ do { \ if (WARN_ON_ONCE(cond)) \ ftrace_kill(); \ } while (0) /* hash bits for specific function selection */ #define FTRACE_HASH_BITS 7 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) /* ftrace_enabled is a method to turn ftrace on or off */ int ftrace_enabled __read_mostly; static int last_ftrace_enabled; /* Quick disabling of function tracer. */ int function_trace_stop; /* List for set_ftrace_pid's pids. */ LIST_HEAD(ftrace_pids); struct ftrace_pid { struct list_head list; struct pid *pid; }; /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */ static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops ftrace_list_end __read_mostly = { .func = ftrace_stub, }; static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; /* * Traverse the ftrace_list, invoking all entries. The reason that we * can use rcu_dereference_raw() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism. The rcu_dereference_raw() calls are needed to handle * concurrent insertions into the ftrace_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */ static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) { struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ while (op != &ftrace_list_end) { op->func(ip, parent_ip); op = rcu_dereference_raw(op->next); /*see above*/ }; } static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip) { if (!test_tsk_trace_trace(current)) return; ftrace_pid_function(ip, parent_ip); } static void set_ftrace_pid_function(ftrace_func_t func) { /* do not set ftrace_pid_function to itself! */ if (func != ftrace_pid_func) ftrace_pid_function = func; } /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing. There may be lag */ void clear_ftrace_function(void) { ftrace_trace_function = ftrace_stub; __ftrace_trace_function = ftrace_stub; ftrace_pid_function = ftrace_stub; } #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST /* * For those archs that do not test ftrace_trace_stop in their * mcount call site, we need to do it from C. */ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) { if (function_trace_stop) return; __ftrace_trace_function(ip, parent_ip); } #endif static int __register_ftrace_function(struct ftrace_ops *ops) { ops->next = ftrace_list; /* * We are entering ops into the ftrace_list but another * CPU might be walking that list. We need to make sure * the ops->next pointer is valid before another CPU sees * the ops pointer included into the ftrace_list. */ rcu_assign_pointer(ftrace_list, ops); if (ftrace_enabled) { ftrace_func_t func; if (ops->next == &ftrace_list_end) func = ops->func; else func = ftrace_list_func; if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } /* * For one func, simply call it directly. * For more than one func, call the chain. */ #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func; #else __ftrace_trace_function = func; ftrace_trace_function = ftrace_test_stop_func; #endif } return 0; } static int __unregister_ftrace_function(struct ftrace_ops *ops) { struct ftrace_ops **p; /* * If we are removing the last function, then simply point * to the ftrace_stub. */ if (ftrace_list == ops && ops->next == &ftrace_list_end) { ftrace_trace_function = ftrace_stub; ftrace_list = &ftrace_list_end; return 0; } for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) if (*p == ops) break; if (*p != ops) return -1; *p = (*p)->next; if (ftrace_enabled) { /* If we only have one func left, then call that directly */ if (ftrace_list->next == &ftrace_list_end) { ftrace_func_t func = ftrace_list->func; if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func; #else __ftrace_trace_function = func; #endif } } return 0; } static void ftrace_update_pid_func(void) { ftrace_func_t func; if (ftrace_trace_function == ftrace_stub) return; #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST func = ftrace_trace_function; #else func = __ftrace_trace_function; #endif if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } else { if (func == ftrace_pid_func) func = ftrace_pid_function; } #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST ftrace_trace_function = func; #else __ftrace_trace_function = func; #endif } #ifdef CONFIG_FUNCTION_PROFILER struct ftrace_profile { struct hlist_node node; unsigned long ip; unsigned long counter; #ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared; #endif }; struct ftrace_profile_page { struct ftrace_profile_page *next; unsigned long index; struct ftrace_profile records[]; }; struct ftrace_profile_stat { atomic_t disabled; struct hlist_head *hash; struct ftrace_profile_page *pages; struct ftrace_profile_page *start; struct tracer_stat stat; }; #define PROFILE_RECORDS_SIZE \ (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) #define PROFILES_PER_PAGE \ (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) static int ftrace_profile_bits __read_mostly; static int ftrace_profile_enabled __read_mostly; /* ftrace_profile_lock - synchronize the enable and disable of the profiler */ static DEFINE_MUTEX(ftrace_profile_lock); static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ static void * function_stat_next(void *v, int idx) { struct ftrace_profile *rec = v; struct ftrace_profile_page *pg; pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); again: if (idx != 0) rec++; if ((void *)rec >= (void *)&pg->records[pg->index]) { pg = pg->next; if (!pg) return NULL; rec = &pg->records[0]; if (!rec->counter) goto again; } return rec; } static void *function_stat_start(struct tracer_stat *trace) { struct ftrace_profile_stat *stat = container_of(trace, struct ftrace_profile_stat, stat); if (!stat || !stat->start) return NULL; return function_stat_next(&stat->start->records[0], 0); } #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* function graph compares on total time */ static int function_stat_cmp(void *p1, void *p2) { struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; if (a->time < b->time) return -1; if (a->time > b->time) return 1; else return 0; } #else /* not function graph compares against hits */ static int function_stat_cmp(void *p1, void *p2) { struct ftrace_profile *a = p1; struct ftrace_profile *b = p2; if (a->counter < b->counter) return -1; if (a->counter > b->counter) return 1; else return 0; } #endif static int function_stat_headers(struct seq_file *m) { #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_printf(m, " Function " "Hit Time Avg s^2\n" " -------- " "--- ---- --- ---\n"); #else seq_printf(m, " Function Hit\n" " -------- ---\n"); #endif return 0; } static int function_stat_show(struct seq_file *m, void *v) { struct ftrace_profile *rec = v; char str[KSYM_SYMBOL_LEN]; int ret = 0; #ifdef CONFIG_FUNCTION_GRAPH_TRACER static struct trace_seq s; unsigned long long avg; unsigned long long stddev; #endif mutex_lock(&ftrace_profile_lock); /* we raced with function_profile_reset() */ if (unlikely(rec->counter == 0)) { ret = -EBUSY; goto out; } kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); seq_printf(m, " %-30.30s %10lu", str, rec->counter); #ifdef CONFIG_FUNCTION_GRAPH_TRACER seq_printf(m, " "); avg = rec->time; do_div(avg, rec->counter); /* Sample standard deviation (s^2) */ if (rec->counter <= 1) stddev = 0; else { stddev = rec->time_squared - rec->counter * avg * avg; /* * Divide only 1000 for ns^2 -> us^2 conversion. * trace_print_graph_duration will divide 1000 again. */ do_div(stddev, (rec->counter - 1) * 1000); } trace_seq_init(&s); trace_print_graph_duration(rec->time, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(avg, &s); trace_seq_puts(&s, " "); trace_print_graph_duration(stddev, &s); trace_print_seq(m, &s); #endif seq_putc(m, '\n'); out: mutex_unlock(&ftrace_profile_lock); return ret; } static void ftrace_profile_reset(struct ftrace_profile_stat *stat) { struct ftrace_profile_page *pg; pg = stat->pages = stat->start; while (pg) { memset(pg->records, 0, PROFILE_RECORDS_SIZE); pg->index = 0; pg = pg->next; } memset(stat->hash, 0, FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); } int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) { struct ftrace_profile_page *pg; int functions; int pages; int i; /* If we already allocated, do nothing */ if (stat->pages) return 0; stat->pages = (void *)get_zeroed_page(GFP_KERNEL); if (!stat->pages) return -ENOMEM; #ifdef CONFIG_DYNAMIC_FTRACE functions = ftrace_update_tot_cnt; #else /* * We do not know the number of functions that exist because * dynamic tracing is what counts them. With past experience * we have around 20K functions. That should be more than enough. * It is highly unlikely we will execute every function in * the kernel. */ functions = 20000; #endif pg = stat->start = stat->pages; pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); for (i = 0; i < pages; i++) { pg->next = (void *)get_zeroed_page(GFP_KERNEL); if (!pg->next) goto out_free; pg = pg->next; } return 0; out_free: pg = stat->start; while (pg) { unsigned long tmp = (unsigned long)pg; pg = pg->next; free_page(tmp); } free_page((unsigned long)stat->pages); stat->pages = NULL;