/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h>#include <linux/clocksource.h>#include <linux/kallsyms.h>#include <linux/seq_file.h>#include <linux/suspend.h>#include <linux/debugfs.h>#include <linux/hardirq.h>#include <linux/kthread.h>#include <linux/uaccess.h>#include <linux/bsearch.h>#include <linux/module.h>#include <linux/ftrace.h>#include <linux/sysctl.h>#include <linux/slab.h>#include <linux/ctype.h>#include <linux/sort.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/setup.h> #include "trace_output.h"#include "trace_stat.h" #define FTRACE_WARN_ON(cond) \ ({ \ int ___r = cond; \ if (WARN_ON(___r)) \ ftrace_kill(); \ ___r; \ }) #define FTRACE_WARN_ON_ONCE(cond) \ ({ \ int ___r = cond; \ if (WARN_ON_ONCE(___r)) \ ftrace_kill(); \ ___r; \ }) /* hash bits for specific function selection */#define FTRACE_HASH_BITS 7#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)#define FTRACE_HASH_DEFAULT_BITS 10#define FTRACE_HASH_MAX_BITS 12 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL) #ifdef CONFIG_DYNAMIC_FTRACE#define INIT_REGEX_LOCK(opsname) \ .regex_lock = __MUTEX_INITIALIZER(opsname.regex_lock),#else#define INIT_REGEX_LOCK(opsname)#endif static struct ftrace_ops ftrace_list_end __read_mostly = { .func = ftrace_stub, .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,}; /* ftrace_enabled is a method to turn ftrace on or off */int ftrace_enabled __read_mostly;static int last_ftrace_enabled; /* Quick disabling of function tracer. */int function_trace_stop __read_mostly; /* Current function tracing op */struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; /* List for set_ftrace_pid's pids. */LIST_HEAD(ftrace_pids);struct ftrace_pid { struct list_head list; struct pid *pid;}; /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;static struct ftrace_ops global_ops;static struct ftrace_ops control_ops; #if ARCH_SUPPORTS_FTRACE_OPSstatic void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs);#else/* See comment below, where ftrace_ops_list_func is defined */static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)#endif /* * Traverse the ftrace_global_list, invoking all entries. The reason that we * can use rcu_dereference_raw_notrace() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle * concurrent insertions into the ftrace_global_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */#define do_for_each_ftrace_op(op, list) \ op = rcu_dereference_raw_notrace(list); \ do /* * Optimized for just a single item in the list (as that is the normal case). */#define while_for_each_ftrace_op(op) \ while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ unlikely((op) != &ftrace_list_end)) static inline void ftrace_ops_init(struct ftrace_ops *ops){#ifdef CONFIG_DYNAMIC_FTRACE if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { mutex_init(&ops->regex_lock); ops->flags |= FTRACE_OPS_FL_INITIALIZED; }#endif} /** * ftrace_nr_registered_ops - return number of ops registered * * Returns the number of ftrace_ops registered and tracing functions */int ftrace_nr_registered_ops(void){ struct ftrace_ops *ops; int cnt = 0; mutex_lock(&ftrace_lock); for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) cnt++; mutex_unlock(&ftrace_lock); return cnt;} static voidftrace_global_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs){ int bit; bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX); if (bit < 0) return; do_for_each_ftrace_op(op, ftrace_global_list) { op->func(ip, parent_ip, op, regs); } while_for_each_ftrace_op(op); trace_clear_recursion(bit);} static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs){ if (!test_tsk_trace_trace(current)) return; ftrace_pid_function(ip, parent_ip, op, regs);} static void set_ftrace_pid_function(ftrace_func_t func){ /* do not set ftrace_pid_function to itself! */ if (func != ftrace_pid_func) ftrace_pid_function = func;} /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing. There may be lag */void clear_ftrace_function(void){ ftrace_trace_function = ftrace_stub; ftrace_pid_function = ftrace_stub;} static void control_ops_disable_all(struct ftrace_ops *ops){ int cpu; for_each_possible_cpu(cpu) *per_cpu_ptr(ops->disabled, cpu) = 1;} static int control_ops_alloc(struct ftrace_ops *ops){ int __percpu *disabled; disabled = alloc_percpu(int); if (!disabled) return -ENOMEM; ops->disabled = disabled; control_ops_disable_all(ops); return 0;} static void control_ops_free(struct ftrace_ops *ops){ free_percpu(ops->disabled);} static void update_global_ops(void){ ftrace_func_t func; /* * If there's only one function registered, then call that * function directly. Otherwise, we need to iterate over the * registered callers. */ if (ftrace_global_list == &ftrace_list_end || ftrace_global_list->next == &ftrace_list_end) { func = ftrace_global_list->func; /* * As we are calling the function directly. * If it does not have recursion protection, * the function_trace_op needs to be updated * accordingly. */ if (ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE; else global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE; } else { func = ftrace_global_list_func; /* The list has its own recursion protection. */ global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE; } /* If we filter on pids, update to use the pid function */ if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } global_ops.func = func;} static void update_ftrace_function(void){ ftrace_func_t func; update_global_ops(); /* * If we are at the end of the list and this ops is * recursion safe and not dynamic and the arch supports passing ops, * then have the mcount trampoline call the function directly. */ if (ftrace_ops_list == &ftrace_list_end || (ftrace_ops_list->next == &ftrace_list_end && !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) && (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) && !FTRACE_FORCE_LIST_FUNC)) { /* Set the ftrace_ops that the arch callback uses */ if (ftrace_ops_list == &global_ops) function_trace_op = ftrace_global_list; else function_trace_op = ftrace_ops_list; func = ftrace_ops_list->func; } else { /* Just use the default ftrace_ops */ function_trace_op = &ftrace_list_end; func = ftrace_ops_list_func; } ftrace_trace_function = func;} static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops){ ops->next = *list; /* * We are entering ops into the list but another * CPU might be walking that list. We need to make sure * the ops->next pointer is valid before another CPU sees * the ops pointer included into the list. */ rcu_assign_pointer(*list, ops);} static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops){ struct ftrace_ops **p; /* * If we are removing the last function, then simply point * to the ftrace_stub. */ if (*list == ops && ops->next == &ftrace_list_end) { *list = &ftrace_list_end; return 0; } for (p = list; *p != &ftrace_list_end; p = &(*p)->next) if (*p == ops) break; if (*p != ops) return -1; *p = (*p)->next; return 0;} static void add_ftrace_list_ops(struct ftrace_ops **list, struct ftrace_ops *main_ops, struct ftrace_ops *ops){ int first = *list == &ftrace_list_end; add_ftrace_ops(list, ops); if (first) add_ftrace_ops(&ftrace_ops_list, main_ops);} static int remove_ftrace_list_ops(struct ftrace_ops **list, struct ftrace_ops *main_ops, struct ftrace_ops *ops){ int ret = remove_ftrace_ops(list, ops); if (!ret && *list == &ftrace_list_end) ret = remove_ftrace_ops(&ftrace_ops_list, main_ops); return ret;} static int __register_ftrace_function(struct ftrace_ops *ops){ if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) return -EBUSY; /* We don't support both control and global flags set. */ if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK) return -EINVAL; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS /* * If the ftrace_ops specifies SAVE_REGS, then it only can be used * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. */ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) return -EINVAL; if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) ops->flags |= FTRACE_OPS_FL_SAVE_REGS;#endif if (!core_kernel_data((unsigned long)ops)) ops->flags |= FTRACE_OPS_FL_DYNAMIC; if (ops->flags & FTRACE_OPS_FL_GLOBAL) { add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops); ops->flags |= FTRACE_OPS_FL_ENABLED; } else if (ops->flags & FTRACE_OPS_FL_CONTROL) { if (control_ops_alloc(ops)) return -ENOMEM; add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops); } else add_ftrace_ops(&ftrace_ops_list, ops); if (ftrace_enabled) update_ftrace_function(); return 0;} static void ftrace_sync(struct work_struct *work){ /* * This function is just a stub to implement a hard force * of synchronize_sched(). This requires synchronizing * tasks even in userspace and idle. * * Yes, function tracing is rude. */} static int __unregister_ftrace_function(struct ftrace_ops *ops){ int ret; if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) return -EBUSY; if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; if (ops->flags & FTRACE_OPS_FL_GLOBAL) { ret = remove_ftrace_list_ops(&ftrace_global_list, &global_ops, ops); if (!ret) ops->flags &= ~FTRACE_OPS_FL_ENABLED; } else if (ops->flags & FTRACE_OPS_FL_CONTROL) { ret = remove_ftrace_list_ops(&ftrace_control_list, &control_ops, ops); if (!ret) { /* * The ftrace_ops is now removed from the list, * so there'll be no new users. We must ensure * all current users are done before we free * the control data. * Note synchronize_sched() is not enough, as we * use preempt_disable() to do RCU, but the function * tracer can be called where RCU is not active * (before user_exit()). */ schedule_on_each_cpu(ftrace_sync); control_ops_free(ops); } } else ret = remove_ftrace_ops(&ftrace_ops_list, ops); if (ret < 0) return ret; if (ftrace_enabled) update_ftrace_function(); /* * Dynamic ops may be freed, we must make sure that all * callers are done before leaving this function. * * Again, normal synchronize_sched() is not good enough. * We need to do a hard force of sched synchronization. */ if (ops->flags & FTRACE_OPS_FL_DYNAMIC) schedule_on_each_cpu(ftrace_sync); return 0;} static void ftrace_update_pid_func(void){ /* Only do something if we are tracing something */ if (ftrace_trace_function == ftrace_stub) return; update_ftrace_function();} #ifdef CONFIG_FUNCTION_PROFILERstruct ftrace_profile { struct hlist_node node; unsigned long ip; unsigned long counter;#ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared;#endif}; struct ftrace_profile_page { struct ftrace_profile_page *next;Lines 1–500 of 5134. Load more
/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * * Copyright (C) 2004-2006 Ingo Molnar * Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h> #include <linux/clocksource.h> #include <linux/kallsyms.h> #include <linux/seq_file.h> #include <linux/suspend.h> #include <linux/debugfs.h> #include <linux/hardirq.h> #include <linux/kthread.h> #include <linux/uaccess.h> #include <linux/bsearch.h> #include <linux/module.h> #include <linux/ftrace.h> #include <linux/sysctl.h> #include <linux/slab.h> #include <linux/ctype.h> #include <linux/sort.h> #include <linux/list.h> #include <linux/hash.h> #include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/setup.h> #include "trace_output.h" #include "trace_stat.h" #define FTRACE_WARN_ON(cond) \ ({ \ int ___r = cond; \ if (WARN_ON(___r)) \ ftrace_kill(); \ ___r; \ }) #define FTRACE_WARN_ON_ONCE(cond) \ ({ \ int ___r = cond; \ if (WARN_ON_ONCE(___r)) \ ftrace_kill(); \ ___r; \ }) /* hash bits for specific function selection */ #define FTRACE_HASH_BITS 7 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) #define FTRACE_HASH_DEFAULT_BITS 10 #define FTRACE_HASH_MAX_BITS 12 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL) #ifdef CONFIG_DYNAMIC_FTRACE #define INIT_REGEX_LOCK(opsname) \ .regex_lock = __MUTEX_INITIALIZER(opsname.regex_lock), #else #define INIT_REGEX_LOCK(opsname) #endif static struct ftrace_ops ftrace_list_end __read_mostly = { .func = ftrace_stub, .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, }; /* ftrace_enabled is a method to turn ftrace on or off */ int ftrace_enabled __read_mostly; static int last_ftrace_enabled; /* Quick disabling of function tracer. */ int function_trace_stop __read_mostly; /* Current function tracing op */ struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; /* List for set_ftrace_pid's pids. */ LIST_HEAD(ftrace_pids); struct ftrace_pid { struct list_head list; struct pid *pid; }; /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */ static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end; static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end; static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end; ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; static struct ftrace_ops global_ops; static struct ftrace_ops control_ops; #if ARCH_SUPPORTS_FTRACE_OPS static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs); #else /* See comment below, where ftrace_ops_list_func is defined */ static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) #endif /* * Traverse the ftrace_global_list, invoking all entries. The reason that we * can use rcu_dereference_raw_notrace() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle * concurrent insertions into the ftrace_global_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */ #define do_for_each_ftrace_op(op, list) \ op = rcu_dereference_raw_notrace(list); \ do /* * Optimized for just a single item in the list (as that is the normal case). */ #define while_for_each_ftrace_op(op) \ while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ unlikely((op) != &ftrace_list_end)) static inline void ftrace_ops_init(struct ftrace_ops *ops) { #ifdef CONFIG_DYNAMIC_FTRACE if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { mutex_init(&ops->regex_lock); ops->flags |= FTRACE_OPS_FL_INITIALIZED; } #endif } /** * ftrace_nr_registered_ops - return number of ops registered * * Returns the number of ftrace_ops registered and tracing functions */ int ftrace_nr_registered_ops(void) { struct ftrace_ops *ops; int cnt = 0; mutex_lock(&ftrace_lock); for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) cnt++; mutex_unlock(&ftrace_lock); return cnt; } static void ftrace_global_list_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs) { int bit; bit = trace_test_and_set_recursion(TRACE_GLOBAL_START, TRACE_GLOBAL_MAX); if (bit < 0) return; do_for_each_ftrace_op(op, ftrace_global_list) { op->func(ip, parent_ip, op, regs); } while_for_each_ftrace_op(op); trace_clear_recursion(bit); } static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *regs) { if (!test_tsk_trace_trace(current)) return; ftrace_pid_function(ip, parent_ip, op, regs); } static void set_ftrace_pid_function(ftrace_func_t func) { /* do not set ftrace_pid_function to itself! */ if (func != ftrace_pid_func) ftrace_pid_function = func; } /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing. There may be lag */ void clear_ftrace_function(void) { ftrace_trace_function = ftrace_stub; ftrace_pid_function = ftrace_stub; } static void control_ops_disable_all(struct ftrace_ops *ops) { int cpu; for_each_possible_cpu(cpu) *per_cpu_ptr(ops->disabled, cpu) = 1; } static int control_ops_alloc(struct ftrace_ops *ops) { int __percpu *disabled; disabled = alloc_percpu(int); if (!disabled) return -ENOMEM; ops->disabled = disabled; control_ops_disable_all(ops); return 0; } static void control_ops_free(struct ftrace_ops *ops) { free_percpu(ops->disabled); } static void update_global_ops(void) { ftrace_func_t func; /* * If there's only one function registered, then call that * function directly. Otherwise, we need to iterate over the * registered callers. */ if (ftrace_global_list == &ftrace_list_end || ftrace_global_list->next == &ftrace_list_end) { func = ftrace_global_list->func; /* * As we are calling the function directly. * If it does not have recursion protection, * the function_trace_op needs to be updated * accordingly. */ if (ftrace_global_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE; else global_ops.flags &= ~FTRACE_OPS_FL_RECURSION_SAFE; } else { func = ftrace_global_list_func; /* The list has its own recursion protection. */ global_ops.flags |= FTRACE_OPS_FL_RECURSION_SAFE; } /* If we filter on pids, update to use the pid function */ if (!list_empty(&ftrace_pids)) { set_ftrace_pid_function(func); func = ftrace_pid_func; } global_ops.func = func; } static void update_ftrace_function(void) { ftrace_func_t func; update_global_ops(); /* * If we are at the end of the list and this ops is * recursion safe and not dynamic and the arch supports passing ops, * then have the mcount trampoline call the function directly. */ if (ftrace_ops_list == &ftrace_list_end || (ftrace_ops_list->next == &ftrace_list_end && !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC) && (ftrace_ops_list->flags & FTRACE_OPS_FL_RECURSION_SAFE) && !FTRACE_FORCE_LIST_FUNC)) { /* Set the ftrace_ops that the arch callback uses */ if (ftrace_ops_list == &global_ops) function_trace_op = ftrace_global_list; else function_trace_op = ftrace_ops_list; func = ftrace_ops_list->func; } else { /* Just use the default ftrace_ops */ function_trace_op = &ftrace_list_end; func = ftrace_ops_list_func; } ftrace_trace_function = func; } static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) { ops->next = *list; /* * We are entering ops into the list but another * CPU might be walking that list. We need to make sure * the ops->next pointer is valid before another CPU sees * the ops pointer included into the list. */ rcu_assign_pointer(*list, ops); } static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops) { struct ftrace_ops **p; /* * If we are removing the last function, then simply point * to the ftrace_stub. */ if (*list == ops && ops->next == &ftrace_list_end) { *list = &ftrace_list_end; return 0; } for (p = list; *p != &ftrace_list_end; p = &(*p)->next) if (*p == ops) break; if (*p != ops) return -1; *p = (*p)->next; return 0; } static void add_ftrace_list_ops(struct ftrace_ops **list, struct ftrace_ops *main_ops, struct ftrace_ops *ops) { int first = *list == &ftrace_list_end; add_ftrace_ops(list, ops); if (first) add_ftrace_ops(&ftrace_ops_list, main_ops); } static int remove_ftrace_list_ops(struct ftrace_ops **list, struct ftrace_ops *main_ops, struct ftrace_ops *ops) { int ret = remove_ftrace_ops(list, ops); if (!ret && *list == &ftrace_list_end) ret = remove_ftrace_ops(&ftrace_ops_list, main_ops); return ret; } static int __register_ftrace_function(struct ftrace_ops *ops) { if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) return -EBUSY; /* We don't support both control and global flags set. */ if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK) return -EINVAL; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS /* * If the ftrace_ops specifies SAVE_REGS, then it only can be used * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. */ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) return -EINVAL; if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) ops->flags |= FTRACE_OPS_FL_SAVE_REGS; #endif if (!core_kernel_data((unsigned long)ops)) ops->flags |= FTRACE_OPS_FL_DYNAMIC; if (ops->flags & FTRACE_OPS_FL_GLOBAL) { add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops); ops->flags |= FTRACE_OPS_FL_ENABLED; } else if (ops->flags & FTRACE_OPS_FL_CONTROL) { if (control_ops_alloc(ops)) return -ENOMEM; add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops); } else add_ftrace_ops(&ftrace_ops_list, ops); if (ftrace_enabled) update_ftrace_function(); return 0; } static void ftrace_sync(struct work_struct *work) { /* * This function is just a stub to implement a hard force * of synchronize_sched(). This requires synchronizing * tasks even in userspace and idle. * * Yes, function tracing is rude. */ } static int __unregister_ftrace_function(struct ftrace_ops *ops) { int ret; if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) return -EBUSY; if (FTRACE_WARN_ON(ops == &global_ops)) return -EINVAL; if (ops->flags & FTRACE_OPS_FL_GLOBAL) { ret = remove_ftrace_list_ops(&ftrace_global_list, &global_ops, ops); if (!ret) ops->flags &= ~FTRACE_OPS_FL_ENABLED; } else if (ops->flags & FTRACE_OPS_FL_CONTROL) { ret = remove_ftrace_list_ops(&ftrace_control_list, &control_ops, ops); if (!ret) { /* * The ftrace_ops is now removed from the list, * so there'll be no new users. We must ensure * all current users are done before we free * the control data. * Note synchronize_sched() is not enough, as we * use preempt_disable() to do RCU, but the function * tracer can be called where RCU is not active * (before user_exit()). */ schedule_on_each_cpu(ftrace_sync); control_ops_free(ops); } } else ret = remove_ftrace_ops(&ftrace_ops_list, ops); if (ret < 0) return ret; if (ftrace_enabled) update_ftrace_function(); /* * Dynamic ops may be freed, we must make sure that all * callers are done before leaving this function. * * Again, normal synchronize_sched() is not good enough. * We need to do a hard force of sched synchronization. */ if (ops->flags & FTRACE_OPS_FL_DYNAMIC) schedule_on_each_cpu(ftrace_sync); return 0; } static void ftrace_update_pid_func(void) { /* Only do something if we are tracing something */ if (ftrace_trace_function == ftrace_stub) return; update_ftrace_function(); } #ifdef CONFIG_FUNCTION_PROFILER struct ftrace_profile { struct hlist_node node; unsigned long ip; unsigned long counter; #ifdef CONFIG_FUNCTION_GRAPH_TRACER unsigned long long time; unsigned long long time_squared; #endif }; struct ftrace_profile_page { struct ftrace_profile_page *next;