/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * *  Copyright (C) 2004-2006 Ingo Molnar *  Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h>#include <linux/clocksource.h>#include <linux/kallsyms.h>#include <linux/seq_file.h>#include <linux/suspend.h>#include <linux/tracefs.h>#include <linux/hardirq.h>#include <linux/kthread.h>#include <linux/uaccess.h>#include <linux/bsearch.h>#include <linux/module.h>#include <linux/ftrace.h>#include <linux/sysctl.h>#include <linux/slab.h>#include <linux/ctype.h>#include <linux/sort.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/setup.h> #include "trace_output.h"#include "trace_stat.h" #define FTRACE_WARN_ON(cond)			\	({					\		int ___r = cond;		\		if (WARN_ON(___r))		\			ftrace_kill();		\		___r;				\	}) #define FTRACE_WARN_ON_ONCE(cond)		\	({					\		int ___r = cond;		\		if (WARN_ON_ONCE(___r))		\			ftrace_kill();		\		___r;				\	}) /* hash bits for specific function selection */#define FTRACE_HASH_BITS 7#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)#define FTRACE_HASH_DEFAULT_BITS 10#define FTRACE_HASH_MAX_BITS 12 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_CONTROL) #ifdef CONFIG_DYNAMIC_FTRACE#define INIT_OPS_HASH(opsname)	\	.func_hash		= &opsname.local_hash,			\	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),#define ASSIGN_OPS_HASH(opsname, val) \	.func_hash		= val, \	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),#else#define INIT_OPS_HASH(opsname)#define ASSIGN_OPS_HASH(opsname, val)#endif static struct ftrace_ops ftrace_list_end __read_mostly = {	.func		= ftrace_stub,	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,	INIT_OPS_HASH(ftrace_list_end)}; /* ftrace_enabled is a method to turn ftrace on or off */int ftrace_enabled __read_mostly;static int last_ftrace_enabled; /* Current function tracing op */struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;/* What to set function_trace_op to */static struct ftrace_ops *set_function_trace_op; /* List for set_ftrace_pid's pids. */LIST_HEAD(ftrace_pids);struct ftrace_pid {	struct list_head list;	struct pid *pid;}; static bool ftrace_pids_enabled(void){	return !list_empty(&ftrace_pids);} static void ftrace_update_trampoline(struct ftrace_ops *ops); /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;static struct ftrace_ops global_ops;static struct ftrace_ops control_ops; static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,				   struct ftrace_ops *op, struct pt_regs *regs); #if ARCH_SUPPORTS_FTRACE_OPSstatic void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,				 struct ftrace_ops *op, struct pt_regs *regs);#else/* See comment below, where ftrace_ops_list_func is defined */static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)#endif /* * Traverse the ftrace_global_list, invoking all entries.  The reason that we * can use rcu_dereference_raw_notrace() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle * concurrent insertions into the ftrace_global_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */#define do_for_each_ftrace_op(op, list)			\	op = rcu_dereference_raw_notrace(list);			\	do /* * Optimized for just a single item in the list (as that is the normal case). */#define while_for_each_ftrace_op(op)				\	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\	       unlikely((op) != &ftrace_list_end)) static inline void ftrace_ops_init(struct ftrace_ops *ops){#ifdef CONFIG_DYNAMIC_FTRACE	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {		mutex_init(&ops->local_hash.regex_lock);		ops->func_hash = &ops->local_hash;		ops->flags |= FTRACE_OPS_FL_INITIALIZED;	}#endif} /** * ftrace_nr_registered_ops - return number of ops registered * * Returns the number of ftrace_ops registered and tracing functions */int ftrace_nr_registered_ops(void){	struct ftrace_ops *ops;	int cnt = 0; 	mutex_lock(&ftrace_lock); 	for (ops = ftrace_ops_list;	     ops != &ftrace_list_end; ops = ops->next)		cnt++; 	mutex_unlock(&ftrace_lock); 	return cnt;} static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,			    struct ftrace_ops *op, struct pt_regs *regs){	if (!test_tsk_trace_trace(current))		return; 	op->saved_func(ip, parent_ip, op, regs);} /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing.  There may be lag */void clear_ftrace_function(void){	ftrace_trace_function = ftrace_stub;} static void control_ops_disable_all(struct ftrace_ops *ops){	int cpu; 	for_each_possible_cpu(cpu)		*per_cpu_ptr(ops->disabled, cpu) = 1;} static int control_ops_alloc(struct ftrace_ops *ops){	int __percpu *disabled; 	disabled = alloc_percpu(int);	if (!disabled)		return -ENOMEM; 	ops->disabled = disabled;	control_ops_disable_all(ops);	return 0;} static void ftrace_sync(struct work_struct *work){	/*	 * This function is just a stub to implement a hard force	 * of synchronize_sched(). This requires synchronizing	 * tasks even in userspace and idle.	 *	 * Yes, function tracing is rude.	 */} static void ftrace_sync_ipi(void *data){	/* Probably not needed, but do it anyway */	smp_rmb();} #ifdef CONFIG_FUNCTION_GRAPH_TRACERstatic void update_function_graph_func(void);#elsestatic inline void update_function_graph_func(void) { }#endif  static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops){	/*	 * If this is a dynamic ops or we force list func,	 * then it needs to call the list anyway.	 */	if (ops->flags & FTRACE_OPS_FL_DYNAMIC || FTRACE_FORCE_LIST_FUNC)		return ftrace_ops_list_func; 	return ftrace_ops_get_func(ops);} static void update_ftrace_function(void){	ftrace_func_t func; 	/*	 * Prepare the ftrace_ops that the arch callback will use.	 * If there's only one ftrace_ops registered, the ftrace_ops_list	 * will point to the ops we want.	 */	set_function_trace_op = ftrace_ops_list; 	/* If there's no ftrace_ops registered, just call the stub function */	if (ftrace_ops_list == &ftrace_list_end) {		func = ftrace_stub; 	/*	 * If we are at the end of the list and this ops is	 * recursion safe and not dynamic and the arch supports passing ops,	 * then have the mcount trampoline call the function directly.	 */	} else if (ftrace_ops_list->next == &ftrace_list_end) {		func = ftrace_ops_get_list_func(ftrace_ops_list); 	} else {		/* Just use the default ftrace_ops */		set_function_trace_op = &ftrace_list_end;		func = ftrace_ops_list_func;	} 	update_function_graph_func(); 	/* If there's no change, then do nothing more here */	if (ftrace_trace_function == func)		return; 	/*	 * If we are using the list function, it doesn't care	 * about the function_trace_ops.	 */	if (func == ftrace_ops_list_func) {		ftrace_trace_function = func;		/*		 * Don't even bother setting function_trace_ops,		 * it would be racy to do so anyway.		 */		return;	} #ifndef CONFIG_DYNAMIC_FTRACE	/*	 * For static tracing, we need to be a bit more careful.	 * The function change takes affect immediately. Thus,	 * we need to coorditate the setting of the function_trace_ops	 * with the setting of the ftrace_trace_function.	 *	 * Set the function to the list ops, which will call the	 * function we want, albeit indirectly, but it handles the	 * ftrace_ops and doesn't depend on function_trace_op.	 */	ftrace_trace_function = ftrace_ops_list_func;	/*	 * Make sure all CPUs see this. Yes this is slow, but static	 * tracing is slow and nasty to have enabled.	 */	schedule_on_each_cpu(ftrace_sync);	/* Now all cpus are using the list ops. */	function_trace_op = set_function_trace_op;	/* Make sure the function_trace_op is visible on all CPUs */	smp_wmb();	/* Nasty way to force a rmb on all cpus */	smp_call_function(ftrace_sync_ipi, NULL, 1);	/* OK, we are all set to update the ftrace_trace_function now! */#endif /* !CONFIG_DYNAMIC_FTRACE */ 	ftrace_trace_function = func;} int using_ftrace_ops_list_func(void){	return ftrace_trace_function == ftrace_ops_list_func;} static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops){	ops->next = *list;	/*	 * We are entering ops into the list but another	 * CPU might be walking that list. We need to make sure	 * the ops->next pointer is valid before another CPU sees	 * the ops pointer included into the list.	 */	rcu_assign_pointer(*list, ops);} static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops){	struct ftrace_ops **p; 	/*	 * If we are removing the last function, then simply point	 * to the ftrace_stub.	 */	if (*list == ops && ops->next == &ftrace_list_end) {		*list = &ftrace_list_end;		return 0;	} 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)		if (*p == ops)			break; 	if (*p != ops)		return -1; 	*p = (*p)->next;	return 0;} static void add_ftrace_list_ops(struct ftrace_ops **list,				struct ftrace_ops *main_ops,				struct ftrace_ops *ops){	int first = *list == &ftrace_list_end;	add_ftrace_ops(list, ops);	if (first)		add_ftrace_ops(&ftrace_ops_list, main_ops);} static int remove_ftrace_list_ops(struct ftrace_ops **list,				  struct ftrace_ops *main_ops,				  struct ftrace_ops *ops){	int ret = remove_ftrace_ops(list, ops);	if (!ret && *list == &ftrace_list_end)		ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);	return ret;} static void ftrace_update_trampoline(struct ftrace_ops *ops); static int __register_ftrace_function(struct ftrace_ops *ops){	if (ops->flags & FTRACE_OPS_FL_DELETED)		return -EINVAL; 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))		return -EBUSY; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS	/*	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.	 */	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))		return -EINVAL; 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;#endif 	if (!core_kernel_data((unsigned long)ops))		ops->flags |= FTRACE_OPS_FL_DYNAMIC; 	if (ops->flags & FTRACE_OPS_FL_CONTROL) {		if (control_ops_alloc(ops))			return -ENOMEM;		add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);		/* The control_ops needs the trampoline update */		ops = &control_ops;	} else		add_ftrace_ops(&ftrace_ops_list, ops); 	/* Always save the function, and reset at unregistering */	ops->saved_func = ops->func; 	if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled())		ops->func = ftrace_pid_func; 	ftrace_update_trampoline(ops); 	if (ftrace_enabled)		update_ftrace_function(); 	return 0;} static int __unregister_ftrace_function(struct ftrace_ops *ops){	int ret; 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))		return -EBUSY; 	if (ops->flags & FTRACE_OPS_FL_CONTROL) {		ret = remove_ftrace_list_ops(&ftrace_control_list,					     &control_ops, ops);	} else		ret = remove_ftrace_ops(&ftrace_ops_list, ops); 	if (ret < 0)		return ret; 	if (ftrace_enabled)		update_ftrace_function(); 	ops->func = ops->saved_func; 	return 0;} static void ftrace_update_pid_func(void){	bool enabled = ftrace_pids_enabled();	struct ftrace_ops *op; 	/* Only do something if we are tracing something */	if (ftrace_trace_function == ftrace_stub)		return; 	do_for_each_ftrace_op(op, ftrace_ops_list) {		if (op->flags & FTRACE_OPS_FL_PID) {			op->func = enabled ? ftrace_pid_func :				op->saved_func;			ftrace_update_trampoline(op);		}	} while_for_each_ftrace_op(op); 	update_ftrace_function();} #ifdef CONFIG_FUNCTION_PROFILERstruct ftrace_profile {	struct hlist_node		node;	unsigned long			ip;	unsigned long			counter;#ifdef CONFIG_FUNCTION_GRAPH_TRACER

Lines 1–500 of 5967. Load more