/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * *  Copyright (C) 2004-2006 Ingo Molnar *  Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h>#include <linux/clocksource.h>#include <linux/sched/task.h>#include <linux/kallsyms.h>#include <linux/seq_file.h>#include <linux/suspend.h>#include <linux/tracefs.h>#include <linux/hardirq.h>#include <linux/kthread.h>#include <linux/uaccess.h>#include <linux/bsearch.h>#include <linux/module.h>#include <linux/ftrace.h>#include <linux/sysctl.h>#include <linux/slab.h>#include <linux/ctype.h>#include <linux/sort.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/rcupdate.h> #include <trace/events/sched.h> #include <asm/sections.h>#include <asm/setup.h> #include "trace_output.h"#include "trace_stat.h" #define FTRACE_WARN_ON(cond)			\	({					\		int ___r = cond;		\		if (WARN_ON(___r))		\			ftrace_kill();		\		___r;				\	}) #define FTRACE_WARN_ON_ONCE(cond)		\	({					\		int ___r = cond;		\		if (WARN_ON_ONCE(___r))		\			ftrace_kill();		\		___r;				\	}) /* hash bits for specific function selection */#define FTRACE_HASH_BITS 7#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)#define FTRACE_HASH_DEFAULT_BITS 10#define FTRACE_HASH_MAX_BITS 12 #ifdef CONFIG_DYNAMIC_FTRACE#define INIT_OPS_HASH(opsname)	\	.func_hash		= &opsname.local_hash,			\	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),#define ASSIGN_OPS_HASH(opsname, val) \	.func_hash		= val, \	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),#else#define INIT_OPS_HASH(opsname)#define ASSIGN_OPS_HASH(opsname, val)#endif static struct ftrace_ops ftrace_list_end __read_mostly = {	.func		= ftrace_stub,	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,	INIT_OPS_HASH(ftrace_list_end)}; /* ftrace_enabled is a method to turn ftrace on or off */int ftrace_enabled __read_mostly;static int last_ftrace_enabled; /* Current function tracing op */struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;/* What to set function_trace_op to */static struct ftrace_ops *set_function_trace_op; static bool ftrace_pids_enabled(struct ftrace_ops *ops){	struct trace_array *tr; 	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)		return false; 	tr = ops->private; 	return tr->function_pids != NULL;} static void ftrace_update_trampoline(struct ftrace_ops *ops); /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */static int ftrace_disabled __read_mostly; static DEFINE_MUTEX(ftrace_lock); static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;static struct ftrace_ops global_ops; #if ARCH_SUPPORTS_FTRACE_OPSstatic void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,				 struct ftrace_ops *op, struct pt_regs *regs);#else/* See comment below, where ftrace_ops_list_func is defined */static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)#endif /* * Traverse the ftrace_global_list, invoking all entries.  The reason that we * can use rcu_dereference_raw_notrace() is that elements removed from this list * are simply leaked, so there is no need to interact with a grace-period * mechanism.  The rcu_dereference_raw_notrace() calls are needed to handle * concurrent insertions into the ftrace_global_list. * * Silly Alpha and silly pointer-speculation compiler optimizations! */#define do_for_each_ftrace_op(op, list)			\	op = rcu_dereference_raw_notrace(list);			\	do /* * Optimized for just a single item in the list (as that is the normal case). */#define while_for_each_ftrace_op(op)				\	while (likely(op = rcu_dereference_raw_notrace((op)->next)) &&	\	       unlikely((op) != &ftrace_list_end)) static inline void ftrace_ops_init(struct ftrace_ops *ops){#ifdef CONFIG_DYNAMIC_FTRACE	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {		mutex_init(&ops->local_hash.regex_lock);		ops->func_hash = &ops->local_hash;		ops->flags |= FTRACE_OPS_FL_INITIALIZED;	}#endif} /** * ftrace_nr_registered_ops - return number of ops registered * * Returns the number of ftrace_ops registered and tracing functions */int ftrace_nr_registered_ops(void){	struct ftrace_ops *ops;	int cnt = 0; 	mutex_lock(&ftrace_lock); 	for (ops = rcu_dereference_protected(ftrace_ops_list,					     lockdep_is_held(&ftrace_lock));	     ops != &ftrace_list_end;	     ops = rcu_dereference_protected(ops->next,					     lockdep_is_held(&ftrace_lock)))		cnt++; 	mutex_unlock(&ftrace_lock); 	return cnt;} static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,			    struct ftrace_ops *op, struct pt_regs *regs){	struct trace_array *tr = op->private; 	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))		return; 	op->saved_func(ip, parent_ip, op, regs);} /** * clear_ftrace_function - reset the ftrace function * * This NULLs the ftrace function and in essence stops * tracing.  There may be lag */void clear_ftrace_function(void){	ftrace_trace_function = ftrace_stub;} static void per_cpu_ops_disable_all(struct ftrace_ops *ops){	int cpu; 	for_each_possible_cpu(cpu)		*per_cpu_ptr(ops->disabled, cpu) = 1;} static int per_cpu_ops_alloc(struct ftrace_ops *ops){	int __percpu *disabled; 	if (WARN_ON_ONCE(!(ops->flags & FTRACE_OPS_FL_PER_CPU)))		return -EINVAL; 	disabled = alloc_percpu(int);	if (!disabled)		return -ENOMEM; 	ops->disabled = disabled;	per_cpu_ops_disable_all(ops);	return 0;} static void ftrace_sync(struct work_struct *work){	/*	 * This function is just a stub to implement a hard force	 * of synchronize_sched(). This requires synchronizing	 * tasks even in userspace and idle.	 *	 * Yes, function tracing is rude.	 */} static void ftrace_sync_ipi(void *data){	/* Probably not needed, but do it anyway */	smp_rmb();} #ifdef CONFIG_FUNCTION_GRAPH_TRACERstatic void update_function_graph_func(void); /* Both enabled by default (can be cleared by function_graph tracer flags */static bool fgraph_sleep_time = true;static bool fgraph_graph_time = true; #elsestatic inline void update_function_graph_func(void) { }#endif  static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops){	/*	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,	 * then it needs to call the list anyway.	 */	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_PER_CPU |			  FTRACE_OPS_FL_RCU) || FTRACE_FORCE_LIST_FUNC)		return ftrace_ops_list_func; 	return ftrace_ops_get_func(ops);} static void update_ftrace_function(void){	ftrace_func_t func; 	/*	 * Prepare the ftrace_ops that the arch callback will use.	 * If there's only one ftrace_ops registered, the ftrace_ops_list	 * will point to the ops we want.	 */	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,						lockdep_is_held(&ftrace_lock)); 	/* If there's no ftrace_ops registered, just call the stub function */	if (set_function_trace_op == &ftrace_list_end) {		func = ftrace_stub; 	/*	 * If we are at the end of the list and this ops is	 * recursion safe and not dynamic and the arch supports passing ops,	 * then have the mcount trampoline call the function directly.	 */	} else if (rcu_dereference_protected(ftrace_ops_list->next,			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {		func = ftrace_ops_get_list_func(ftrace_ops_list); 	} else {		/* Just use the default ftrace_ops */		set_function_trace_op = &ftrace_list_end;		func = ftrace_ops_list_func;	} 	update_function_graph_func(); 	/* If there's no change, then do nothing more here */	if (ftrace_trace_function == func)		return; 	/*	 * If we are using the list function, it doesn't care	 * about the function_trace_ops.	 */	if (func == ftrace_ops_list_func) {		ftrace_trace_function = func;		/*		 * Don't even bother setting function_trace_ops,		 * it would be racy to do so anyway.		 */		return;	} #ifndef CONFIG_DYNAMIC_FTRACE	/*	 * For static tracing, we need to be a bit more careful.	 * The function change takes affect immediately. Thus,	 * we need to coorditate the setting of the function_trace_ops	 * with the setting of the ftrace_trace_function.	 *	 * Set the function to the list ops, which will call the	 * function we want, albeit indirectly, but it handles the	 * ftrace_ops and doesn't depend on function_trace_op.	 */	ftrace_trace_function = ftrace_ops_list_func;	/*	 * Make sure all CPUs see this. Yes this is slow, but static	 * tracing is slow and nasty to have enabled.	 */	schedule_on_each_cpu(ftrace_sync);	/* Now all cpus are using the list ops. */	function_trace_op = set_function_trace_op;	/* Make sure the function_trace_op is visible on all CPUs */	smp_wmb();	/* Nasty way to force a rmb on all cpus */	smp_call_function(ftrace_sync_ipi, NULL, 1);	/* OK, we are all set to update the ftrace_trace_function now! */#endif /* !CONFIG_DYNAMIC_FTRACE */ 	ftrace_trace_function = func;} int using_ftrace_ops_list_func(void){	return ftrace_trace_function == ftrace_ops_list_func;} static void add_ftrace_ops(struct ftrace_ops __rcu **list,			   struct ftrace_ops *ops){	rcu_assign_pointer(ops->next, *list); 	/*	 * We are entering ops into the list but another	 * CPU might be walking that list. We need to make sure	 * the ops->next pointer is valid before another CPU sees	 * the ops pointer included into the list.	 */	rcu_assign_pointer(*list, ops);} static int remove_ftrace_ops(struct ftrace_ops __rcu **list,			     struct ftrace_ops *ops){	struct ftrace_ops **p; 	/*	 * If we are removing the last function, then simply point	 * to the ftrace_stub.	 */	if (rcu_dereference_protected(*list,			lockdep_is_held(&ftrace_lock)) == ops &&	    rcu_dereference_protected(ops->next,			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {		*list = &ftrace_list_end;		return 0;	} 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)		if (*p == ops)			break; 	if (*p != ops)		return -1; 	*p = (*p)->next;	return 0;} static void ftrace_update_trampoline(struct ftrace_ops *ops); static int __register_ftrace_function(struct ftrace_ops *ops){	if (ops->flags & FTRACE_OPS_FL_DELETED)		return -EINVAL; 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))		return -EBUSY; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS	/*	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.	 */	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))		return -EINVAL; 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;#endif 	if (!core_kernel_data((unsigned long)ops))		ops->flags |= FTRACE_OPS_FL_DYNAMIC; 	if (ops->flags & FTRACE_OPS_FL_PER_CPU) {		if (per_cpu_ops_alloc(ops))			return -ENOMEM;	} 	add_ftrace_ops(&ftrace_ops_list, ops); 	/* Always save the function, and reset at unregistering */	ops->saved_func = ops->func; 	if (ftrace_pids_enabled(ops))		ops->func = ftrace_pid_func; 	ftrace_update_trampoline(ops); 	if (ftrace_enabled)		update_ftrace_function(); 	return 0;} static int __unregister_ftrace_function(struct ftrace_ops *ops){	int ret; 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))		return -EBUSY; 	ret = remove_ftrace_ops(&ftrace_ops_list, ops); 	if (ret < 0)		return ret; 	if (ftrace_enabled)		update_ftrace_function(); 	ops->func = ops->saved_func; 	return 0;} static void ftrace_update_pid_func(void){	struct ftrace_ops *op; 	/* Only do something if we are tracing something */	if (ftrace_trace_function == ftrace_stub)		return; 	do_for_each_ftrace_op(op, ftrace_ops_list) {		if (op->flags & FTRACE_OPS_FL_PID) {			op->func = ftrace_pids_enabled(op) ?				ftrace_pid_func : op->saved_func;			ftrace_update_trampoline(op);		}	} while_for_each_ftrace_op(op); 	update_ftrace_function();} #ifdef CONFIG_FUNCTION_PROFILERstruct ftrace_profile {	struct hlist_node		node;	unsigned long			ip;	unsigned long			counter;#ifdef CONFIG_FUNCTION_GRAPH_TRACER	unsigned long long		time;	unsigned long long		time_squared;#endif}; struct ftrace_profile_page {	struct ftrace_profile_page	*next;	unsigned long			index;	struct ftrace_profile		records[];};

Lines 1–500 of 6844. Load more