// SPDX-License-Identifier: GPL-2.0/* * Infrastructure for profiling code inserted by 'gcc -pg'. * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> * * Originally ported from the -rt patch by: *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> * * Based on code in the latency_tracer, that is: * *  Copyright (C) 2004-2006 Ingo Molnar *  Copyright (C) 2004 Nadia Yvette Chambers */ #include <linux/stop_machine.h>#include <linux/clocksource.h>#include <linux/sched/task.h>#include <linux/kallsyms.h>#include <linux/security.h>#include <linux/seq_file.h>#include <linux/tracefs.h>#include <linux/hardirq.h>#include <linux/kthread.h>#include <linux/uaccess.h>#include <linux/bsearch.h>#include <linux/module.h>#include <linux/ftrace.h>#include <linux/sysctl.h>#include <linux/slab.h>#include <linux/ctype.h>#include <linux/sort.h>#include <linux/list.h>#include <linux/hash.h>#include <linux/rcupdate.h>#include <linux/kprobes.h> #include <trace/events/sched.h> #include <asm/sections.h>#include <asm/setup.h> #include "ftrace_internal.h"#include "trace_output.h"#include "trace_stat.h" /* Flags that do not get reset */#define FTRACE_NOCLEAR_FLAGS	(FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \				 FTRACE_FL_MODIFIED) #define FTRACE_INVALID_FUNCTION		"__ftrace_invalid_address__" #define FTRACE_WARN_ON(cond)			\	({					\		int ___r = cond;		\		if (WARN_ON(___r))		\			ftrace_kill();		\		___r;				\	}) #define FTRACE_WARN_ON_ONCE(cond)		\	({					\		int ___r = cond;		\		if (WARN_ON_ONCE(___r))		\			ftrace_kill();		\		___r;				\	}) /* hash bits for specific function selection */#define FTRACE_HASH_MAX_BITS 12 #ifdef CONFIG_DYNAMIC_FTRACE#define INIT_OPS_HASH(opsname)	\	.func_hash		= &opsname.local_hash,			\	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \	.subop_list		= LIST_HEAD_INIT(opsname.subop_list),#else#define INIT_OPS_HASH(opsname)#endif enum {	FTRACE_MODIFY_ENABLE_FL		= (1 << 0),	FTRACE_MODIFY_MAY_SLEEP_FL	= (1 << 1),}; struct ftrace_ops ftrace_list_end __read_mostly = {	.func		= ftrace_stub,	.flags		= FTRACE_OPS_FL_STUB,	INIT_OPS_HASH(ftrace_list_end)}; /* ftrace_enabled is a method to turn ftrace on or off */int ftrace_enabled __read_mostly;static int __maybe_unused last_ftrace_enabled; /* Current function tracing op */struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;/* What to set function_trace_op to */static struct ftrace_ops *set_function_trace_op; bool ftrace_pids_enabled(struct ftrace_ops *ops){	struct trace_array *tr; 	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)		return false; 	tr = ops->private; 	return tr->function_pids != NULL || tr->function_no_pids != NULL;} static void ftrace_update_trampoline(struct ftrace_ops *ops); /* * ftrace_disabled is set when an anomaly is discovered. * ftrace_disabled is much stronger than ftrace_enabled. */static int ftrace_disabled __read_mostly; DEFINE_MUTEX(ftrace_lock); struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = (struct ftrace_ops __rcu *)&ftrace_list_end;ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;struct ftrace_ops global_ops; /* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,			  struct ftrace_ops *op, struct ftrace_regs *fregs); #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS/* * Stub used to invoke the list ops without requiring a separate trampoline. */const struct ftrace_ops ftrace_list_ops = {	.func	= ftrace_ops_list_func,	.flags	= FTRACE_OPS_FL_STUB,}; static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip,				struct ftrace_ops *op,				struct ftrace_regs *fregs){	/* do nothing */} /* * Stub used when a call site is disabled. May be called transiently by threads * which have made it into ftrace_caller but haven't yet recovered the ops at * the point the call site is disabled. */const struct ftrace_ops ftrace_nop_ops = {	.func	= ftrace_ops_nop_func,	.flags  = FTRACE_OPS_FL_STUB,};#endif static inline void ftrace_ops_init(struct ftrace_ops *ops){#ifdef CONFIG_DYNAMIC_FTRACE	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {		mutex_init(&ops->local_hash.regex_lock);		INIT_LIST_HEAD(&ops->subop_list);		ops->func_hash = &ops->local_hash;		ops->flags |= FTRACE_OPS_FL_INITIALIZED;	}#endif} /* Call this function for when a callback filters on set_ftrace_pid */static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,			    struct ftrace_ops *op, struct ftrace_regs *fregs){	struct trace_array *tr = op->private;	int pid; 	if (tr) {		pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);		if (pid == FTRACE_PID_IGNORE)			return;		if (pid != FTRACE_PID_TRACE &&		    pid != current->pid)			return;	} 	op->saved_func(ip, parent_ip, op, fregs);} void ftrace_sync_ipi(void *data){	/* Probably not needed, but do it anyway */	smp_rmb();} static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops){	/*	 * If this is a dynamic or RCU ops, or we force list func,	 * then it needs to call the list anyway.	 */	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||	    FTRACE_FORCE_LIST_FUNC)		return ftrace_ops_list_func; 	return ftrace_ops_get_func(ops);} static void update_ftrace_function(void){	ftrace_func_t func; 	/*	 * Prepare the ftrace_ops that the arch callback will use.	 * If there's only one ftrace_ops registered, the ftrace_ops_list	 * will point to the ops we want.	 */	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,						lockdep_is_held(&ftrace_lock)); 	/* If there's no ftrace_ops registered, just call the stub function */	if (set_function_trace_op == &ftrace_list_end) {		func = ftrace_stub; 	/*	 * If we are at the end of the list and this ops is	 * recursion safe and not dynamic and the arch supports passing ops,	 * then have the mcount trampoline call the function directly.	 */	} else if (rcu_dereference_protected(ftrace_ops_list->next,			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {		func = ftrace_ops_get_list_func(ftrace_ops_list); 	} else {		/* Just use the default ftrace_ops */		set_function_trace_op = &ftrace_list_end;		func = ftrace_ops_list_func;	} 	/* If there's no change, then do nothing more here */	if (ftrace_trace_function == func)		return; 	/*	 * If we are using the list function, it doesn't care	 * about the function_trace_ops.	 */	if (func == ftrace_ops_list_func) {		ftrace_trace_function = func;		/*		 * Don't even bother setting function_trace_ops,		 * it would be racy to do so anyway.		 */		return;	} #ifndef CONFIG_DYNAMIC_FTRACE	/*	 * For static tracing, we need to be a bit more careful.	 * The function change takes affect immediately. Thus,	 * we need to coordinate the setting of the function_trace_ops	 * with the setting of the ftrace_trace_function.	 *	 * Set the function to the list ops, which will call the	 * function we want, albeit indirectly, but it handles the	 * ftrace_ops and doesn't depend on function_trace_op.	 */	ftrace_trace_function = ftrace_ops_list_func;	/*	 * Make sure all CPUs see this. Yes this is slow, but static	 * tracing is slow and nasty to have enabled.	 */	synchronize_rcu_tasks_rude();	/* Now all cpus are using the list ops. */	function_trace_op = set_function_trace_op;	/* Make sure the function_trace_op is visible on all CPUs */	smp_wmb();	/* Nasty way to force a rmb on all cpus */	smp_call_function(ftrace_sync_ipi, NULL, 1);	/* OK, we are all set to update the ftrace_trace_function now! */#endif /* !CONFIG_DYNAMIC_FTRACE */ 	ftrace_trace_function = func;} static void add_ftrace_ops(struct ftrace_ops __rcu **list,			   struct ftrace_ops *ops){	rcu_assign_pointer(ops->next, *list); 	/*	 * We are entering ops into the list but another	 * CPU might be walking that list. We need to make sure	 * the ops->next pointer is valid before another CPU sees	 * the ops pointer included into the list.	 */	rcu_assign_pointer(*list, ops);} static int remove_ftrace_ops(struct ftrace_ops __rcu **list,			     struct ftrace_ops *ops){	struct ftrace_ops **p; 	/*	 * If we are removing the last function, then simply point	 * to the ftrace_stub.	 */	if (rcu_dereference_protected(*list,			lockdep_is_held(&ftrace_lock)) == ops &&	    rcu_dereference_protected(ops->next,			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {		rcu_assign_pointer(*list, &ftrace_list_end);		return 0;	} 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)		if (*p == ops)			break; 	if (*p != ops)		return -1; 	*p = (*p)->next;	return 0;} static void ftrace_update_trampoline(struct ftrace_ops *ops); int __register_ftrace_function(struct ftrace_ops *ops){	if (ops->flags & FTRACE_OPS_FL_DELETED)		return -EINVAL; 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))		return -EBUSY; #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS	/*	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.	 */	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))		return -EINVAL; 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;#endif	if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))		return -EBUSY; 	if (!is_kernel_core_data((unsigned long)ops))		ops->flags |= FTRACE_OPS_FL_DYNAMIC; 	add_ftrace_ops(&ftrace_ops_list, ops); 	/* Always save the function, and reset at unregistering */	ops->saved_func = ops->func; 	if (ftrace_pids_enabled(ops))		ops->func = ftrace_pid_func; 	ftrace_update_trampoline(ops); 	if (ftrace_enabled)		update_ftrace_function(); 	return 0;} int __unregister_ftrace_function(struct ftrace_ops *ops){	int ret; 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))		return -EBUSY; 	ret = remove_ftrace_ops(&ftrace_ops_list, ops); 	if (ret < 0)		return ret; 	if (ftrace_enabled)		update_ftrace_function(); 	ops->func = ops->saved_func; 	return 0;} static void ftrace_update_pid_func(void){	struct ftrace_ops *op; 	/* Only do something if we are tracing something */	if (ftrace_trace_function == ftrace_stub)		return; 	do_for_each_ftrace_op(op, ftrace_ops_list) {		if (op->flags & FTRACE_OPS_FL_PID) {			op->func = ftrace_pids_enabled(op) ?				ftrace_pid_func : op->saved_func;			ftrace_update_trampoline(op);		}	} while_for_each_ftrace_op(op); 	fgraph_update_pid_func(); 	update_ftrace_function();} #ifdef CONFIG_FUNCTION_PROFILERstruct ftrace_profile {	struct hlist_node		node;	unsigned long			ip;	unsigned long			counter;#ifdef CONFIG_FUNCTION_GRAPH_TRACER	unsigned long long		time;	unsigned long long		time_squared;#endif}; struct ftrace_profile_page {	struct ftrace_profile_page	*next;	unsigned long			index;	struct ftrace_profile		records[];}; struct ftrace_profile_stat {	atomic_t			disabled;	struct hlist_head		*hash;	struct ftrace_profile_page	*pages;	struct ftrace_profile_page	*start;	struct tracer_stat		stat;}; #define PROFILE_RECORDS_SIZE						\	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) #define PROFILES_PER_PAGE					\	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) static int ftrace_profile_enabled __read_mostly; /* ftrace_profile_lock - synchronize the enable and disable of the profiler */static DEFINE_MUTEX(ftrace_profile_lock); static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); #define FTRACE_PROFILE_HASH_BITS 10#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) static void *function_stat_next(void *v, int idx){	struct ftrace_profile *rec = v;	struct ftrace_profile_page *pg; 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);  again:	if (idx != 0)		rec++; 	if ((void *)rec >= (void *)&pg->records[pg->index]) {		pg = pg->next;		if (!pg)			return NULL;		rec = &pg->records[0];		if (!rec->counter)			goto again;	} 	return rec;} static void *function_stat_start(struct tracer_stat *trace){	struct ftrace_profile_stat *stat =		container_of(trace, struct ftrace_profile_stat, stat); 	if (!stat || !stat->start)		return NULL; 	return function_stat_next(&stat->start->records[0], 0);} #ifdef CONFIG_FUNCTION_GRAPH_TRACER/* function graph compares on total time */static int function_stat_cmp(const void *p1, const void *p2){	const struct ftrace_profile *a = p1;	const struct ftrace_profile *b = p2; 	if (a->time < b->time)		return -1;	if (a->time > b->time)		return 1;

Lines 1–500 of 9475. Load more