// SPDX-License-Identifier: GPL-2.0/* * ring buffer based function tracer * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> * * Based on code from the latency_tracer, that is: * *  Copyright (C) 2004-2006 Ingo Molnar *  Copyright (C) 2004 Nadia Yvette Chambers */#include <linux/ring_buffer.h>#include <linux/debugfs.h>#include <linux/uaccess.h>#include <linux/ftrace.h>#include <linux/slab.h>#include <linux/fs.h> #include "trace.h" static void tracing_start_function_trace(struct trace_array *tr);static void tracing_stop_function_trace(struct trace_array *tr);static voidfunction_trace_call(unsigned long ip, unsigned long parent_ip,		    struct ftrace_ops *op, struct ftrace_regs *fregs);static voidfunction_args_trace_call(unsigned long ip, unsigned long parent_ip,			 struct ftrace_ops *op, struct ftrace_regs *fregs);static voidfunction_stack_trace_call(unsigned long ip, unsigned long parent_ip,			  struct ftrace_ops *op, struct ftrace_regs *fregs);static voidfunction_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,			       struct ftrace_ops *op, struct ftrace_regs *fregs);static voidfunction_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,				     struct ftrace_ops *op,				     struct ftrace_regs *fregs);static struct tracer_flags func_flags; /* Our option */enum { 	TRACE_FUNC_NO_OPTS		= 0x0, /* No flags set. */	TRACE_FUNC_OPT_STACK		= 0x1,	TRACE_FUNC_OPT_NO_REPEATS	= 0x2,	TRACE_FUNC_OPT_ARGS		= 0x4, 	/* Update this to next highest bit. */	TRACE_FUNC_OPT_HIGHEST_BIT	= 0x8}; #define TRACE_FUNC_OPT_MASK	(TRACE_FUNC_OPT_HIGHEST_BIT - 1) int ftrace_allocate_ftrace_ops(struct trace_array *tr){	struct ftrace_ops *ops; 	/* The top level array uses the "global_ops" */	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)		return 0; 	ops = kzalloc_obj(*ops);	if (!ops)		return -ENOMEM; 	/* Currently only the non stack version is supported */	ops->func = function_trace_call;	ops->flags = FTRACE_OPS_FL_PID; 	tr->ops = ops;	ops->private = tr; 	return 0;} void ftrace_free_ftrace_ops(struct trace_array *tr){	kfree(tr->ops);	tr->ops = NULL;} int ftrace_create_function_files(struct trace_array *tr,				 struct dentry *parent){	int ret;	/*	 * The top level array uses the "global_ops", and the files are	 * created on boot up.	 */	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)		return 0; 	if (!tr->ops)		return -EINVAL; 	ret = allocate_fgraph_ops(tr, tr->ops);	if (ret) {		kfree(tr->ops);		return ret;	} 	ftrace_create_filter_files(tr->ops, parent); 	return 0;} void ftrace_destroy_function_files(struct trace_array *tr){	ftrace_destroy_filter_files(tr->ops);	ftrace_free_ftrace_ops(tr);	free_fgraph_ops(tr);} static ftrace_func_t select_trace_function(u32 flags_val){	switch (flags_val & TRACE_FUNC_OPT_MASK) {	case TRACE_FUNC_NO_OPTS:		return function_trace_call;	case TRACE_FUNC_OPT_ARGS:		return function_args_trace_call;	case TRACE_FUNC_OPT_STACK:		return function_stack_trace_call;	case TRACE_FUNC_OPT_NO_REPEATS:		return function_no_repeats_trace_call;	case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS:		return function_stack_no_repeats_trace_call;	default:		return NULL;	}} static bool handle_func_repeats(struct trace_array *tr, u32 flags_val){	if (!tr->last_func_repeats &&	    (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) {		tr->last_func_repeats = alloc_percpu(struct trace_func_repeats);		if (!tr->last_func_repeats)			return false;	} 	return true;} static int function_trace_init(struct trace_array *tr){	ftrace_func_t func;	/*	 * Instance trace_arrays get their ops allocated	 * at instance creation. Unless it failed	 * the allocation.	 */	if (!tr->ops)		return -ENOMEM; 	func = select_trace_function(tr->current_trace_flags->val);	if (!func)		return -EINVAL; 	if (!handle_func_repeats(tr, tr->current_trace_flags->val))		return -ENOMEM; 	ftrace_init_array_ops(tr, func); 	tr->array_buffer.cpu = raw_smp_processor_id(); 	tracing_start_cmdline_record();	tracing_start_function_trace(tr);	return 0;} static void function_trace_reset(struct trace_array *tr){	tracing_stop_function_trace(tr);	tracing_stop_cmdline_record();	ftrace_reset_array_ops(tr);} static void function_trace_start(struct trace_array *tr){	tracing_reset_online_cpus(&tr->array_buffer);} /* fregs are guaranteed not to be NULL if HAVE_DYNAMIC_FTRACE_WITH_ARGS is set */#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS)static __always_inline unsigned longfunction_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs){	unsigned long true_parent_ip;	int idx = 0; 	true_parent_ip = parent_ip;	if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs)		true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip,				(unsigned long *)ftrace_regs_get_stack_pointer(fregs));	return true_parent_ip;}#elsestatic __always_inline unsigned longfunction_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs){	return parent_ip;}#endif static voidfunction_trace_call(unsigned long ip, unsigned long parent_ip,		    struct ftrace_ops *op, struct ftrace_regs *fregs){	struct trace_array *tr = op->private;	unsigned int trace_ctx;	int bit; 	if (unlikely(!tr->function_enabled))		return; 	bit = ftrace_test_recursion_trylock(ip, parent_ip);	if (bit < 0)		return; 	parent_ip = function_get_true_parent_ip(parent_ip, fregs); 	trace_ctx = tracing_gen_ctx_dec(); 	trace_function(tr, ip, parent_ip, trace_ctx, NULL); 	ftrace_test_recursion_unlock(bit);} static voidfunction_args_trace_call(unsigned long ip, unsigned long parent_ip,			 struct ftrace_ops *op, struct ftrace_regs *fregs){	struct trace_array *tr = op->private;	unsigned int trace_ctx;	int bit; 	if (unlikely(!tr->function_enabled))		return; 	bit = ftrace_test_recursion_trylock(ip, parent_ip);	if (bit < 0)		return; 	trace_ctx = tracing_gen_ctx(); 	trace_function(tr, ip, parent_ip, trace_ctx, fregs); 	ftrace_test_recursion_unlock(bit);} #ifdef CONFIG_UNWINDER_ORC/* * Skip 2: * *   function_stack_trace_call() *   ftrace_call() */#define STACK_SKIP 2#else/* * Skip 3: *   __trace_stack() *   function_stack_trace_call() *   ftrace_call() */#define STACK_SKIP 3#endif static voidfunction_stack_trace_call(unsigned long ip, unsigned long parent_ip,			  struct ftrace_ops *op, struct ftrace_regs *fregs){	struct trace_array *tr = op->private;	struct trace_array_cpu *data;	unsigned long flags;	long disabled;	int cpu;	unsigned int trace_ctx;	int skip = STACK_SKIP; 	if (unlikely(!tr->function_enabled))		return; 	/*	 * Need to use raw, since this must be called before the	 * recursive protection is performed.	 */	local_irq_save(flags);	parent_ip = function_get_true_parent_ip(parent_ip, fregs);	cpu = raw_smp_processor_id();	data = per_cpu_ptr(tr->array_buffer.data, cpu);	disabled = local_inc_return(&data->disabled); 	if (likely(disabled == 1)) {		trace_ctx = tracing_gen_ctx_flags(flags);		trace_function(tr, ip, parent_ip, trace_ctx, NULL);#ifdef CONFIG_UNWINDER_FRAME_POINTER		if (ftrace_pids_enabled(op))			skip++;#endif		__trace_stack(tr, trace_ctx, skip);	} 	local_dec(&data->disabled);	local_irq_restore(flags);} static inline bool is_repeat_check(struct trace_array *tr,				   struct trace_func_repeats *last_info,				   unsigned long ip, unsigned long parent_ip){	if (last_info->ip == ip &&	    last_info->parent_ip == parent_ip &&	    last_info->count < U16_MAX) {		last_info->ts_last_call =			ring_buffer_time_stamp(tr->array_buffer.buffer);		last_info->count++;		return true;	} 	return false;} static inline void process_repeats(struct trace_array *tr,				   unsigned long ip, unsigned long parent_ip,				   struct trace_func_repeats *last_info,				   unsigned int trace_ctx){	if (last_info->count) {		trace_last_func_repeats(tr, last_info, trace_ctx);		last_info->count = 0;	} 	last_info->ip = ip;	last_info->parent_ip = parent_ip;} static voidfunction_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,			       struct ftrace_ops *op,			       struct ftrace_regs *fregs){	struct trace_func_repeats *last_info;	struct trace_array *tr = op->private;	unsigned int trace_ctx;	int bit; 	if (unlikely(!tr->function_enabled))		return; 	bit = ftrace_test_recursion_trylock(ip, parent_ip);	if (bit < 0)		return; 	parent_ip = function_get_true_parent_ip(parent_ip, fregs);	if (!tracer_tracing_is_on(tr))		goto out; 	/*	 * An interrupt may happen at any place here. But as far as I can see,	 * the only damage that this can cause is to mess up the repetition	 * counter without valuable data being lost.	 * TODO: think about a solution that is better than just hoping to be	 * lucky.	 */	last_info = this_cpu_ptr(tr->last_func_repeats);	if (is_repeat_check(tr, last_info, ip, parent_ip))		goto out; 	trace_ctx = tracing_gen_ctx_dec();	process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 	trace_function(tr, ip, parent_ip, trace_ctx, NULL); out:	ftrace_test_recursion_unlock(bit);} static voidfunction_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,				     struct ftrace_ops *op,				     struct ftrace_regs *fregs){	struct trace_func_repeats *last_info;	struct trace_array *tr = op->private;	struct trace_array_cpu *data;	unsigned long flags;	long disabled;	int cpu;	unsigned int trace_ctx; 	if (unlikely(!tr->function_enabled))		return; 	/*	 * Need to use raw, since this must be called before the	 * recursive protection is performed.	 */	local_irq_save(flags);	parent_ip = function_get_true_parent_ip(parent_ip, fregs);	cpu = raw_smp_processor_id();	data = per_cpu_ptr(tr->array_buffer.data, cpu);	disabled = local_inc_return(&data->disabled); 	if (likely(disabled == 1)) {		last_info = per_cpu_ptr(tr->last_func_repeats, cpu);		if (is_repeat_check(tr, last_info, ip, parent_ip))			goto out; 		trace_ctx = tracing_gen_ctx_flags(flags);		process_repeats(tr, ip, parent_ip, last_info, trace_ctx); 		trace_function(tr, ip, parent_ip, trace_ctx, NULL);		__trace_stack(tr, trace_ctx, STACK_SKIP);	}  out:	local_dec(&data->disabled);	local_irq_restore(flags);} static struct tracer_opt func_opts[] = {#ifdef CONFIG_STACKTRACE	{ TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },#endif	{ TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) },#ifdef CONFIG_FUNCTION_TRACE_ARGS	{ TRACER_OPT(func-args, TRACE_FUNC_OPT_ARGS) },#endif	{ } /* Always set a last empty entry */}; static struct tracer_flags func_flags = {	.val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */	.opts = func_opts}; static void tracing_start_function_trace(struct trace_array *tr){	tr->function_enabled = 0;	register_ftrace_function(tr->ops);	tr->function_enabled = 1;} static void tracing_stop_function_trace(struct trace_array *tr){	tr->function_enabled = 0;	unregister_ftrace_function(tr->ops);} static struct tracer function_trace; static intfunc_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set){	ftrace_func_t func;	u32 new_flags; 	/* Do nothing if already set. */	if (!!set == !!(tr->current_trace_flags->val & bit))		return 0; 	/* We can change this flag only when not running. */	if (tr->current_trace != &function_trace)		return 0; 	new_flags = (tr->current_trace_flags->val & ~bit) | (set ? bit : 0);	func = select_trace_function(new_flags);	if (!func)		return -EINVAL; 	/* Check if there's anything to change. */	if (tr->ops->func == func)		return 0; 	if (!handle_func_repeats(tr, new_flags))		return -ENOMEM; 	unregister_ftrace_function(tr->ops);	tr->ops->func = func;	register_ftrace_function(tr->ops); 	return 0;} static struct tracer function_trace __tracer_data ={	.name		= "function",	.init		= function_trace_init,	.reset		= function_trace_reset,	.start		= function_trace_start,	.default_flags	= &func_flags,	.set_flag	= func_set_flag,	.allow_instances = true,#ifdef CONFIG_FTRACE_SELFTEST	.selftest	= trace_selftest_startup_function,#endif};

Lines 1–500 of 1030. Load more