// SPDX-License-Identifier: GPL-2.0/* * trace task wakeup timings * * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> * * Based on code from the latency_tracer, that is: * *  Copyright (C) 2004-2006 Ingo Molnar *  Copyright (C) 2004 Nadia Yvette Chambers */#include <linux/module.h>#include <linux/kallsyms.h>#include <linux/uaccess.h>#include <linux/ftrace.h>#include <linux/sched/rt.h>#include <linux/sched/deadline.h>#include <trace/events/sched.h>#include "trace.h" static struct trace_array	*wakeup_trace;static int __read_mostly	tracer_enabled; static struct task_struct	*wakeup_task;static int			wakeup_cpu;static int			wakeup_current_cpu;static unsigned			wakeup_prio = -1;static bool			wakeup_rt;static bool			wakeup_dl;static bool			tracing_dl; static arch_spinlock_t wakeup_lock =	(arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; static void wakeup_reset(struct trace_array *tr);static void __wakeup_reset(struct trace_array *tr);static int start_func_tracer(struct trace_array *tr, int graph);static void stop_func_tracer(struct trace_array *tr, int graph); static int save_flags; #ifdef CONFIG_FUNCTION_GRAPH_TRACER# define is_graph(tr) ((tr)->trace_flags & TRACE_ITER(DISPLAY_GRAPH))#else# define is_graph(tr) false#endif #ifdef CONFIG_FUNCTION_TRACER static bool function_enabled; /* * Prologue for the wakeup function tracers. * * Returns 1 if it is OK to continue, and preemption *            is disabled and data->disabled is incremented. *         0 if the trace is to be ignored, and preemption *            is not disabled and data->disabled is *            kept the same. * * Note, this function is also used outside this ifdef but *  inside the #ifdef of the function graph tracer below. *  This is OK, since the function graph tracer is *  dependent on the function tracer. */static intfunc_prolog_preempt_disable(struct trace_array *tr,			    struct trace_array_cpu **data,			    unsigned int *trace_ctx){	long disabled;	int cpu; 	if (likely(!wakeup_task))		return 0; 	*trace_ctx = tracing_gen_ctx();	preempt_disable_notrace(); 	cpu = raw_smp_processor_id();	if (cpu != wakeup_current_cpu)		goto out_enable; 	*data = per_cpu_ptr(tr->array_buffer.data, cpu);	disabled = local_inc_return(&(*data)->disabled);	if (unlikely(disabled != 1))		goto out; 	return 1; out:	local_dec(&(*data)->disabled); out_enable:	preempt_enable_notrace();	return 0;} #ifdef CONFIG_FUNCTION_GRAPH_TRACER static int wakeup_display_graph(struct trace_array *tr, int set){	if (!(is_graph(tr) ^ set))		return 0; 	stop_func_tracer(tr, !set); 	wakeup_reset(wakeup_trace);	tr->max_latency = 0; 	return start_func_tracer(tr, set);} static int wakeup_graph_entry(struct ftrace_graph_ent *trace,			      struct fgraph_ops *gops,			      struct ftrace_regs *fregs){	struct trace_array *tr = wakeup_trace;	struct trace_array_cpu *data;	unsigned int trace_ctx;	u64 *calltime;	int ret = 0; 	if (ftrace_graph_ignore_func(gops, trace))		return 0;	/*	 * Do not trace a function if it's filtered by set_graph_notrace.	 * Make the index of ret stack negative to indicate that it should	 * ignore further functions.  But it needs its own ret stack entry	 * to recover the original index in order to continue tracing after	 * returning from the function.	 */	if (ftrace_graph_notrace_addr(trace->func))		return 1; 	if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))		return 0; 	calltime = fgraph_reserve_data(gops->idx, sizeof(*calltime));	if (calltime) {		*calltime = trace_clock_local();		ret = __trace_graph_entry(tr, trace, trace_ctx);	}	local_dec(&data->disabled);	preempt_enable_notrace(); 	return ret;} static void wakeup_graph_return(struct ftrace_graph_ret *trace,				struct fgraph_ops *gops,				struct ftrace_regs *fregs){	struct trace_array *tr = wakeup_trace;	struct trace_array_cpu *data;	unsigned int trace_ctx;	u64 *calltime;	u64 rettime;	int size; 	ftrace_graph_addr_finish(gops, trace); 	if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))		return; 	rettime = trace_clock_local(); 	calltime = fgraph_retrieve_data(gops->idx, &size);	if (calltime)		__trace_graph_return(tr, trace, trace_ctx, *calltime, rettime); 	local_dec(&data->disabled);	preempt_enable_notrace();	return;} static struct fgraph_ops fgraph_wakeup_ops = {	.entryfunc = &wakeup_graph_entry,	.retfunc = &wakeup_graph_return,}; static void wakeup_trace_open(struct trace_iterator *iter){	if (is_graph(iter->tr))		graph_trace_open(iter);} static void wakeup_trace_close(struct trace_iterator *iter){	if (iter->private)		graph_trace_close(iter);} #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \			    TRACE_GRAPH_PRINT_CPU |  \			    TRACE_GRAPH_PRINT_REL_TIME | \			    TRACE_GRAPH_PRINT_DURATION | \			    TRACE_GRAPH_PRINT_OVERHEAD | \			    TRACE_GRAPH_PRINT_IRQS) static enum print_line_t wakeup_print_line(struct trace_iterator *iter){	/*	 * In graph mode call the graph tracer output function,	 * otherwise go with the TRACE_FN event handler	 */	if (is_graph(iter->tr))		return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); 	return TRACE_TYPE_UNHANDLED;} static void wakeup_print_header(struct seq_file *s){	if (is_graph(wakeup_trace))		print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);	else		trace_default_header(s);}#endif /* else CONFIG_FUNCTION_GRAPH_TRACER */ /* * wakeup uses its own tracer function to keep the overhead down: */static voidwakeup_tracer_call(unsigned long ip, unsigned long parent_ip,		   struct ftrace_ops *op, struct ftrace_regs *fregs){	struct trace_array *tr = wakeup_trace;	struct trace_array_cpu *data;	unsigned long flags;	unsigned int trace_ctx; 	if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))		return; 	local_irq_save(flags);	trace_function(tr, ip, parent_ip, trace_ctx, fregs);	local_irq_restore(flags); 	local_dec(&data->disabled);	preempt_enable_notrace();} static int register_wakeup_function(struct trace_array *tr, int graph, int set){	int ret; 	/* 'set' is set if TRACE_ITER(FUNCTION) is about to be set */	if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER(FUNCTION))))		return 0; 	if (graph)		ret = register_ftrace_graph(&fgraph_wakeup_ops);	else		ret = register_ftrace_function(tr->ops); 	if (!ret)		function_enabled = true; 	return ret;} static void unregister_wakeup_function(struct trace_array *tr, int graph){	if (!function_enabled)		return; 	if (graph)		unregister_ftrace_graph(&fgraph_wakeup_ops);	else		unregister_ftrace_function(tr->ops); 	function_enabled = false;} static int wakeup_function_set(struct trace_array *tr, u32 mask, int set){	if (!(mask & TRACE_ITER(FUNCTION)))		return 0; 	if (set)		register_wakeup_function(tr, is_graph(tr), 1);	else		unregister_wakeup_function(tr, is_graph(tr));	return 1;}#else /* CONFIG_FUNCTION_TRACER */static int register_wakeup_function(struct trace_array *tr, int graph, int set){	return 0;}static void unregister_wakeup_function(struct trace_array *tr, int graph) { }static int wakeup_function_set(struct trace_array *tr, u32 mask, int set){	return 0;}#endif /* else CONFIG_FUNCTION_TRACER */ #ifndef CONFIG_FUNCTION_GRAPH_TRACERstatic enum print_line_t wakeup_print_line(struct trace_iterator *iter){	return TRACE_TYPE_UNHANDLED;} static void wakeup_trace_open(struct trace_iterator *iter) { }static void wakeup_trace_close(struct trace_iterator *iter) { } static void wakeup_print_header(struct seq_file *s){	trace_default_header(s);}#endif /* !CONFIG_FUNCTION_GRAPH_TRACER */ static void__trace_function(struct trace_array *tr,		 unsigned long ip, unsigned long parent_ip,		 unsigned int trace_ctx){	if (is_graph(tr))		trace_graph_function(tr, ip, parent_ip, trace_ctx);	else		trace_function(tr, ip, parent_ip, trace_ctx, NULL);} static int wakeup_flag_changed(struct trace_array *tr, u64 mask, int set){	struct tracer *tracer = tr->current_trace; 	if (wakeup_function_set(tr, mask, set))		return 0; #ifdef CONFIG_FUNCTION_GRAPH_TRACER	if (mask & TRACE_ITER(DISPLAY_GRAPH))		return wakeup_display_graph(tr, set);#endif 	return trace_keep_overwrite(tracer, mask, set);} static int start_func_tracer(struct trace_array *tr, int graph){	int ret; 	ret = register_wakeup_function(tr, graph, 0); 	if (!ret && tracing_is_enabled())		tracer_enabled = 1;	else		tracer_enabled = 0; 	return ret;} static void stop_func_tracer(struct trace_array *tr, int graph){	tracer_enabled = 0; 	unregister_wakeup_function(tr, graph);} /* * Should this new latency be reported/recorded? */static bool report_latency(struct trace_array *tr, u64 delta){	if (tracing_thresh) {		if (delta < tracing_thresh)			return false;	} else {		if (delta <= tr->max_latency)			return false;	}	return true;} static voidprobe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu){	if (task != wakeup_task)		return; 	wakeup_current_cpu = cpu;} static voidtracing_sched_switch_trace(struct trace_array *tr,			   struct task_struct *prev,			   struct task_struct *next,			   unsigned int trace_ctx){	struct trace_buffer *buffer = tr->array_buffer.buffer;	struct ring_buffer_event *event;	struct ctx_switch_entry *entry; 	event = trace_buffer_lock_reserve(buffer, TRACE_CTX,					  sizeof(*entry), trace_ctx);	if (!event)		return;	entry	= ring_buffer_event_data(event);	entry->prev_pid			= prev->pid;	entry->prev_prio		= prev->prio;	entry->prev_state		= task_state_index(prev);	entry->next_pid			= next->pid;	entry->next_prio		= next->prio;	entry->next_state		= task_state_index(next);	entry->next_cpu	= task_cpu(next); 	trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);} static voidtracing_sched_wakeup_trace(struct trace_array *tr,			   struct task_struct *wakee,			   struct task_struct *curr,			   unsigned int trace_ctx){	struct ring_buffer_event *event;	struct ctx_switch_entry *entry;	struct trace_buffer *buffer = tr->array_buffer.buffer; 	event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,					  sizeof(*entry), trace_ctx);	if (!event)		return;	entry	= ring_buffer_event_data(event);	entry->prev_pid			= curr->pid;	entry->prev_prio		= curr->prio;	entry->prev_state		= task_state_index(curr);	entry->next_pid			= wakee->pid;	entry->next_prio		= wakee->prio;	entry->next_state		= task_state_index(wakee);	entry->next_cpu			= task_cpu(wakee); 	trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);} static void notraceprobe_wakeup_sched_switch(void *ignore, bool preempt,			  struct task_struct *prev, struct task_struct *next,			  unsigned int prev_state){	struct trace_array_cpu *data;	u64 T0, T1, delta;	unsigned long flags;	long disabled;	int cpu;	unsigned int trace_ctx; 	tracing_record_cmdline(prev); 	if (unlikely(!tracer_enabled))		return; 	/*	 * When we start a new trace, we set wakeup_task to NULL	 * and then set tracer_enabled = 1. We want to make sure	 * that another CPU does not see the tracer_enabled = 1	 * and the wakeup_task with an older task, that might	 * actually be the same as next.	 */	smp_rmb(); 	if (next != wakeup_task)		return; 	/* disable local data, not wakeup_cpu data */	cpu = raw_smp_processor_id();	disabled = local_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);	if (likely(disabled != 1))		goto out; 	local_irq_save(flags);	trace_ctx = tracing_gen_ctx_flags(flags); 	arch_spin_lock(&wakeup_lock); 	/* We could race with grabbing wakeup_lock */	if (unlikely(!tracer_enabled || next != wakeup_task))		goto out_unlock; 	/* The task we are waiting for is waking up */	data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu); 	__trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, trace_ctx);	tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx);	__trace_stack(wakeup_trace, trace_ctx, 0); 	T0 = data->preempt_timestamp;	T1 = ftrace_now(cpu);	delta = T1-T0; 	if (!report_latency(wakeup_trace, delta))		goto out_unlock; 	if (likely(!is_tracing_stopped())) {		wakeup_trace->max_latency = delta;		update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);	}

Lines 1–500 of 830. Load more