// SPDX-License-Identifier: GPL-2.0/* * trace_events_hist - trace event hist triggers * * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> */ #include <linux/module.h>#include <linux/kallsyms.h>#include <linux/security.h>#include <linux/mutex.h>#include <linux/slab.h>#include <linux/stacktrace.h>#include <linux/rculist.h>#include <linux/tracefs.h> /* for gfp flag names */#include <linux/trace_events.h>#include <trace/events/mmflags.h> #include "tracing_map.h"#include "trace_synth.h" #define ERRORS								\	C(NONE,			"No error"),				\	C(DUPLICATE_VAR,	"Variable already defined"),		\	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \	C(TOO_MANY_VARS,	"Too many variables defined"),		\	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\	C(TOO_MANY_PARAMS,	"Too many action params"),		\	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\	C(INVALID_PARAM,	"Invalid action param"),		\	C(ACTION_NOT_FOUND,	"No action found"),			\	C(NO_SAVE_PARAMS,	"No params found for save()"),		\	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\	C(NO_CLOSING_PAREN,	"No closing paren found"),		\	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \	C(DIVISION_BY_ZERO,	"Division by zero"),			\	C(NEED_NOHC_VAL,	"Non-hitcount value is required for 'nohitcount'"), #undef C#define C(a, b)		HIST_ERR_##a enum { ERRORS }; #undef C#define C(a, b)		b static const char *err_text[] = { ERRORS }; struct hist_field; typedef u64 (*hist_field_fn_t) (struct hist_field *field,				struct tracing_map_elt *elt,				struct trace_buffer *buffer,				struct ring_buffer_event *rbe,				void *event); #define HIST_FIELD_OPERANDS_MAX	2#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)#define HIST_ACTIONS_MAX	8#define HIST_CONST_DIGITS_MAX	21#define HIST_DIV_SHIFT		20  /* For optimizing division by constants */ enum field_op_id {	FIELD_OP_NONE,	FIELD_OP_PLUS,	FIELD_OP_MINUS,	FIELD_OP_UNARY_MINUS,	FIELD_OP_DIV,	FIELD_OP_MULT,}; enum hist_field_fn {	HIST_FIELD_FN_NOP,	HIST_FIELD_FN_VAR_REF,	HIST_FIELD_FN_COUNTER,	HIST_FIELD_FN_CONST,	HIST_FIELD_FN_LOG2,	HIST_FIELD_FN_BUCKET,	HIST_FIELD_FN_TIMESTAMP,	HIST_FIELD_FN_CPU,	HIST_FIELD_FN_STRING,	HIST_FIELD_FN_DYNSTRING,	HIST_FIELD_FN_RELDYNSTRING,	HIST_FIELD_FN_PSTRING,	HIST_FIELD_FN_S64,	HIST_FIELD_FN_U64,	HIST_FIELD_FN_S32,	HIST_FIELD_FN_U32,	HIST_FIELD_FN_S16,	HIST_FIELD_FN_U16,	HIST_FIELD_FN_S8,	HIST_FIELD_FN_U8,	HIST_FIELD_FN_UMINUS,	HIST_FIELD_FN_MINUS,	HIST_FIELD_FN_PLUS,	HIST_FIELD_FN_DIV,	HIST_FIELD_FN_MULT,	HIST_FIELD_FN_DIV_POWER2,	HIST_FIELD_FN_DIV_NOT_POWER2,	HIST_FIELD_FN_DIV_MULT_SHIFT,	HIST_FIELD_FN_EXECNAME,	HIST_FIELD_FN_STACK,}; /* * A hist_var (histogram variable) contains variable information for * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF * flag set.  A hist_var has a variable name e.g. ts0, and is * associated with a given histogram trigger, as specified by * hist_data.  The hist_var idx is the unique index assigned to the * variable by the hist trigger's tracing_map.  The idx is what is * used to set a variable's value and, by a variable reference, to * retrieve it. */struct hist_var {	char				*name;	struct hist_trigger_data	*hist_data;	unsigned int			idx;}; struct hist_field {	struct ftrace_event_field	*field;	unsigned long			flags;	unsigned long			buckets;	const char			*type;	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];	struct hist_trigger_data	*hist_data;	enum hist_field_fn		fn_num;	unsigned int			ref;	unsigned int			size;	unsigned int			offset;	unsigned int                    is_signed; 	/*	 * Variable fields contain variable-specific info in var.	 */	struct hist_var			var;	enum field_op_id		operator;	char				*system;	char				*event_name; 	/*	 * The name field is used for EXPR and VAR_REF fields.  VAR	 * fields contain the variable name in var.name.	 */	char				*name; 	/*	 * When a histogram trigger is hit, if it has any references	 * to variables, the values of those variables are collected	 * into a var_ref_vals array by resolve_var_refs().  The	 * current value of each variable is read from the tracing_map	 * using the hist field's hist_var.idx and entered into the	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].	 */	unsigned int			var_ref_idx;	bool                            read_once; 	unsigned int			var_str_idx; 	/* Numeric literals are represented as u64 */	u64				constant;	/* Used to optimize division by constants */	u64				div_multiplier;}; static u64 hist_fn_call(struct hist_field *hist_field,			struct tracing_map_elt *elt,			struct trace_buffer *buffer,			struct ring_buffer_event *rbe,			void *event); static u64 hist_field_const(struct hist_field *field,			   struct tracing_map_elt *elt,			   struct trace_buffer *buffer,			   struct ring_buffer_event *rbe,			   void *event){	return field->constant;} static u64 hist_field_counter(struct hist_field *field,			      struct tracing_map_elt *elt,			      struct trace_buffer *buffer,			      struct ring_buffer_event *rbe,			      void *event){	return 1;} static u64 hist_field_string(struct hist_field *hist_field,			     struct tracing_map_elt *elt,			     struct trace_buffer *buffer,			     struct ring_buffer_event *rbe,			     void *event){	char *addr = (char *)(event + hist_field->field->offset); 	return (u64)(unsigned long)addr;} static u64 hist_field_dynstring(struct hist_field *hist_field,				struct tracing_map_elt *elt,				struct trace_buffer *buffer,				struct ring_buffer_event *rbe,				void *event){	u32 str_item = *(u32 *)(event + hist_field->field->offset);	int str_loc = str_item & 0xffff;	char *addr = (char *)(event + str_loc); 	return (u64)(unsigned long)addr;} static u64 hist_field_reldynstring(struct hist_field *hist_field,				   struct tracing_map_elt *elt,				   struct trace_buffer *buffer,				   struct ring_buffer_event *rbe,				   void *event){	u32 *item = event + hist_field->field->offset;	u32 str_item = *item;	int str_loc = str_item & 0xffff;	char *addr = (char *)&item[1] + str_loc; 	return (u64)(unsigned long)addr;} static u64 hist_field_pstring(struct hist_field *hist_field,			      struct tracing_map_elt *elt,			      struct trace_buffer *buffer,			      struct ring_buffer_event *rbe,			      void *event){	char **addr = (char **)(event + hist_field->field->offset); 	return (u64)(unsigned long)*addr;} static u64 hist_field_log2(struct hist_field *hist_field,			   struct tracing_map_elt *elt,			   struct trace_buffer *buffer,			   struct ring_buffer_event *rbe,			   void *event){	struct hist_field *operand = hist_field->operands[0]; 	u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 	return (u64) ilog2(roundup_pow_of_two(val));} static u64 hist_field_bucket(struct hist_field *hist_field,			     struct tracing_map_elt *elt,			     struct trace_buffer *buffer,			     struct ring_buffer_event *rbe,			     void *event){	struct hist_field *operand = hist_field->operands[0];	unsigned long buckets = hist_field->buckets; 	u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 	if (WARN_ON_ONCE(!buckets))		return val; 	if (val >= LONG_MAX)		val = div64_ul(val, buckets);	else		val = (u64)((unsigned long)val / buckets);	return val * buckets;} static u64 hist_field_plus(struct hist_field *hist_field,			   struct tracing_map_elt *elt,			   struct trace_buffer *buffer,			   struct ring_buffer_event *rbe,			   void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 	return val1 + val2;} static u64 hist_field_minus(struct hist_field *hist_field,			    struct tracing_map_elt *elt,			    struct trace_buffer *buffer,			    struct ring_buffer_event *rbe,			    void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 	return val1 - val2;} static u64 hist_field_div(struct hist_field *hist_field,			   struct tracing_map_elt *elt,			   struct trace_buffer *buffer,			   struct ring_buffer_event *rbe,			   void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 	/* Return -1 for the undefined case */	if (!val2)		return -1; 	/* Use shift if the divisor is a power of 2 */	if (!(val2 & (val2 - 1)))		return val1 >> __ffs64(val2); 	return div64_u64(val1, val2);} static u64 div_by_power_of_two(struct hist_field *hist_field,				struct tracing_map_elt *elt,				struct trace_buffer *buffer,				struct ring_buffer_event *rbe,				void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 	return val1 >> __ffs64(operand2->constant);} static u64 div_by_not_power_of_two(struct hist_field *hist_field,				struct tracing_map_elt *elt,				struct trace_buffer *buffer,				struct ring_buffer_event *rbe,				void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 	return div64_u64(val1, operand2->constant);} static u64 div_by_mult_and_shift(struct hist_field *hist_field,				struct tracing_map_elt *elt,				struct trace_buffer *buffer,				struct ring_buffer_event *rbe,				void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 	/*	 * If the divisor is a constant, do a multiplication and shift instead.	 *	 * Choose Z = some power of 2. If Y <= Z, then:	 *     X / Y = (X * (Z / Y)) / Z	 *	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:	 *     X / Y = (X * mult) / Z	 *	 * The division by Z can be replaced by a shift since Z is a power of 2:	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT	 *	 * As long, as X < Z the results will not be off by more than 1.	 */	if (val1 < (1 << HIST_DIV_SHIFT)) {		u64 mult = operand2->div_multiplier; 		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;	} 	return div64_u64(val1, operand2->constant);} static u64 hist_field_mult(struct hist_field *hist_field,			   struct tracing_map_elt *elt,			   struct trace_buffer *buffer,			   struct ring_buffer_event *rbe,			   void *event){	struct hist_field *operand1 = hist_field->operands[0];	struct hist_field *operand2 = hist_field->operands[1]; 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 	return val1 * val2;} static u64 hist_field_unary_minus(struct hist_field *hist_field,				  struct tracing_map_elt *elt,				  struct trace_buffer *buffer,				  struct ring_buffer_event *rbe,				  void *event){	struct hist_field *operand = hist_field->operands[0]; 	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);	u64 val = (u64)-sval; 	return val;} #define DEFINE_HIST_FIELD_FN(type)					\	static u64 hist_field_##type(struct hist_field *hist_field,	\				     struct tracing_map_elt *elt,	\				     struct trace_buffer *buffer,	\				     struct ring_buffer_event *rbe,	\				     void *event)			\{									\	type *addr = (type *)(event + hist_field->field->offset);	\									\	return (u64)(unsigned long)*addr;				\} DEFINE_HIST_FIELD_FN(s64);DEFINE_HIST_FIELD_FN(u64);DEFINE_HIST_FIELD_FN(s32);DEFINE_HIST_FIELD_FN(u32);DEFINE_HIST_FIELD_FN(s16);DEFINE_HIST_FIELD_FN(u16);DEFINE_HIST_FIELD_FN(s8);DEFINE_HIST_FIELD_FN(u8); #define for_each_hist_field(i, hist_data)	\	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) #define for_each_hist_val_field(i, hist_data)	\	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) #define for_each_hist_key_field(i, hist_data)	\	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) #define HITCOUNT_IDX		0#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) enum hist_field_flags {	HIST_FIELD_FL_HITCOUNT		= 1 << 0,	HIST_FIELD_FL_KEY		= 1 << 1,	HIST_FIELD_FL_STRING		= 1 << 2,	HIST_FIELD_FL_HEX		= 1 << 3,	HIST_FIELD_FL_SYM		= 1 << 4,	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,	HIST_FIELD_FL_EXECNAME		= 1 << 6,	HIST_FIELD_FL_SYSCALL		= 1 << 7,	HIST_FIELD_FL_STACKTRACE	= 1 << 8,	HIST_FIELD_FL_LOG2		= 1 << 9,	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,	HIST_FIELD_FL_VAR		= 1 << 12,

Lines 1–500 of 6855. Load more