// SPDX-License-Identifier: GPL-2.0/* * Infrastructure to took into function calls and returns. * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> * Mostly borrowed from function tracer which * is Copyright (c) Steven Rostedt <srostedt@redhat.com> * * Highly modified by Steven Rostedt (VMware). */#include <linux/bits.h>#include <linux/jump_label.h>#include <linux/suspend.h>#include <linux/ftrace.h>#include <linux/static_call.h>#include <linux/slab.h> #include <trace/events/sched.h> #include "ftrace_internal.h"#include "trace.h" /* * FGRAPH_FRAME_SIZE:	Size in bytes of the meta data on the shadow stack * FGRAPH_FRAME_OFFSET:	Size in long words of the meta data frame */#define FGRAPH_FRAME_SIZE	sizeof(struct ftrace_ret_stack)#define FGRAPH_FRAME_OFFSET	DIV_ROUND_UP(FGRAPH_FRAME_SIZE, sizeof(long)) /* * On entry to a function (via function_graph_enter()), a new fgraph frame * (ftrace_ret_stack) is pushed onto the stack as well as a word that * holds a bitmask and a type (called "bitmap"). The bitmap is defined as: * * bits:  0 -  9	offset in words from the previous ftrace_ret_stack * * bits: 10 - 11	Type of storage *			  0 - reserved *			  1 - bitmap of fgraph_array index *			  2 - reserved data * * For type with "bitmap of fgraph_array index" (FGRAPH_TYPE_BITMAP): *  bits: 12 - 27	The bitmap of fgraph_ops fgraph_array index *			That is, it's a bitmask of 0-15 (16 bits) *			where if a corresponding ops in the fgraph_array[] *			expects a callback from the return of the function *			it's corresponding bit will be set. * * * The top of the ret_stack (when not empty) will always have a reference * word that points to the last fgraph frame that was saved. * * For reserved data: *  bits: 12 - 17	The size in words that is stored *  bits: 18 - 23	The index of fgraph_array, which shows who is stored * * That is, at the end of function_graph_enter, if the first and forth * fgraph_ops on the fgraph_array[] (index 0 and 3) needs their retfunc called * on the return of the function being traced, and the forth fgraph_ops * stored two words of data, this is what will be on the task's shadow * ret_stack: (the stack grows upward) * *  ret_stack[SHADOW_STACK_OFFSET] * | SHADOW_STACK_TASK_VARS(ret_stack)[15]      | * ... * | SHADOW_STACK_TASK_VARS(ret_stack)[0]       | *  ret_stack[SHADOW_STACK_MAX_OFFSET] * ... * |                                            | <- task->curr_ret_stack * +--------------------------------------------+ * | (3 << 12) | (3 << 10) | FGRAPH_FRAME_OFFSET| * |         *or put another way*               | * | (3 << FGRAPH_DATA_INDEX_SHIFT)| \          | This is for fgraph_ops[3]. * | ((2 - 1) << FGRAPH_DATA_SHIFT)| \          | The data size is 2 words. * | (FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT)| \ | * | (offset2:FGRAPH_FRAME_OFFSET+3)            | <- the offset2 is from here * +--------------------------------------------+ ( It is 4 words from the ret_stack) * |            STORED DATA WORD 2              | * |            STORED DATA WORD 1              | * +--------------------------------------------+ * | (9 << 12) | (1 << 10) | FGRAPH_FRAME_OFFSET| * |         *or put another way*               | * | (BIT(3)|BIT(0)) << FGRAPH_INDEX_SHIFT | \  | * | FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT| \ | * | (offset1:FGRAPH_FRAME_OFFSET)              | <- the offset1 is from here * +--------------------------------------------+ * | struct ftrace_ret_stack                    | * |   (stores the saved ret pointer)           | <- the offset points here * +--------------------------------------------+ * |                 (X) | (N)                  | ( N words away from * |                                            |   previous ret_stack) * ... * ret_stack[0] * * If a backtrace is required, and the real return pointer needs to be * fetched, then it looks at the task's curr_ret_stack offset, if it * is greater than zero (reserved, or right before popped), it would mask * the value by FGRAPH_FRAME_OFFSET_MASK to get the offset of the * ftrace_ret_stack structure stored on the shadow stack. */ /* * The following is for the top word on the stack: * *   FGRAPH_FRAME_OFFSET (0-9) holds the offset delta to the fgraph frame *   FGRAPH_TYPE (10-11) holds the type of word this is. *     (RESERVED or BITMAP) */#define FGRAPH_FRAME_OFFSET_BITS	10#define FGRAPH_FRAME_OFFSET_MASK	GENMASK(FGRAPH_FRAME_OFFSET_BITS - 1, 0) #define FGRAPH_TYPE_BITS	2#define FGRAPH_TYPE_MASK	GENMASK(FGRAPH_TYPE_BITS - 1, 0)#define FGRAPH_TYPE_SHIFT	FGRAPH_FRAME_OFFSET_BITS enum {	FGRAPH_TYPE_RESERVED	= 0,	FGRAPH_TYPE_BITMAP	= 1,	FGRAPH_TYPE_DATA	= 2,}; /* * For BITMAP type: *   FGRAPH_INDEX (12-27) bits holding the gops index wanting return callback called */#define FGRAPH_INDEX_BITS	16#define FGRAPH_INDEX_MASK	GENMASK(FGRAPH_INDEX_BITS - 1, 0)#define FGRAPH_INDEX_SHIFT	(FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS) /* * For DATA type: *  FGRAPH_DATA (12-17) bits hold the size of data (in words) *  FGRAPH_INDEX (18-23) bits hold the index for which gops->idx the data is for * * Note: *  data_size == 0 means 1 word, and 31 (=2^5 - 1) means 32 words. */#define FGRAPH_DATA_BITS	5#define FGRAPH_DATA_MASK	GENMASK(FGRAPH_DATA_BITS - 1, 0)#define FGRAPH_DATA_SHIFT	(FGRAPH_TYPE_SHIFT + FGRAPH_TYPE_BITS)#define FGRAPH_MAX_DATA_SIZE (sizeof(long) * (1 << FGRAPH_DATA_BITS)) #define FGRAPH_DATA_INDEX_BITS	4#define FGRAPH_DATA_INDEX_MASK	GENMASK(FGRAPH_DATA_INDEX_BITS - 1, 0)#define FGRAPH_DATA_INDEX_SHIFT	(FGRAPH_DATA_SHIFT + FGRAPH_DATA_BITS) #define FGRAPH_MAX_INDEX	\	((FGRAPH_INDEX_SIZE << FGRAPH_DATA_BITS) + FGRAPH_RET_INDEX) #define FGRAPH_ARRAY_SIZE	FGRAPH_INDEX_BITS /* * SHADOW_STACK_SIZE:	The size in bytes of the entire shadow stack * SHADOW_STACK_OFFSET:	The size in long words of the shadow stack * SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added */#define SHADOW_STACK_SIZE	(PAGE_SIZE)#define SHADOW_STACK_OFFSET	(SHADOW_STACK_SIZE / sizeof(long))/* Leave on a buffer at the end */#define SHADOW_STACK_MAX_OFFSET				\	(SHADOW_STACK_OFFSET - (FGRAPH_FRAME_OFFSET + 1 + FGRAPH_ARRAY_SIZE)) /* RET_STACK():		Return the frame from a given @offset from task @t */#define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset])) /* * Each fgraph_ops has a reservered unsigned long at the end (top) of the * ret_stack to store task specific state. */#define SHADOW_STACK_TASK_VARS(ret_stack) \	((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE])) DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph);int ftrace_graph_active; static struct fgraph_ops *fgraph_array[FGRAPH_ARRAY_SIZE];static unsigned long fgraph_array_bitmask; /* LRU index table for fgraph_array */static int fgraph_lru_table[FGRAPH_ARRAY_SIZE];static int fgraph_lru_next;static int fgraph_lru_last; /* Initialize fgraph_lru_table with unused index */static void fgraph_lru_init(void){	int i; 	for (i = 0; i < FGRAPH_ARRAY_SIZE; i++)		fgraph_lru_table[i] = i;} /* Release the used index to the LRU table */static int fgraph_lru_release_index(int idx){	if (idx < 0 || idx >= FGRAPH_ARRAY_SIZE ||	    WARN_ON_ONCE(fgraph_lru_table[fgraph_lru_last] != -1))		return -1; 	fgraph_lru_table[fgraph_lru_last] = idx;	fgraph_lru_last = (fgraph_lru_last + 1) % FGRAPH_ARRAY_SIZE; 	clear_bit(idx, &fgraph_array_bitmask);	return 0;} /* Allocate a new index from LRU table */static int fgraph_lru_alloc_index(void){	int idx = fgraph_lru_table[fgraph_lru_next]; 	/* No id is available */	if (idx == -1)		return -1; 	fgraph_lru_table[fgraph_lru_next] = -1;	fgraph_lru_next = (fgraph_lru_next + 1) % FGRAPH_ARRAY_SIZE; 	set_bit(idx, &fgraph_array_bitmask);	return idx;} /* Get the offset to the fgraph frame from a ret_stack value */static inline int __get_offset(unsigned long val){	return val & FGRAPH_FRAME_OFFSET_MASK;} /* Get the type of word from a ret_stack value */static inline int __get_type(unsigned long val){	return (val >> FGRAPH_TYPE_SHIFT) & FGRAPH_TYPE_MASK;} /* Get the data_index for a DATA type ret_stack word */static inline int __get_data_index(unsigned long val){	return (val >> FGRAPH_DATA_INDEX_SHIFT) & FGRAPH_DATA_INDEX_MASK;} /* Get the data_size for a DATA type ret_stack word */static inline int __get_data_size(unsigned long val){	return ((val >> FGRAPH_DATA_SHIFT) & FGRAPH_DATA_MASK) + 1;} /* Get the word from the ret_stack at @offset */static inline unsigned long get_fgraph_entry(struct task_struct *t, int offset){	return t->ret_stack[offset];} /* Get the FRAME_OFFSET from the word from the @offset on ret_stack */static inline int get_frame_offset(struct task_struct *t, int offset){	return __get_offset(t->ret_stack[offset]);} /* For BITMAP type: get the bitmask from the @offset at ret_stack */static inline unsigned longget_bitmap_bits(struct task_struct *t, int offset){	return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK;} /* Write the bitmap to the ret_stack at @offset (does index, offset and bitmask) */static inline voidset_bitmap(struct task_struct *t, int offset, unsigned long bitmap){	t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) |		(FGRAPH_TYPE_BITMAP << FGRAPH_TYPE_SHIFT) | FGRAPH_FRAME_OFFSET;} /* For DATA type: get the data saved under the ret_stack word at @offset */static inline void *get_data_type_data(struct task_struct *t, int offset){	unsigned long val = t->ret_stack[offset]; 	if (__get_type(val) != FGRAPH_TYPE_DATA)		return NULL;	offset -= __get_data_size(val);	return (void *)&t->ret_stack[offset];} /* Create the ret_stack word for a DATA type */static inline unsigned long make_data_type_val(int idx, int size, int offset){	return (idx << FGRAPH_DATA_INDEX_SHIFT) |		((size - 1) << FGRAPH_DATA_SHIFT) |		(FGRAPH_TYPE_DATA << FGRAPH_TYPE_SHIFT) | offset;} /* ftrace_graph_entry set to this to tell some archs to run function graph */static int entry_run(struct ftrace_graph_ent *trace, struct fgraph_ops *ops){	return 0;} /* ftrace_graph_return set to this to tell some archs to run function graph */static void return_run(struct ftrace_graph_ret *trace, struct fgraph_ops *ops){} static void ret_stack_set_task_var(struct task_struct *t, int idx, long val){	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack); 	gvals[idx] = val;} static unsigned long *ret_stack_get_task_var(struct task_struct *t, int idx){	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack); 	return &gvals[idx];} static void ret_stack_init_task_vars(unsigned long *ret_stack){	unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack); 	memset(gvals, 0, sizeof(*gvals) * FGRAPH_ARRAY_SIZE);} /** * fgraph_reserve_data - Reserve storage on the task's ret_stack * @idx:	The index of fgraph_array * @size_bytes: The size in bytes to reserve * * Reserves space of up to FGRAPH_MAX_DATA_SIZE bytes on the * task's ret_stack shadow stack, for a given fgraph_ops during * the entryfunc() call. If entryfunc() returns zero, the storage * is discarded. An entryfunc() can only call this once per iteration. * The fgraph_ops retfunc() can retrieve this stored data with * fgraph_retrieve_data(). * * Returns: On success, a pointer to the data on the stack. *   Otherwise, NULL if there's not enough space left on the *   ret_stack for the data, or if fgraph_reserve_data() was called *   more than once for a single entryfunc() call. */void *fgraph_reserve_data(int idx, int size_bytes){	unsigned long val;	void *data;	int curr_ret_stack = current->curr_ret_stack;	int data_size; 	if (size_bytes > FGRAPH_MAX_DATA_SIZE)		return NULL; 	/* Convert the data size to number of longs. */	data_size = (size_bytes + sizeof(long) - 1) >> (sizeof(long) == 4 ? 2 : 3); 	val = get_fgraph_entry(current, curr_ret_stack - 1);	data = &current->ret_stack[curr_ret_stack]; 	curr_ret_stack += data_size + 1;	if (unlikely(curr_ret_stack >= SHADOW_STACK_MAX_OFFSET))		return NULL; 	val = make_data_type_val(idx, data_size, __get_offset(val) + data_size + 1); 	/* Set the last word to be reserved */	current->ret_stack[curr_ret_stack - 1] = val; 	/* Make sure interrupts see this */	barrier();	current->curr_ret_stack = curr_ret_stack;	/* Again sync with interrupts, and reset reserve */	current->ret_stack[curr_ret_stack - 1] = val; 	return data;} /** * fgraph_retrieve_data - Retrieve stored data from fgraph_reserve_data() * @idx:	the index of fgraph_array (fgraph_ops::idx) * @size_bytes: pointer to retrieved data size. * * This is to be called by a fgraph_ops retfunc(), to retrieve data that * was stored by the fgraph_ops entryfunc() on the function entry. * That is, this will retrieve the data that was reserved on the * entry of the function that corresponds to the exit of the function * that the fgraph_ops retfunc() is called on. * * Returns: The stored data from fgraph_reserve_data() called by the *    matching entryfunc() for the retfunc() this is called from. *   Or NULL if there was nothing stored. */void *fgraph_retrieve_data(int idx, int *size_bytes){	int offset = current->curr_ret_stack - 1;	unsigned long val; 	val = get_fgraph_entry(current, offset);	while (__get_type(val) == FGRAPH_TYPE_DATA) {		if (__get_data_index(val) == idx)			goto found;		offset -= __get_data_size(val) + 1;		val = get_fgraph_entry(current, offset);	}	return NULL;found:	if (size_bytes)		*size_bytes = __get_data_size(val) * sizeof(long);	return get_data_type_data(current, offset);} /** * fgraph_get_task_var - retrieve a task specific state variable * @gops: The ftrace_ops that owns the task specific variable * * Every registered fgraph_ops has a task state variable * reserved on the task's ret_stack. This function returns the * address to that variable. * * Returns the address to the fgraph_ops @gops tasks specific * unsigned long variable. */unsigned long *fgraph_get_task_var(struct fgraph_ops *gops){	return ret_stack_get_task_var(current, gops->idx);} /* * @offset: The offset into @t->ret_stack to find the ret_stack entry * @frame_offset: Where to place the offset into @t->ret_stack of that entry * * Returns a pointer to the previous ret_stack below @offset or NULL *   when it reaches the bottom of the stack. * * Calling this with: * *   offset = task->curr_ret_stack; *   do { *	ret_stack = get_ret_stack(task, offset, &offset); *   } while (ret_stack); * * Will iterate through all the ret_stack entries from curr_ret_stack * down to the first one. */static inline struct ftrace_ret_stack *get_ret_stack(struct task_struct *t, int offset, int *frame_offset){	int offs; 	BUILD_BUG_ON(FGRAPH_FRAME_SIZE % sizeof(long)); 	if (unlikely(offset <= 0))		return NULL; 	offs = get_frame_offset(t, --offset);	if (WARN_ON_ONCE(offs <= 0 || offs > offset))		return NULL; 	offset -= offs; 	*frame_offset = offset;	return RET_STACK(t, offset);} /* Both enabled by default (can be cleared by function_graph tracer flags */static bool fgraph_sleep_time = true; #ifdef CONFIG_DYNAMIC_FTRACE/* * archs can override this function if they must do something * to enable hook for graph tracer. */int __weak ftrace_enable_ftrace_graph_caller(void){	return 0;} /* * archs can override this function if they must do something * to disable hook for graph tracer. */int __weak ftrace_disable_ftrace_graph_caller(void){	return 0;}#endif int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace,			    struct fgraph_ops *gops){	return 0;} static void ftrace_graph_ret_stub(struct ftrace_graph_ret *trace,				  struct fgraph_ops *gops){} static struct fgraph_ops fgraph_stub = {	.entryfunc = ftrace_graph_entry_stub,	.retfunc = ftrace_graph_ret_stub,};

Lines 1–500 of 1356. Load more