// SPDX-License-Identifier: GPL-2.0/* * Fprobe-based tracing events * Copyright (C) 2022 Google LLC. */#define pr_fmt(fmt)	"trace_fprobe: " fmt #include <linux/fprobe.h>#include <linux/module.h>#include <linux/rculist.h>#include <linux/security.h>#include <linux/tracepoint.h>#include <linux/uaccess.h> #include "trace_dynevent.h"#include "trace_probe.h"#include "trace_probe_kernel.h"#include "trace_probe_tmpl.h" #define FPROBE_EVENT_SYSTEM "fprobes"#define TRACEPOINT_EVENT_SYSTEM "tracepoints"#define RETHOOK_MAXACTIVE_MAX 4096 static int trace_fprobe_create(const char *raw_command);static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);static int trace_fprobe_release(struct dyn_event *ev);static bool trace_fprobe_is_busy(struct dyn_event *ev);static bool trace_fprobe_match(const char *system, const char *event,			int argc, const char **argv, struct dyn_event *ev); static struct dyn_event_operations trace_fprobe_ops = {	.create = trace_fprobe_create,	.show = trace_fprobe_show,	.is_busy = trace_fprobe_is_busy,	.free = trace_fprobe_release,	.match = trace_fprobe_match,}; /* * Fprobe event core functions */struct trace_fprobe {	struct dyn_event	devent;	struct fprobe		fp;	const char		*symbol;	struct tracepoint	*tpoint;	struct module		*mod;	struct trace_probe	tp;}; static bool is_trace_fprobe(struct dyn_event *ev){	return ev->ops == &trace_fprobe_ops;} static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev){	return container_of(ev, struct trace_fprobe, devent);} /** * for_each_trace_fprobe - iterate over the trace_fprobe list * @pos:	the struct trace_fprobe * for each entry * @dpos:	the struct dyn_event * to use as a loop cursor */#define for_each_trace_fprobe(pos, dpos)	\	for_each_dyn_event(dpos)		\		if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos))) static bool trace_fprobe_is_return(struct trace_fprobe *tf){	return tf->fp.exit_handler != NULL;} static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf){	return tf->tpoint != NULL;} static const char *trace_fprobe_symbol(struct trace_fprobe *tf){	return tf->symbol ? tf->symbol : "unknown";} static bool trace_fprobe_is_busy(struct dyn_event *ev){	struct trace_fprobe *tf = to_trace_fprobe(ev); 	return trace_probe_is_enabled(&tf->tp);} static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,					    int argc, const char **argv){	char buf[MAX_ARGSTR_LEN + 1]; 	if (!argc)		return true; 	snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));	if (strcmp(buf, argv[0]))		return false;	argc--; argv++; 	return trace_probe_match_command_args(&tf->tp, argc, argv);} static bool trace_fprobe_match(const char *system, const char *event,			int argc, const char **argv, struct dyn_event *ev){	struct trace_fprobe *tf = to_trace_fprobe(ev); 	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))		return false; 	if (system && strcmp(trace_probe_group_name(&tf->tp), system))		return false; 	return trace_fprobe_match_command_head(tf, argc, argv);} static bool trace_fprobe_is_registered(struct trace_fprobe *tf){	return fprobe_is_registered(&tf->fp);} /* * Note that we don't verify the fetch_insn code, since it does not come * from user space. */static intprocess_fetch_insn(struct fetch_insn *code, void *rec, void *dest,		   void *base){	struct pt_regs *regs = rec;	unsigned long val;	int ret; retry:	/* 1st stage: get value from context */	switch (code->op) {	case FETCH_OP_STACK:		val = regs_get_kernel_stack_nth(regs, code->param);		break;	case FETCH_OP_STACKP:		val = kernel_stack_pointer(regs);		break;	case FETCH_OP_RETVAL:		val = regs_return_value(regs);		break;#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API	case FETCH_OP_ARG:		val = regs_get_kernel_argument(regs, code->param);		break;#endif	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */		code++;		goto retry;	default:		ret = process_common_fetch_insn(code, &val);		if (ret < 0)			return ret;	}	code++; 	return process_fetch_insn_bottom(code, val, dest, base);}NOKPROBE_SYMBOL(process_fetch_insn) /* function entry handler */static nokprobe_inline void__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,		    struct pt_regs *regs,		    struct trace_event_file *trace_file){	struct fentry_trace_entry_head *entry;	struct trace_event_call *call = trace_probe_event_call(&tf->tp);	struct trace_event_buffer fbuffer;	int dsize; 	if (WARN_ON_ONCE(call != trace_file->event_call))		return; 	if (trace_trigger_soft_disabled(trace_file))		return; 	dsize = __get_data_size(&tf->tp, regs); 	entry = trace_event_buffer_reserve(&fbuffer, trace_file,					   sizeof(*entry) + tf->tp.size + dsize);	if (!entry)		return; 	fbuffer.regs = regs;	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);	entry->ip = entry_ip;	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 	trace_event_buffer_commit(&fbuffer);} static voidfentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,		  struct pt_regs *regs){	struct event_file_link *link; 	trace_probe_for_each_link_rcu(link, &tf->tp)		__fentry_trace_func(tf, entry_ip, regs, link->file);}NOKPROBE_SYMBOL(fentry_trace_func); /* Kretprobe handler */static nokprobe_inline void__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,		   unsigned long ret_ip, struct pt_regs *regs,		   struct trace_event_file *trace_file){	struct fexit_trace_entry_head *entry;	struct trace_event_buffer fbuffer;	struct trace_event_call *call = trace_probe_event_call(&tf->tp);	int dsize; 	if (WARN_ON_ONCE(call != trace_file->event_call))		return; 	if (trace_trigger_soft_disabled(trace_file))		return; 	dsize = __get_data_size(&tf->tp, regs); 	entry = trace_event_buffer_reserve(&fbuffer, trace_file,					   sizeof(*entry) + tf->tp.size + dsize);	if (!entry)		return; 	fbuffer.regs = regs;	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);	entry->func = entry_ip;	entry->ret_ip = ret_ip;	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize); 	trace_event_buffer_commit(&fbuffer);} static voidfexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,		 unsigned long ret_ip, struct pt_regs *regs){	struct event_file_link *link; 	trace_probe_for_each_link_rcu(link, &tf->tp)		__fexit_trace_func(tf, entry_ip, ret_ip, regs, link->file);}NOKPROBE_SYMBOL(fexit_trace_func); #ifdef CONFIG_PERF_EVENTS static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,			    struct pt_regs *regs){	struct trace_event_call *call = trace_probe_event_call(&tf->tp);	struct fentry_trace_entry_head *entry;	struct hlist_head *head;	int size, __size, dsize;	int rctx; 	head = this_cpu_ptr(call->perf_events);	if (hlist_empty(head))		return 0; 	dsize = __get_data_size(&tf->tp, regs);	__size = sizeof(*entry) + tf->tp.size + dsize;	size = ALIGN(__size + sizeof(u32), sizeof(u64));	size -= sizeof(u32); 	entry = perf_trace_buf_alloc(size, NULL, &rctx);	if (!entry)		return 0; 	entry->ip = entry_ip;	memset(&entry[1], 0, dsize);	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,			      head, NULL);	return 0;}NOKPROBE_SYMBOL(fentry_perf_func); static voidfexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,		unsigned long ret_ip, struct pt_regs *regs){	struct trace_event_call *call = trace_probe_event_call(&tf->tp);	struct fexit_trace_entry_head *entry;	struct hlist_head *head;	int size, __size, dsize;	int rctx; 	head = this_cpu_ptr(call->perf_events);	if (hlist_empty(head))		return; 	dsize = __get_data_size(&tf->tp, regs);	__size = sizeof(*entry) + tf->tp.size + dsize;	size = ALIGN(__size + sizeof(u32), sizeof(u64));	size -= sizeof(u32); 	entry = perf_trace_buf_alloc(size, NULL, &rctx);	if (!entry)		return; 	entry->func = entry_ip;	entry->ret_ip = ret_ip;	store_trace_args(&entry[1], &tf->tp, regs, sizeof(*entry), dsize);	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,			      head, NULL);}NOKPROBE_SYMBOL(fexit_perf_func);#endif	/* CONFIG_PERF_EVENTS */ static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,			     unsigned long ret_ip, struct pt_regs *regs,			     void *entry_data){	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);	int ret = 0; 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))		fentry_trace_func(tf, entry_ip, regs);#ifdef CONFIG_PERF_EVENTS	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))		ret = fentry_perf_func(tf, entry_ip, regs);#endif	return ret;}NOKPROBE_SYMBOL(fentry_dispatcher); static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,			     unsigned long ret_ip, struct pt_regs *regs,			     void *entry_data){	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp); 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))		fexit_trace_func(tf, entry_ip, ret_ip, regs);#ifdef CONFIG_PERF_EVENTS	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))		fexit_perf_func(tf, entry_ip, ret_ip, regs);#endif}NOKPROBE_SYMBOL(fexit_dispatcher); static void free_trace_fprobe(struct trace_fprobe *tf){	if (tf) {		trace_probe_cleanup(&tf->tp);		kfree(tf->symbol);		kfree(tf);	}} /* * Allocate new trace_probe and initialize it (including fprobe). */static struct trace_fprobe *alloc_trace_fprobe(const char *group,					       const char *event,					       const char *symbol,					       struct tracepoint *tpoint,					       int maxactive,					       int nargs, bool is_return){	struct trace_fprobe *tf;	int ret = -ENOMEM; 	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);	if (!tf)		return ERR_PTR(ret); 	tf->symbol = kstrdup(symbol, GFP_KERNEL);	if (!tf->symbol)		goto error; 	if (is_return)		tf->fp.exit_handler = fexit_dispatcher;	else		tf->fp.entry_handler = fentry_dispatcher; 	tf->tpoint = tpoint;	tf->fp.nr_maxactive = maxactive; 	ret = trace_probe_init(&tf->tp, event, group, false);	if (ret < 0)		goto error; 	dyn_event_init(&tf->devent, &trace_fprobe_ops);	return tf;error:	free_trace_fprobe(tf);	return ERR_PTR(ret);} static struct trace_fprobe *find_trace_fprobe(const char *event,					      const char *group){	struct dyn_event *pos;	struct trace_fprobe *tf; 	for_each_trace_fprobe(tf, pos)		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)			return tf;	return NULL;} static inline int __enable_trace_fprobe(struct trace_fprobe *tf){	if (trace_fprobe_is_registered(tf))		enable_fprobe(&tf->fp); 	return 0;} static void __disable_trace_fprobe(struct trace_probe *tp){	struct trace_fprobe *tf; 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {		if (!trace_fprobe_is_registered(tf))			continue;		disable_fprobe(&tf->fp);	}} /* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */static int enable_trace_fprobe(struct trace_event_call *call,			       struct trace_event_file *file){	struct trace_probe *tp;	struct trace_fprobe *tf;	bool enabled;	int ret = 0; 	tp = trace_probe_primary_from_call(call);	if (WARN_ON_ONCE(!tp))		return -ENODEV;	enabled = trace_probe_is_enabled(tp); 	/* This also changes "enabled" state */	if (file) {		ret = trace_probe_add_file(tp, file);		if (ret)			return ret;	} else		trace_probe_set_flag(tp, TP_FLAG_PROFILE); 	if (!enabled) {		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {			/* TODO: check the fprobe is gone */			__enable_trace_fprobe(tf);		}	} 	return 0;} /* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */static int disable_trace_fprobe(struct trace_event_call *call,				struct trace_event_file *file){	struct trace_probe *tp; 	tp = trace_probe_primary_from_call(call);	if (WARN_ON_ONCE(!tp))		return -ENODEV; 	if (file) {		if (!trace_probe_get_file_link(tp, file))			return -ENOENT;		if (!trace_probe_has_single_file(tp))			goto out;		trace_probe_clear_flag(tp, TP_FLAG_TRACE);	} else		trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 	if (!trace_probe_is_enabled(tp))		__disable_trace_fprobe(tp);  out:	if (file)		/*		 * Synchronization is done in below function. For perf event,		 * file == NULL and perf_trace_event_unreg() calls		 * tracepoint_synchronize_unregister() to ensure synchronize

Lines 1–500 of 1199. Load more