// SPDX-License-Identifier: GPL-2.0/* * Kprobes-based tracing events * * Created by Masami Hiramatsu <mhiramat@redhat.com> * */#define pr_fmt(fmt)	"trace_kprobe: " fmt #include <linux/module.h>#include <linux/uaccess.h>#include <linux/rculist.h>#include <linux/error-injection.h> #include "trace_dynevent.h"#include "trace_kprobe_selftest.h"#include "trace_probe.h"#include "trace_probe_tmpl.h" #define KPROBE_EVENT_SYSTEM "kprobes"#define KRETPROBE_MAXACTIVE_MAX 4096 static int trace_kprobe_create(int argc, const char **argv);static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);static int trace_kprobe_release(struct dyn_event *ev);static bool trace_kprobe_is_busy(struct dyn_event *ev);static bool trace_kprobe_match(const char *system, const char *event,			       struct dyn_event *ev); static struct dyn_event_operations trace_kprobe_ops = {	.create = trace_kprobe_create,	.show = trace_kprobe_show,	.is_busy = trace_kprobe_is_busy,	.free = trace_kprobe_release,	.match = trace_kprobe_match,}; /** * Kprobe event core functions */struct trace_kprobe {	struct dyn_event	devent;	struct kretprobe	rp;	/* Use rp.kp for kprobe use */	unsigned long __percpu *nhit;	const char		*symbol;	/* symbol name */	struct trace_probe	tp;}; static bool is_trace_kprobe(struct dyn_event *ev){	return ev->ops == &trace_kprobe_ops;} static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev){	return container_of(ev, struct trace_kprobe, devent);} /** * for_each_trace_kprobe - iterate over the trace_kprobe list * @pos:	the struct trace_kprobe * for each entry * @dpos:	the struct dyn_event * to use as a loop cursor */#define for_each_trace_kprobe(pos, dpos)	\	for_each_dyn_event(dpos)		\		if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) #define SIZEOF_TRACE_KPROBE(n)				\	(offsetof(struct trace_kprobe, tp.args) +	\	(sizeof(struct probe_arg) * (n))) static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk){	return tk->rp.handler != NULL;} static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk){	return tk->symbol ? tk->symbol : "unknown";} static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk){	return tk->rp.kp.offset;} static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk){	return !!(kprobe_gone(&tk->rp.kp));} static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,						 struct module *mod){	int len = strlen(mod->name);	const char *name = trace_kprobe_symbol(tk);	return strncmp(mod->name, name, len) == 0 && name[len] == ':';} static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk){	char *p;	bool ret; 	if (!tk->symbol)		return false;	p = strchr(tk->symbol, ':');	if (!p)		return true;	*p = '\0';	mutex_lock(&module_mutex);	ret = !!find_module(tk->symbol);	mutex_unlock(&module_mutex);	*p = ':'; 	return ret;} static bool trace_kprobe_is_busy(struct dyn_event *ev){	struct trace_kprobe *tk = to_trace_kprobe(ev); 	return trace_probe_is_enabled(&tk->tp);} static bool trace_kprobe_match(const char *system, const char *event,			       struct dyn_event *ev){	struct trace_kprobe *tk = to_trace_kprobe(ev); 	return strcmp(trace_event_name(&tk->tp.call), event) == 0 &&	    (!system || strcmp(tk->tp.call.class->system, system) == 0);} static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk){	unsigned long nhit = 0;	int cpu; 	for_each_possible_cpu(cpu)		nhit += *per_cpu_ptr(tk->nhit, cpu); 	return nhit;} /* Return 0 if it fails to find the symbol address */static nokprobe_inlineunsigned long trace_kprobe_address(struct trace_kprobe *tk){	unsigned long addr; 	if (tk->symbol) {		addr = (unsigned long)			kallsyms_lookup_name(trace_kprobe_symbol(tk));		if (addr)			addr += tk->rp.kp.offset;	} else {		addr = (unsigned long)tk->rp.kp.addr;	}	return addr;} bool trace_kprobe_on_func_entry(struct trace_event_call *call){	struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 	return kprobe_on_func_entry(tk->rp.kp.addr,			tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,			tk->rp.kp.addr ? 0 : tk->rp.kp.offset);} bool trace_kprobe_error_injectable(struct trace_event_call *call){	struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 	return within_error_injection_list(trace_kprobe_address(tk));} static int register_kprobe_event(struct trace_kprobe *tk);static int unregister_kprobe_event(struct trace_kprobe *tk); static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);static int kretprobe_dispatcher(struct kretprobe_instance *ri,				struct pt_regs *regs); /* * Allocate new trace_probe and initialize it (including kprobes). */static struct trace_kprobe *alloc_trace_kprobe(const char *group,					     const char *event,					     void *addr,					     const char *symbol,					     unsigned long offs,					     int maxactive,					     int nargs, bool is_return){	struct trace_kprobe *tk;	int ret = -ENOMEM; 	tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);	if (!tk)		return ERR_PTR(ret); 	tk->nhit = alloc_percpu(unsigned long);	if (!tk->nhit)		goto error; 	if (symbol) {		tk->symbol = kstrdup(symbol, GFP_KERNEL);		if (!tk->symbol)			goto error;		tk->rp.kp.symbol_name = tk->symbol;		tk->rp.kp.offset = offs;	} else		tk->rp.kp.addr = addr; 	if (is_return)		tk->rp.handler = kretprobe_dispatcher;	else		tk->rp.kp.pre_handler = kprobe_dispatcher; 	tk->rp.maxactive = maxactive; 	if (!event || !is_good_name(event)) {		ret = -EINVAL;		goto error;	} 	tk->tp.call.class = &tk->tp.class;	tk->tp.call.name = kstrdup(event, GFP_KERNEL);	if (!tk->tp.call.name)		goto error; 	if (!group || !is_good_name(group)) {		ret = -EINVAL;		goto error;	} 	tk->tp.class.system = kstrdup(group, GFP_KERNEL);	if (!tk->tp.class.system)		goto error; 	dyn_event_init(&tk->devent, &trace_kprobe_ops);	INIT_LIST_HEAD(&tk->tp.files);	return tk;error:	kfree(tk->tp.call.name);	kfree(tk->symbol);	free_percpu(tk->nhit);	kfree(tk);	return ERR_PTR(ret);} static void free_trace_kprobe(struct trace_kprobe *tk){	int i; 	if (!tk)		return; 	for (i = 0; i < tk->tp.nr_args; i++)		traceprobe_free_probe_arg(&tk->tp.args[i]); 	kfree(tk->tp.call.class->system);	kfree(tk->tp.call.name);	kfree(tk->symbol);	free_percpu(tk->nhit);	kfree(tk);} static struct trace_kprobe *find_trace_kprobe(const char *event,					      const char *group){	struct dyn_event *pos;	struct trace_kprobe *tk; 	for_each_trace_kprobe(tk, pos)		if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&		    strcmp(tk->tp.call.class->system, group) == 0)			return tk;	return NULL;} static inline int __enable_trace_kprobe(struct trace_kprobe *tk){	int ret = 0; 	if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {		if (trace_kprobe_is_return(tk))			ret = enable_kretprobe(&tk->rp);		else			ret = enable_kprobe(&tk->rp.kp);	} 	return ret;} /* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */static intenable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file){	struct event_file_link *link;	int ret = 0; 	if (file) {		link = kmalloc(sizeof(*link), GFP_KERNEL);		if (!link) {			ret = -ENOMEM;			goto out;		} 		link->file = file;		list_add_tail_rcu(&link->list, &tk->tp.files); 		tk->tp.flags |= TP_FLAG_TRACE;		ret = __enable_trace_kprobe(tk);		if (ret) {			list_del_rcu(&link->list);			kfree(link);			tk->tp.flags &= ~TP_FLAG_TRACE;		} 	} else {		tk->tp.flags |= TP_FLAG_PROFILE;		ret = __enable_trace_kprobe(tk);		if (ret)			tk->tp.flags &= ~TP_FLAG_PROFILE;	} out:	return ret;} /* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */static intdisable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file){	struct event_file_link *link = NULL;	int wait = 0;	int ret = 0; 	if (file) {		link = find_event_file_link(&tk->tp, file);		if (!link) {			ret = -EINVAL;			goto out;		} 		list_del_rcu(&link->list);		wait = 1;		if (!list_empty(&tk->tp.files))			goto out; 		tk->tp.flags &= ~TP_FLAG_TRACE;	} else		tk->tp.flags &= ~TP_FLAG_PROFILE; 	if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {		if (trace_kprobe_is_return(tk))			disable_kretprobe(&tk->rp);		else			disable_kprobe(&tk->rp.kp);		wait = 1;	} 	/*	 * if tk is not added to any list, it must be a local trace_kprobe	 * created with perf_event_open. We don't need to wait for these	 * trace_kprobes	 */	if (list_empty(&tk->devent.list))		wait = 0; out:	if (wait) {		/*		 * Synchronize with kprobe_trace_func/kretprobe_trace_func		 * to ensure disabled (all running handlers are finished).		 * This is not only for kfree(), but also the caller,		 * trace_remove_event_call() supposes it for releasing		 * event_call related objects, which will be accessed in		 * the kprobe_trace_func/kretprobe_trace_func.		 */		synchronize_rcu();		kfree(link);	/* Ignored if link == NULL */	} 	return ret;} #if defined(CONFIG_KPROBES_ON_FTRACE) && \	!defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)static bool within_notrace_func(struct trace_kprobe *tk){	unsigned long offset, size, addr; 	addr = trace_kprobe_address(tk);	if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))		return false; 	/* Get the entry address of the target function */	addr -= offset; 	/*	 * Since ftrace_location_range() does inclusive range check, we need	 * to subtract 1 byte from the end address.	 */	return !ftrace_location_range(addr, addr + size - 1);}#else#define within_notrace_func(tk)	(false)#endif /* Internal register function - just handle k*probes and flags */static int __register_trace_kprobe(struct trace_kprobe *tk){	int i, ret; 	if (trace_probe_is_registered(&tk->tp))		return -EINVAL; 	if (within_notrace_func(tk)) {		pr_warn("Could not probe notrace function %s\n",			trace_kprobe_symbol(tk));		return -EINVAL;	} 	for (i = 0; i < tk->tp.nr_args; i++) {		ret = traceprobe_update_arg(&tk->tp.args[i]);		if (ret)			return ret;	} 	/* Set/clear disabled flag according to tp->flag */	if (trace_probe_is_enabled(&tk->tp))		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;	else		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 	if (trace_kprobe_is_return(tk))		ret = register_kretprobe(&tk->rp);	else		ret = register_kprobe(&tk->rp.kp); 	if (ret == 0) {		tk->tp.flags |= TP_FLAG_REGISTERED;	} else if (ret == -EILSEQ) {		pr_warn("Probing address(0x%p) is not an instruction boundary.\n",			tk->rp.kp.addr);		ret = -EINVAL;	}	return ret;} /* Internal unregister function - just handle k*probes and flags */static void __unregister_trace_kprobe(struct trace_kprobe *tk){	if (trace_probe_is_registered(&tk->tp)) {		if (trace_kprobe_is_return(tk))			unregister_kretprobe(&tk->rp);		else			unregister_kprobe(&tk->rp.kp);		tk->tp.flags &= ~TP_FLAG_REGISTERED;		/* Cleanup kprobe for reuse */		if (tk->rp.kp.symbol_name)			tk->rp.kp.addr = NULL;	}} /* Unregister a trace_probe and probe_event */static int unregister_trace_kprobe(struct trace_kprobe *tk){	/* Enabled event can not be unregistered */	if (trace_probe_is_enabled(&tk->tp))		return -EBUSY; 	/* Will fail if probe is being used by ftrace or perf */	if (unregister_kprobe_event(tk))		return -EBUSY; 	__unregister_trace_kprobe(tk);	dyn_event_remove(&tk->devent); 	return 0;} /* Register a trace_probe and probe_event */static int register_trace_kprobe(struct trace_kprobe *tk){	struct trace_kprobe *old_tk;	int ret; 	mutex_lock(&event_mutex); 	/* Delete old (same name) event if exist */	old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),

Lines 1–500 of 1630. Load more