// SPDX-License-Identifier: GPL-2.0/* * Kprobes-based tracing events * * Created by Masami Hiramatsu <mhiramat@redhat.com> * */#define pr_fmt(fmt)	"trace_kprobe: " fmt #include <linux/module.h>#include <linux/uaccess.h>#include <linux/rculist.h>#include <linux/error-injection.h> #include <asm/setup.h>  /* for COMMAND_LINE_SIZE */ #include "trace_dynevent.h"#include "trace_kprobe_selftest.h"#include "trace_probe.h"#include "trace_probe_tmpl.h" #define KPROBE_EVENT_SYSTEM "kprobes"#define KRETPROBE_MAXACTIVE_MAX 4096#define MAX_KPROBE_CMDLINE_SIZE 1024 /* Kprobe early definition from command line */static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;static bool kprobe_boot_events_enabled __initdata; static int __init set_kprobe_boot_events(char *str){	strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);	return 0;}__setup("kprobe_event=", set_kprobe_boot_events); static int trace_kprobe_create(int argc, const char **argv);static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);static int trace_kprobe_release(struct dyn_event *ev);static bool trace_kprobe_is_busy(struct dyn_event *ev);static bool trace_kprobe_match(const char *system, const char *event,			       struct dyn_event *ev); static struct dyn_event_operations trace_kprobe_ops = {	.create = trace_kprobe_create,	.show = trace_kprobe_show,	.is_busy = trace_kprobe_is_busy,	.free = trace_kprobe_release,	.match = trace_kprobe_match,}; /* * Kprobe event core functions */struct trace_kprobe {	struct dyn_event	devent;	struct kretprobe	rp;	/* Use rp.kp for kprobe use */	unsigned long __percpu *nhit;	const char		*symbol;	/* symbol name */	struct trace_probe	tp;}; static bool is_trace_kprobe(struct dyn_event *ev){	return ev->ops == &trace_kprobe_ops;} static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev){	return container_of(ev, struct trace_kprobe, devent);} /** * for_each_trace_kprobe - iterate over the trace_kprobe list * @pos:	the struct trace_kprobe * for each entry * @dpos:	the struct dyn_event * to use as a loop cursor */#define for_each_trace_kprobe(pos, dpos)	\	for_each_dyn_event(dpos)		\		if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) #define SIZEOF_TRACE_KPROBE(n)				\	(offsetof(struct trace_kprobe, tp.args) +	\	(sizeof(struct probe_arg) * (n))) static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk){	return tk->rp.handler != NULL;} static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk){	return tk->symbol ? tk->symbol : "unknown";} static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk){	return tk->rp.kp.offset;} static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk){	return !!(kprobe_gone(&tk->rp.kp));} static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,						 struct module *mod){	int len = strlen(mod->name);	const char *name = trace_kprobe_symbol(tk);	return strncmp(mod->name, name, len) == 0 && name[len] == ':';} static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk){	char *p;	bool ret; 	if (!tk->symbol)		return false;	p = strchr(tk->symbol, ':');	if (!p)		return true;	*p = '\0';	mutex_lock(&module_mutex);	ret = !!find_module(tk->symbol);	mutex_unlock(&module_mutex);	*p = ':'; 	return ret;} static bool trace_kprobe_is_busy(struct dyn_event *ev){	struct trace_kprobe *tk = to_trace_kprobe(ev); 	return trace_probe_is_enabled(&tk->tp);} static bool trace_kprobe_match(const char *system, const char *event,			       struct dyn_event *ev){	struct trace_kprobe *tk = to_trace_kprobe(ev); 	return strcmp(trace_probe_name(&tk->tp), event) == 0 &&	    (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0);} static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk){	unsigned long nhit = 0;	int cpu; 	for_each_possible_cpu(cpu)		nhit += *per_cpu_ptr(tk->nhit, cpu); 	return nhit;} static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk){	return !(list_empty(&tk->rp.kp.list) &&		 hlist_unhashed(&tk->rp.kp.hlist));} /* Return 0 if it fails to find the symbol address */static nokprobe_inlineunsigned long trace_kprobe_address(struct trace_kprobe *tk){	unsigned long addr; 	if (tk->symbol) {		addr = (unsigned long)			kallsyms_lookup_name(trace_kprobe_symbol(tk));		if (addr)			addr += tk->rp.kp.offset;	} else {		addr = (unsigned long)tk->rp.kp.addr;	}	return addr;} bool trace_kprobe_on_func_entry(struct trace_event_call *call){	struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 	return kprobe_on_func_entry(tk->rp.kp.addr,			tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,			tk->rp.kp.addr ? 0 : tk->rp.kp.offset);} bool trace_kprobe_error_injectable(struct trace_event_call *call){	struct trace_kprobe *tk = (struct trace_kprobe *)call->data; 	return within_error_injection_list(trace_kprobe_address(tk));} static int register_kprobe_event(struct trace_kprobe *tk);static int unregister_kprobe_event(struct trace_kprobe *tk); static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);static int kretprobe_dispatcher(struct kretprobe_instance *ri,				struct pt_regs *regs); static void free_trace_kprobe(struct trace_kprobe *tk){	if (tk) {		trace_probe_cleanup(&tk->tp);		kfree(tk->symbol);		free_percpu(tk->nhit);		kfree(tk);	}} /* * Allocate new trace_probe and initialize it (including kprobes). */static struct trace_kprobe *alloc_trace_kprobe(const char *group,					     const char *event,					     void *addr,					     const char *symbol,					     unsigned long offs,					     int maxactive,					     int nargs, bool is_return){	struct trace_kprobe *tk;	int ret = -ENOMEM; 	tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);	if (!tk)		return ERR_PTR(ret); 	tk->nhit = alloc_percpu(unsigned long);	if (!tk->nhit)		goto error; 	if (symbol) {		tk->symbol = kstrdup(symbol, GFP_KERNEL);		if (!tk->symbol)			goto error;		tk->rp.kp.symbol_name = tk->symbol;		tk->rp.kp.offset = offs;	} else		tk->rp.kp.addr = addr; 	if (is_return)		tk->rp.handler = kretprobe_dispatcher;	else		tk->rp.kp.pre_handler = kprobe_dispatcher; 	tk->rp.maxactive = maxactive;	INIT_HLIST_NODE(&tk->rp.kp.hlist);	INIT_LIST_HEAD(&tk->rp.kp.list); 	ret = trace_probe_init(&tk->tp, event, group);	if (ret < 0)		goto error; 	dyn_event_init(&tk->devent, &trace_kprobe_ops);	return tk;error:	free_trace_kprobe(tk);	return ERR_PTR(ret);} static struct trace_kprobe *find_trace_kprobe(const char *event,					      const char *group){	struct dyn_event *pos;	struct trace_kprobe *tk; 	for_each_trace_kprobe(tk, pos)		if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&		    strcmp(trace_probe_group_name(&tk->tp), group) == 0)			return tk;	return NULL;} static inline int __enable_trace_kprobe(struct trace_kprobe *tk){	int ret = 0; 	if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {		if (trace_kprobe_is_return(tk))			ret = enable_kretprobe(&tk->rp);		else			ret = enable_kprobe(&tk->rp.kp);	} 	return ret;} /* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */static intenable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file){	bool enabled = trace_probe_is_enabled(&tk->tp);	int ret = 0; 	if (file) {		ret = trace_probe_add_file(&tk->tp, file);		if (ret)			return ret;	} else		trace_probe_set_flag(&tk->tp, TP_FLAG_PROFILE); 	if (enabled)		return 0; 	ret = __enable_trace_kprobe(tk);	if (ret) {		if (file)			trace_probe_remove_file(&tk->tp, file);		else			trace_probe_clear_flag(&tk->tp, TP_FLAG_PROFILE);	} 	return ret;} /* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */static intdisable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file){	struct trace_probe *tp = &tk->tp;	int ret = 0; 	if (file) {		if (!trace_probe_get_file_link(tp, file))			return -ENOENT;		if (!trace_probe_has_single_file(tp))			goto out;		trace_probe_clear_flag(tp, TP_FLAG_TRACE);	} else		trace_probe_clear_flag(tp, TP_FLAG_PROFILE); 	if (!trace_probe_is_enabled(tp) && trace_kprobe_is_registered(tk)) {		if (trace_kprobe_is_return(tk))			disable_kretprobe(&tk->rp);		else			disable_kprobe(&tk->rp.kp);	}  out:	if (file)		/*		 * Synchronization is done in below function. For perf event,		 * file == NULL and perf_trace_event_unreg() calls		 * tracepoint_synchronize_unregister() to ensure synchronize		 * event. We don't need to care about it.		 */		trace_probe_remove_file(tp, file); 	return ret;} #if defined(CONFIG_KPROBES_ON_FTRACE) && \	!defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)static bool within_notrace_func(struct trace_kprobe *tk){	unsigned long offset, size, addr; 	addr = trace_kprobe_address(tk);	if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))		return false; 	/* Get the entry address of the target function */	addr -= offset; 	/*	 * Since ftrace_location_range() does inclusive range check, we need	 * to subtract 1 byte from the end address.	 */	return !ftrace_location_range(addr, addr + size - 1);}#else#define within_notrace_func(tk)	(false)#endif /* Internal register function - just handle k*probes and flags */static int __register_trace_kprobe(struct trace_kprobe *tk){	int i, ret; 	if (trace_kprobe_is_registered(tk))		return -EINVAL; 	if (within_notrace_func(tk)) {		pr_warn("Could not probe notrace function %s\n",			trace_kprobe_symbol(tk));		return -EINVAL;	} 	for (i = 0; i < tk->tp.nr_args; i++) {		ret = traceprobe_update_arg(&tk->tp.args[i]);		if (ret)			return ret;	} 	/* Set/clear disabled flag according to tp->flag */	if (trace_probe_is_enabled(&tk->tp))		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;	else		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; 	if (trace_kprobe_is_return(tk))		ret = register_kretprobe(&tk->rp);	else		ret = register_kprobe(&tk->rp.kp); 	return ret;} /* Internal unregister function - just handle k*probes and flags */static void __unregister_trace_kprobe(struct trace_kprobe *tk){	if (trace_kprobe_is_registered(tk)) {		if (trace_kprobe_is_return(tk))			unregister_kretprobe(&tk->rp);		else			unregister_kprobe(&tk->rp.kp);		/* Cleanup kprobe for reuse and mark it unregistered */		INIT_HLIST_NODE(&tk->rp.kp.hlist);		INIT_LIST_HEAD(&tk->rp.kp.list);		if (tk->rp.kp.symbol_name)			tk->rp.kp.addr = NULL;	}} /* Unregister a trace_probe and probe_event */static int unregister_trace_kprobe(struct trace_kprobe *tk){	/* Enabled event can not be unregistered */	if (trace_probe_is_enabled(&tk->tp))		return -EBUSY; 	/* Will fail if probe is being used by ftrace or perf */	if (unregister_kprobe_event(tk))		return -EBUSY; 	__unregister_trace_kprobe(tk);	dyn_event_remove(&tk->devent); 	return 0;} /* Register a trace_probe and probe_event */static int register_trace_kprobe(struct trace_kprobe *tk){	struct trace_kprobe *old_tk;	int ret; 	mutex_lock(&event_mutex); 	/* Delete old (same name) event if exist */	old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),				   trace_probe_group_name(&tk->tp));	if (old_tk) {		ret = unregister_trace_kprobe(old_tk);		if (ret < 0)			goto end;		free_trace_kprobe(old_tk);	} 	/* Register new event */	ret = register_kprobe_event(tk);	if (ret) {		pr_warn("Failed to register probe event(%d)\n", ret);		goto end;	} 	/* Register k*probe */	ret = __register_trace_kprobe(tk);	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {		pr_warn("This probe might be able to register after target module is loaded. Continue.\n");		ret = 0;	} 	if (ret < 0)		unregister_kprobe_event(tk);	else		dyn_event_add(&tk->devent); end:	mutex_unlock(&event_mutex);	return ret;} /* Module notifier call back, checking event on the module */static int trace_kprobe_module_callback(struct notifier_block *nb,				       unsigned long val, void *data){	struct module *mod = data;

Lines 1–500 of 1687. Load more