// SPDX-License-Identifier: GPL-2.0-or-later/* *  Kernel Probes (KProbes) * * Copyright (C) IBM Corporation, 2002, 2004 * * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel *		Probes initial implementation (includes suggestions from *		Rusty Russell). * 2004-Aug	Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with *		hlists and exceptions notifier as suggested by Andi Kleen. * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes *		interface to access function arguments. * 2004-Sep	Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes *		exceptions notifier to be first on the priority list. * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi *		<prasanna@in.ibm.com> added function-return probes. */ #define pr_fmt(fmt) "kprobes: " fmt #include <linux/kprobes.h>#include <linux/hash.h>#include <linux/init.h>#include <linux/slab.h>#include <linux/stddef.h>#include <linux/export.h>#include <linux/kallsyms.h>#include <linux/freezer.h>#include <linux/seq_file.h>#include <linux/debugfs.h>#include <linux/sysctl.h>#include <linux/kdebug.h>#include <linux/memory.h>#include <linux/ftrace.h>#include <linux/cpu.h>#include <linux/jump_label.h>#include <linux/static_call.h>#include <linux/perf_event.h>#include <linux/execmem.h> #include <asm/sections.h>#include <asm/cacheflush.h>#include <asm/errno.h>#include <linux/uaccess.h> #define KPROBE_HASH_BITS 6#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL)#define kprobe_sysctls_init() do { } while (0)#endif static int kprobes_initialized;/* kprobe_table can be accessed by * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held. * Or * - RCU hlist traversal under disabling preempt (breakpoint handlers) */static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; /* NOTE: change this value only with 'kprobe_mutex' held */static bool kprobes_all_disarmed; /* This protects 'kprobe_table' and 'optimizing_list' */static DEFINE_MUTEX(kprobe_mutex);static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,					unsigned int __unused){	return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));} /* * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where * kprobes can not probe. */static LIST_HEAD(kprobe_blacklist); #ifdef __ARCH_WANT_KPROBES_INSN_SLOT/* * 'kprobe::ainsn.insn' points to the copy of the instruction to be * single-stepped. x86_64, POWER4 and above have no-exec support and * stepping on the instruction on a vmalloced/kmalloced/data page * is a recipe for disaster */struct kprobe_insn_page {	struct list_head list;	kprobe_opcode_t *insns;		/* Page of instruction slots */	struct kprobe_insn_cache *cache;	int nused;	int ngarbage;	char slot_used[];}; static int slots_per_page(struct kprobe_insn_cache *c){	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));} enum kprobe_slot_state {	SLOT_CLEAN = 0,	SLOT_DIRTY = 1,	SLOT_USED = 2,}; void __weak *alloc_insn_page(void){	/*	 * Use execmem_alloc() so this page is within +/- 2GB of where the	 * kernel image and loaded module images reside. This is required	 * for most of the architectures.	 * (e.g. x86-64 needs this to handle the %rip-relative fixups.)	 */	return execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE);} static void free_insn_page(void *page){	execmem_free(page);} struct kprobe_insn_cache kprobe_insn_slots = {	.mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),	.alloc = alloc_insn_page,	.free = free_insn_page,	.sym = KPROBE_INSN_PAGE_SYM,	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),	.insn_size = MAX_INSN_SIZE,	.nr_garbage = 0,};static int collect_garbage_slots(struct kprobe_insn_cache *c); /** * __get_insn_slot() - Find a slot on an executable page for an instruction. * We allocate an executable page if there's no room on existing ones. */kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c){	struct kprobe_insn_page *kip;	kprobe_opcode_t *slot = NULL; 	/* Since the slot array is not protected by rcu, we need a mutex */	mutex_lock(&c->mutex); retry:	rcu_read_lock();	list_for_each_entry_rcu(kip, &c->pages, list) {		if (kip->nused < slots_per_page(c)) {			int i; 			for (i = 0; i < slots_per_page(c); i++) {				if (kip->slot_used[i] == SLOT_CLEAN) {					kip->slot_used[i] = SLOT_USED;					kip->nused++;					slot = kip->insns + (i * c->insn_size);					rcu_read_unlock();					goto out;				}			}			/* kip->nused is broken. Fix it. */			kip->nused = slots_per_page(c);			WARN_ON(1);		}	}	rcu_read_unlock(); 	/* If there are any garbage slots, collect it and try again. */	if (c->nr_garbage && collect_garbage_slots(c) == 0)		goto retry; 	/* All out of space.  Need to allocate a new page. */	kip = kmalloc(struct_size(kip, slot_used, slots_per_page(c)), GFP_KERNEL);	if (!kip)		goto out; 	kip->insns = c->alloc();	if (!kip->insns) {		kfree(kip);		goto out;	}	INIT_LIST_HEAD(&kip->list);	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));	kip->slot_used[0] = SLOT_USED;	kip->nused = 1;	kip->ngarbage = 0;	kip->cache = c;	list_add_rcu(&kip->list, &c->pages);	slot = kip->insns; 	/* Record the perf ksymbol register event after adding the page */	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,			   PAGE_SIZE, false, c->sym);out:	mutex_unlock(&c->mutex);	return slot;} /* Return true if all garbages are collected, otherwise false. */static bool collect_one_slot(struct kprobe_insn_page *kip, int idx){	kip->slot_used[idx] = SLOT_CLEAN;	kip->nused--;	if (kip->nused != 0)		return false; 	/*	 * Page is no longer in use.  Free it unless	 * it's the last one.  We keep the last one	 * so as not to have to set it up again the	 * next time somebody inserts a probe.	 */	if (!list_is_singular(&kip->list)) {		/*		 * Record perf ksymbol unregister event before removing		 * the page.		 */		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,				   (unsigned long)kip->insns, PAGE_SIZE, true,				   kip->cache->sym);		list_del_rcu(&kip->list);		synchronize_rcu();		kip->cache->free(kip->insns);		kfree(kip);	}	return true;} static int collect_garbage_slots(struct kprobe_insn_cache *c){	struct kprobe_insn_page *kip, *next; 	/* Ensure no-one is interrupted on the garbages */	synchronize_rcu(); 	list_for_each_entry_safe(kip, next, &c->pages, list) {		int i; 		if (kip->ngarbage == 0)			continue;		kip->ngarbage = 0;	/* we will collect all garbages */		for (i = 0; i < slots_per_page(c); i++) {			if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))				break;		}	}	c->nr_garbage = 0;	return 0;} void __free_insn_slot(struct kprobe_insn_cache *c,		      kprobe_opcode_t *slot, int dirty){	struct kprobe_insn_page *kip;	long idx; 	mutex_lock(&c->mutex);	rcu_read_lock();	list_for_each_entry_rcu(kip, &c->pages, list) {		idx = ((long)slot - (long)kip->insns) /			(c->insn_size * sizeof(kprobe_opcode_t));		if (idx >= 0 && idx < slots_per_page(c))			goto out;	}	/* Could not find this slot. */	WARN_ON(1);	kip = NULL;out:	rcu_read_unlock();	/* Mark and sweep: this may sleep */	if (kip) {		/* Check double free */		WARN_ON(kip->slot_used[idx] != SLOT_USED);		if (dirty) {			kip->slot_used[idx] = SLOT_DIRTY;			kip->ngarbage++;			if (++c->nr_garbage > slots_per_page(c))				collect_garbage_slots(c);		} else {			collect_one_slot(kip, idx);		}	}	mutex_unlock(&c->mutex);} /* * Check given address is on the page of kprobe instruction slots. * This will be used for checking whether the address on a stack * is on a text area or not. */bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr){	struct kprobe_insn_page *kip;	bool ret = false; 	rcu_read_lock();	list_for_each_entry_rcu(kip, &c->pages, list) {		if (addr >= (unsigned long)kip->insns &&		    addr < (unsigned long)kip->insns + PAGE_SIZE) {			ret = true;			break;		}	}	rcu_read_unlock(); 	return ret;} int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,			     unsigned long *value, char *type, char *sym){	struct kprobe_insn_page *kip;	int ret = -ERANGE; 	rcu_read_lock();	list_for_each_entry_rcu(kip, &c->pages, list) {		if ((*symnum)--)			continue;		strscpy(sym, c->sym, KSYM_NAME_LEN);		*type = 't';		*value = (unsigned long)kip->insns;		ret = 0;		break;	}	rcu_read_unlock(); 	return ret;} #ifdef CONFIG_OPTPROBESvoid __weak *alloc_optinsn_page(void){	return alloc_insn_page();} void __weak free_optinsn_page(void *page){	free_insn_page(page);} /* For optimized_kprobe buffer */struct kprobe_insn_cache kprobe_optinsn_slots = {	.mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),	.alloc = alloc_optinsn_page,	.free = free_optinsn_page,	.sym = KPROBE_OPTINSN_PAGE_SYM,	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),	/* .insn_size is initialized later */	.nr_garbage = 0,};#endif /* CONFIG_OPTPROBES */#endif /* __ARCH_WANT_KPROBES_INSN_SLOT */ /* We have preemption disabled.. so it is safe to use __ versions */static inline void set_kprobe_instance(struct kprobe *kp){	__this_cpu_write(kprobe_instance, kp);} static inline void reset_kprobe_instance(void){	__this_cpu_write(kprobe_instance, NULL);} /* * This routine is called either: *	- under the 'kprobe_mutex' - during kprobe_[un]register(). *				OR *	- with preemption disabled - from architecture specific code. */struct kprobe *get_kprobe(void *addr){	struct hlist_head *head;	struct kprobe *p; 	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];	hlist_for_each_entry_rcu(p, head, hlist,				 lockdep_is_held(&kprobe_mutex)) {		if (p->addr == addr)			return p;	} 	return NULL;}NOKPROBE_SYMBOL(get_kprobe); static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); /* Return true if 'p' is an aggregator */static inline bool kprobe_aggrprobe(struct kprobe *p){	return p->pre_handler == aggr_pre_handler;} /* Return true if 'p' is unused */static inline bool kprobe_unused(struct kprobe *p){	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&	       list_empty(&p->list);} /* Keep all fields in the kprobe consistent. */static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p){	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));} #ifdef CONFIG_OPTPROBES/* NOTE: This is protected by 'kprobe_mutex'. */static bool kprobes_allow_optimization; /* * Call all 'kprobe::pre_handler' on the list, but ignores its return value. * This must be called from arch-dep optimized caller. */void opt_pre_handler(struct kprobe *p, struct pt_regs *regs){	struct kprobe *kp; 	list_for_each_entry_rcu(kp, &p->list, list) {		if (kp->pre_handler && likely(!kprobe_disabled(kp))) {			set_kprobe_instance(kp);			kp->pre_handler(kp, regs);		}		reset_kprobe_instance();	}}NOKPROBE_SYMBOL(opt_pre_handler); /* Free optimized instructions and optimized_kprobe */static void free_aggr_kprobe(struct kprobe *p){	struct optimized_kprobe *op; 	op = container_of(p, struct optimized_kprobe, kp);	arch_remove_optimized_kprobe(op);	arch_remove_kprobe(p);	kfree(op);} /* Return true if the kprobe is ready for optimization. */static inline int kprobe_optready(struct kprobe *p){	struct optimized_kprobe *op; 	if (kprobe_aggrprobe(p)) {		op = container_of(p, struct optimized_kprobe, kp);		return arch_prepared_optinsn(&op->optinsn);	} 	return 0;} /* Return true if the kprobe is disarmed. Note: p must be on hash list */bool kprobe_disarmed(struct kprobe *p){	struct optimized_kprobe *op; 	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */	if (!kprobe_aggrprobe(p))		return kprobe_disabled(p); 	op = container_of(p, struct optimized_kprobe, kp); 	return kprobe_disabled(p) && list_empty(&op->list);} /* Return true if the probe is queued on (un)optimizing lists */static bool kprobe_queued(struct kprobe *p){	struct optimized_kprobe *op; 	if (kprobe_aggrprobe(p)) {		op = container_of(p, struct optimized_kprobe, kp);		if (!list_empty(&op->list))			return true;	}	return false;} /* * Return an optimized kprobe whose optimizing code replaces * instructions including 'addr' (exclude breakpoint). */static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr){	int i;	struct kprobe *p = NULL;	struct optimized_kprobe *op; 	/* Don't check i == 0, since that is a breakpoint case. */	for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)		p = get_kprobe(addr - i); 	if (p && kprobe_optready(p)) {		op = container_of(p, struct optimized_kprobe, kp);		if (arch_within_optimized_kprobe(op, addr))			return p;

Lines 1–500 of 3049. Load more