/* *  Kernel Probes (KProbes) *  kernel/kprobes.c * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) IBM Corporation, 2002, 2004 * * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel *		Probes initial implementation (includes suggestions from *		Rusty Russell). * 2004-Aug	Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with *		hlists and exceptions notifier as suggested by Andi Kleen. * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes *		interface to access function arguments. * 2004-Sep	Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes *		exceptions notifier to be first on the priority list. * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi *		<prasanna@in.ibm.com> added function-return probes. */#include <linux/kprobes.h>#include <linux/hash.h>#include <linux/init.h>#include <linux/slab.h>#include <linux/stddef.h>#include <linux/export.h>#include <linux/moduleloader.h>#include <linux/kallsyms.h>#include <linux/freezer.h>#include <linux/seq_file.h>#include <linux/debugfs.h>#include <linux/sysctl.h>#include <linux/kdebug.h>#include <linux/memory.h>#include <linux/ftrace.h>#include <linux/cpu.h>#include <linux/jump_label.h> #include <asm-generic/sections.h>#include <asm/cacheflush.h>#include <asm/errno.h>#include <asm/uaccess.h> #define KPROBE_HASH_BITS 6#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)  /* * Some oddball architectures like 64bit powerpc have function descriptors * so this must be overridable. */#ifndef kprobe_lookup_name#define kprobe_lookup_name(name, addr) \	addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))#endif static int kprobes_initialized;static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; /* NOTE: change this value only with kprobe_mutex held */static bool kprobes_all_disarmed; /* This protects kprobe_table and optimizing_list */static DEFINE_MUTEX(kprobe_mutex);static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;static struct {	raw_spinlock_t lock ____cacheline_aligned_in_smp;} kretprobe_table_locks[KPROBE_TABLE_SIZE]; static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash){	return &(kretprobe_table_locks[hash].lock);} /* * Normally, functions that we'd want to prohibit kprobes in, are marked * __kprobes. But, there are cases where such functions already belong to * a different section (__sched for preempt_schedule) * * For such cases, we now have a blacklist */static struct kprobe_blackpoint kprobe_blacklist[] = {	{"preempt_schedule",},	{"native_get_debugreg",},	{"irq_entries_start",},	{"common_interrupt",},	{"mcount",},	/* mcount can be called from everywhere */	{NULL}    /* Terminator */}; #ifdef __ARCH_WANT_KPROBES_INSN_SLOT/* * kprobe->ainsn.insn points to the copy of the instruction to be * single-stepped. x86_64, POWER4 and above have no-exec support and * stepping on the instruction on a vmalloced/kmalloced/data page * is a recipe for disaster */struct kprobe_insn_page {	struct list_head list;	kprobe_opcode_t *insns;		/* Page of instruction slots */	int nused;	int ngarbage;	char slot_used[];}; #define KPROBE_INSN_PAGE_SIZE(slots)			\	(offsetof(struct kprobe_insn_page, slot_used) +	\	 (sizeof(char) * (slots))) struct kprobe_insn_cache {	struct list_head pages;	/* list of kprobe_insn_page */	size_t insn_size;	/* size of instruction slot */	int nr_garbage;}; static int slots_per_page(struct kprobe_insn_cache *c){	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));} enum kprobe_slot_state {	SLOT_CLEAN = 0,	SLOT_DIRTY = 1,	SLOT_USED = 2,}; static DEFINE_MUTEX(kprobe_insn_mutex);	/* Protects kprobe_insn_slots */static struct kprobe_insn_cache kprobe_insn_slots = {	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),	.insn_size = MAX_INSN_SIZE,	.nr_garbage = 0,};static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c); /** * __get_insn_slot() - Find a slot on an executable page for an instruction. * We allocate an executable page if there's no room on existing ones. */static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c){	struct kprobe_insn_page *kip;  retry:	list_for_each_entry(kip, &c->pages, list) {		if (kip->nused < slots_per_page(c)) {			int i;			for (i = 0; i < slots_per_page(c); i++) {				if (kip->slot_used[i] == SLOT_CLEAN) {					kip->slot_used[i] = SLOT_USED;					kip->nused++;					return kip->insns + (i * c->insn_size);				}			}			/* kip->nused is broken. Fix it. */			kip->nused = slots_per_page(c);			WARN_ON(1);		}	} 	/* If there are any garbage slots, collect it and try again. */	if (c->nr_garbage && collect_garbage_slots(c) == 0)		goto retry; 	/* All out of space.  Need to allocate a new page. */	kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);	if (!kip)		return NULL; 	/*	 * Use module_alloc so this page is within +/- 2GB of where the	 * kernel image and loaded module images reside. This is required	 * so x86_64 can correctly handle the %rip-relative fixups.	 */	kip->insns = module_alloc(PAGE_SIZE);	if (!kip->insns) {		kfree(kip);		return NULL;	}	INIT_LIST_HEAD(&kip->list);	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));	kip->slot_used[0] = SLOT_USED;	kip->nused = 1;	kip->ngarbage = 0;	list_add(&kip->list, &c->pages);	return kip->insns;}  kprobe_opcode_t __kprobes *get_insn_slot(void){	kprobe_opcode_t *ret = NULL; 	mutex_lock(&kprobe_insn_mutex);	ret = __get_insn_slot(&kprobe_insn_slots);	mutex_unlock(&kprobe_insn_mutex); 	return ret;} /* Return 1 if all garbages are collected, otherwise 0. */static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx){	kip->slot_used[idx] = SLOT_CLEAN;	kip->nused--;	if (kip->nused == 0) {		/*		 * Page is no longer in use.  Free it unless		 * it's the last one.  We keep the last one		 * so as not to have to set it up again the		 * next time somebody inserts a probe.		 */		if (!list_is_singular(&kip->list)) {			list_del(&kip->list);			module_free(NULL, kip->insns);			kfree(kip);		}		return 1;	}	return 0;} static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c){	struct kprobe_insn_page *kip, *next; 	/* Ensure no-one is interrupted on the garbages */	synchronize_sched(); 	list_for_each_entry_safe(kip, next, &c->pages, list) {		int i;		if (kip->ngarbage == 0)			continue;		kip->ngarbage = 0;	/* we will collect all garbages */		for (i = 0; i < slots_per_page(c); i++) {			if (kip->slot_used[i] == SLOT_DIRTY &&			    collect_one_slot(kip, i))				break;		}	}	c->nr_garbage = 0;	return 0;} static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,				       kprobe_opcode_t *slot, int dirty){	struct kprobe_insn_page *kip; 	list_for_each_entry(kip, &c->pages, list) {		long idx = ((long)slot - (long)kip->insns) /				(c->insn_size * sizeof(kprobe_opcode_t));		if (idx >= 0 && idx < slots_per_page(c)) {			WARN_ON(kip->slot_used[idx] != SLOT_USED);			if (dirty) {				kip->slot_used[idx] = SLOT_DIRTY;				kip->ngarbage++;				if (++c->nr_garbage > slots_per_page(c))					collect_garbage_slots(c);			} else				collect_one_slot(kip, idx);			return;		}	}	/* Could not free this slot. */	WARN_ON(1);} void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty){	mutex_lock(&kprobe_insn_mutex);	__free_insn_slot(&kprobe_insn_slots, slot, dirty);	mutex_unlock(&kprobe_insn_mutex);}#ifdef CONFIG_OPTPROBES/* For optimized_kprobe buffer */static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */static struct kprobe_insn_cache kprobe_optinsn_slots = {	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),	/* .insn_size is initialized later */	.nr_garbage = 0,};/* Get a slot for optimized_kprobe buffer */kprobe_opcode_t __kprobes *get_optinsn_slot(void){	kprobe_opcode_t *ret = NULL; 	mutex_lock(&kprobe_optinsn_mutex);	ret = __get_insn_slot(&kprobe_optinsn_slots);	mutex_unlock(&kprobe_optinsn_mutex); 	return ret;} void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty){	mutex_lock(&kprobe_optinsn_mutex);	__free_insn_slot(&kprobe_optinsn_slots, slot, dirty);	mutex_unlock(&kprobe_optinsn_mutex);}#endif#endif /* We have preemption disabled.. so it is safe to use __ versions */static inline void set_kprobe_instance(struct kprobe *kp){	__this_cpu_write(kprobe_instance, kp);} static inline void reset_kprobe_instance(void){	__this_cpu_write(kprobe_instance, NULL);} /* * This routine is called either: * 	- under the kprobe_mutex - during kprobe_[un]register() * 				OR * 	- with preemption disabled - from arch/xxx/kernel/kprobes.c */struct kprobe __kprobes *get_kprobe(void *addr){	struct hlist_head *head;	struct hlist_node *node;	struct kprobe *p; 	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];	hlist_for_each_entry_rcu(p, node, head, hlist) {		if (p->addr == addr)			return p;	} 	return NULL;} static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); /* Return true if the kprobe is an aggregator */static inline int kprobe_aggrprobe(struct kprobe *p){	return p->pre_handler == aggr_pre_handler;} /* Return true(!0) if the kprobe is unused */static inline int kprobe_unused(struct kprobe *p){	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&	       list_empty(&p->list);} /* * Keep all fields in the kprobe consistent */static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p){	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));} #ifdef CONFIG_OPTPROBES/* NOTE: change this value only with kprobe_mutex held */static bool kprobes_allow_optimization; /* * Call all pre_handler on the list, but ignores its return value. * This must be called from arch-dep optimized caller. */void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs){	struct kprobe *kp; 	list_for_each_entry_rcu(kp, &p->list, list) {		if (kp->pre_handler && likely(!kprobe_disabled(kp))) {			set_kprobe_instance(kp);			kp->pre_handler(kp, regs);		}		reset_kprobe_instance();	}} /* Free optimized instructions and optimized_kprobe */static __kprobes void free_aggr_kprobe(struct kprobe *p){	struct optimized_kprobe *op; 	op = container_of(p, struct optimized_kprobe, kp);	arch_remove_optimized_kprobe(op);	arch_remove_kprobe(p);	kfree(op);} /* Return true(!0) if the kprobe is ready for optimization. */static inline int kprobe_optready(struct kprobe *p){	struct optimized_kprobe *op; 	if (kprobe_aggrprobe(p)) {		op = container_of(p, struct optimized_kprobe, kp);		return arch_prepared_optinsn(&op->optinsn);	} 	return 0;} /* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */static inline int kprobe_disarmed(struct kprobe *p){	struct optimized_kprobe *op; 	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */	if (!kprobe_aggrprobe(p))		return kprobe_disabled(p); 	op = container_of(p, struct optimized_kprobe, kp); 	return kprobe_disabled(p) && list_empty(&op->list);} /* Return true(!0) if the probe is queued on (un)optimizing lists */static int __kprobes kprobe_queued(struct kprobe *p){	struct optimized_kprobe *op; 	if (kprobe_aggrprobe(p)) {		op = container_of(p, struct optimized_kprobe, kp);		if (!list_empty(&op->list))			return 1;	}	return 0;} /* * Return an optimized kprobe whose optimizing code replaces * instructions including addr (exclude breakpoint). */static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr){	int i;	struct kprobe *p = NULL;	struct optimized_kprobe *op; 	/* Don't check i == 0, since that is a breakpoint case. */	for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)		p = get_kprobe((void *)(addr - i)); 	if (p && kprobe_optready(p)) {		op = container_of(p, struct optimized_kprobe, kp);		if (arch_within_optimized_kprobe(op, addr))			return p;	} 	return NULL;} /* Optimization staging list, protected by kprobe_mutex */static LIST_HEAD(optimizing_list);static LIST_HEAD(unoptimizing_list); static void kprobe_optimizer(struct work_struct *work);static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);static DECLARE_COMPLETION(optimizer_comp);#define OPTIMIZE_DELAY 5 /* * Optimize (replace a breakpoint with a jump) kprobes listed on * optimizing_list. */static __kprobes void do_optimize_kprobes(void){	/* Optimization never be done when disarmed */	if (kprobes_all_disarmed || !kprobes_allow_optimization ||	    list_empty(&optimizing_list))		return; 	/*	 * The optimization/unoptimization refers online_cpus via	 * stop_machine() and cpu-hotplug modifies online_cpus.	 * And same time, text_mutex will be held in cpu-hotplug and here.	 * This combination can cause a deadlock (cpu-hotplug try to lock	 * text_mutex but stop_machine can not be done because online_cpus	 * has been changed)	 * To avoid this deadlock, we need to call get_online_cpus()	 * for preventing cpu-hotplug outside of text_mutex locking.	 */	get_online_cpus();	mutex_lock(&text_mutex);	arch_optimize_kprobes(&optimizing_list);

Lines 1–500 of 2261. Load more