/* * jump label support * * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> * */#include <linux/jump_label.h>#include <linux/memory.h>#include <linux/uaccess.h>#include <linux/module.h>#include <linux/list.h>#include <linux/jhash.h>#include <linux/slab.h>#include <linux/sort.h>#include <linux/err.h> #ifdef HAVE_JUMP_LABEL #define JUMP_LABEL_HASH_BITS 6#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; /* mutex to protect coming/going of the the jump_label table */static DEFINE_MUTEX(jump_label_mutex); struct jump_label_entry {	struct hlist_node hlist;	struct jump_entry *table;	int nr_entries;	/* hang modules off here */	struct hlist_head modules;	unsigned long key;}; struct jump_label_module_entry {	struct hlist_node hlist;	struct jump_entry *table;	int nr_entries;	struct module *mod;}; void jump_label_lock(void){	mutex_lock(&jump_label_mutex);} void jump_label_unlock(void){	mutex_unlock(&jump_label_mutex);} static int jump_label_cmp(const void *a, const void *b){	const struct jump_entry *jea = a;	const struct jump_entry *jeb = b; 	if (jea->key < jeb->key)		return -1; 	if (jea->key > jeb->key)		return 1; 	return 0;} static voidsort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop){	unsigned long size; 	size = (((unsigned long)stop - (unsigned long)start)					/ sizeof(struct jump_entry));	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);} static struct jump_label_entry *get_jump_label_entry(jump_label_t key){	struct hlist_head *head;	struct hlist_node *node;	struct jump_label_entry *e;	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); 	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];	hlist_for_each_entry(e, node, head, hlist) {		if (key == e->key)			return e;	}	return NULL;} static struct jump_label_entry *add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table){	struct hlist_head *head;	struct jump_label_entry *e;	u32 hash; 	e = get_jump_label_entry(key);	if (e)		return ERR_PTR(-EEXIST); 	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);	if (!e)		return ERR_PTR(-ENOMEM); 	hash = jhash((void *)&key, sizeof(jump_label_t), 0);	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];	e->key = key;	e->table = table;	e->nr_entries = nr_entries;	INIT_HLIST_HEAD(&(e->modules));	hlist_add_head(&e->hlist, head);	return e;} static intbuild_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop){	struct jump_entry *iter, *iter_begin;	struct jump_label_entry *entry;	int count; 	sort_jump_label_entries(start, stop);	iter = start;	while (iter < stop) {		entry = get_jump_label_entry(iter->key);		if (!entry) {			iter_begin = iter;			count = 0;			while ((iter < stop) &&				(iter->key == iter_begin->key)) {				iter++;				count++;			}			entry = add_jump_label_entry(iter_begin->key,							count, iter_begin);			if (IS_ERR(entry))				return PTR_ERR(entry);		 } else {			WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");			return -1;		}	}	return 0;} /*** * jump_label_update - update jump label text * @key -  key value associated with a a jump label * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE * * Will enable/disable the jump for jump label @key, depending on the * value of @type. * */ void jump_label_update(unsigned long key, enum jump_label_type type){	struct jump_entry *iter;	struct jump_label_entry *entry;	struct hlist_node *module_node;	struct jump_label_module_entry *e_module;	int count; 	jump_label_lock();	entry = get_jump_label_entry((jump_label_t)key);	if (entry) {		count = entry->nr_entries;		iter = entry->table;		while (count--) {			if (kernel_text_address(iter->code))				arch_jump_label_transform(iter, type);			iter++;		}		/* eanble/disable jump labels in modules */		hlist_for_each_entry(e_module, module_node, &(entry->modules),							hlist) {			count = e_module->nr_entries;			iter = e_module->table;			while (count--) {				if (iter->key &&						kernel_text_address(iter->code))					arch_jump_label_transform(iter, type);				iter++;			}		}	}	jump_label_unlock();} static int addr_conflict(struct jump_entry *entry, void *start, void *end){	if (entry->code <= (unsigned long)end &&		entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)		return 1; 	return 0;} #ifdef CONFIG_MODULES static int module_conflict(void *start, void *end){	struct hlist_head *head;	struct hlist_node *node, *node_next, *module_node, *module_node_next;	struct jump_label_entry *e;	struct jump_label_module_entry *e_module;	struct jump_entry *iter;	int i, count;	int conflict = 0; 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {		head = &jump_label_table[i];		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {			hlist_for_each_entry_safe(e_module, module_node,							module_node_next,							&(e->modules), hlist) {				count = e_module->nr_entries;				iter = e_module->table;				while (count--) {					if (addr_conflict(iter, start, end)) {						conflict = 1;						goto out;					}					iter++;				}			}		}	}out:	return conflict;} #endif /*** * jump_label_text_reserved - check if addr range is reserved * @start: start text addr * @end: end text addr * * checks if the text addr located between @start and @end * overlaps with any of the jump label patch addresses. Code * that wants to modify kernel text should first verify that * it does not overlap with any of the jump label addresses. * Caller must hold jump_label_mutex. * * returns 1 if there is an overlap, 0 otherwise */int jump_label_text_reserved(void *start, void *end){	struct jump_entry *iter;	struct jump_entry *iter_start = __start___jump_table;	struct jump_entry *iter_stop = __start___jump_table;	int conflict = 0; 	iter = iter_start;	while (iter < iter_stop) {		if (addr_conflict(iter, start, end)) {			conflict = 1;			goto out;		}		iter++;	} 	/* now check modules */#ifdef CONFIG_MODULES	conflict = module_conflict(start, end);#endifout:	return conflict;} /* * Not all archs need this. */void __weak arch_jump_label_text_poke_early(jump_label_t addr){} static __init int init_jump_label(void){	int ret;	struct jump_entry *iter_start = __start___jump_table;	struct jump_entry *iter_stop = __stop___jump_table;	struct jump_entry *iter; 	jump_label_lock();	ret = build_jump_label_hashtable(__start___jump_table,					 __stop___jump_table);	iter = iter_start;	while (iter < iter_stop) {		arch_jump_label_text_poke_early(iter->code);		iter++;	}	jump_label_unlock();	return ret;}early_initcall(init_jump_label); #ifdef CONFIG_MODULES static struct jump_label_module_entry *add_jump_label_module_entry(struct jump_label_entry *entry,			    struct jump_entry *iter_begin,			    int count, struct module *mod){	struct jump_label_module_entry *e; 	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);	if (!e)		return ERR_PTR(-ENOMEM);	e->mod = mod;	e->nr_entries = count;	e->table = iter_begin;	hlist_add_head(&e->hlist, &entry->modules);	return e;} static int add_jump_label_module(struct module *mod){	struct jump_entry *iter, *iter_begin;	struct jump_label_entry *entry;	struct jump_label_module_entry *module_entry;	int count; 	/* if the module doesn't have jump label entries, just return */	if (!mod->num_jump_entries)		return 0; 	sort_jump_label_entries(mod->jump_entries,				mod->jump_entries + mod->num_jump_entries);	iter = mod->jump_entries;	while (iter < mod->jump_entries + mod->num_jump_entries) {		entry = get_jump_label_entry(iter->key);		iter_begin = iter;		count = 0;		while ((iter < mod->jump_entries + mod->num_jump_entries) &&			(iter->key == iter_begin->key)) {				iter++;				count++;		}		if (!entry) {			entry = add_jump_label_entry(iter_begin->key, 0, NULL);			if (IS_ERR(entry))				return PTR_ERR(entry);		}		module_entry = add_jump_label_module_entry(entry, iter_begin,							   count, mod);		if (IS_ERR(module_entry))			return PTR_ERR(module_entry);	}	return 0;} static void remove_jump_label_module(struct module *mod){	struct hlist_head *head;	struct hlist_node *node, *node_next, *module_node, *module_node_next;	struct jump_label_entry *e;	struct jump_label_module_entry *e_module;	int i; 	/* if the module doesn't have jump label entries, just return */	if (!mod->num_jump_entries)		return; 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {		head = &jump_label_table[i];		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {			hlist_for_each_entry_safe(e_module, module_node,						  module_node_next,						  &(e->modules), hlist) {				if (e_module->mod == mod) {					hlist_del(&e_module->hlist);					kfree(e_module);				}			}			if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {				hlist_del(&e->hlist);				kfree(e);			}		}	}} static void remove_jump_label_module_init(struct module *mod){	struct hlist_head *head;	struct hlist_node *node, *node_next, *module_node, *module_node_next;	struct jump_label_entry *e;	struct jump_label_module_entry *e_module;	struct jump_entry *iter;	int i, count; 	/* if the module doesn't have jump label entries, just return */	if (!mod->num_jump_entries)		return; 	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {		head = &jump_label_table[i];		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {			hlist_for_each_entry_safe(e_module, module_node,						  module_node_next,						  &(e->modules), hlist) {				if (e_module->mod != mod)					continue;				count = e_module->nr_entries;				iter = e_module->table;				while (count--) {					if (within_module_init(iter->code, mod))						iter->key = 0;					iter++;				}			}		}	}} static intjump_label_module_notify(struct notifier_block *self, unsigned long val,			 void *data){	struct module *mod = data;	int ret = 0; 	switch (val) {	case MODULE_STATE_COMING:		jump_label_lock();		ret = add_jump_label_module(mod);		if (ret)			remove_jump_label_module(mod);		jump_label_unlock();		break;	case MODULE_STATE_GOING:		jump_label_lock();		remove_jump_label_module(mod);		jump_label_unlock();		break;	case MODULE_STATE_LIVE:		jump_label_lock();		remove_jump_label_module_init(mod);		jump_label_unlock();		break;	}	return ret;} /*** * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() * @mod: module to patch * * Allow for run-time selection of the optimal nops. Before the module * loads patch these with arch_get_jump_label_nop(), which is specified by * the arch specific jump label code. */void jump_label_apply_nops(struct module *mod){	struct jump_entry *iter; 	/* if the module doesn't have jump label entries, just return */	if (!mod->num_jump_entries)		return; 	iter = mod->jump_entries;	while (iter < mod->jump_entries + mod->num_jump_entries) {		arch_jump_label_text_poke_early(iter->code);		iter++;	}} struct notifier_block jump_label_module_nb = {	.notifier_call = jump_label_module_notify,	.priority = 0,}; static __init int init_jump_label_module(void){	return register_module_notifier(&jump_label_module_nb);}early_initcall(init_jump_label_module); #endif /* CONFIG_MODULES */ #endif