5152 lines · 126.0 KB · C
RawDownload
/* * kernel/lockdep.c * * Runtime locking correctness validator * * Started by Ingo Molnar: * *  Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra * * this code maps all the lock dependencies as they occur in a live kernel * and will warn about the following classes of locking bugs: * * - lock inversion scenarios * - circular lock dependencies * - hardirq/softirq safe/unsafe locking bugs * * Bugs are reported even if the current locking scenario does not cause * any deadlock at this point. * * I.e. if anytime in the past two locks were taken in a different order, * even if it happened for another task, even if those were different * locks (but of the same class as this lock), this code will detect it. * * Thanks to Arjan van de Ven for coming up with the initial idea of * mapping lock dependencies runtime. */#define DISABLE_BRANCH_PROFILING#include <linux/mutex.h>#include <linux/sched.h>#include <linux/sched/clock.h>#include <linux/sched/task.h>#include <linux/sched/mm.h>#include <linux/delay.h>#include <linux/module.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/spinlock.h>#include <linux/kallsyms.h>#include <linux/interrupt.h>#include <linux/stacktrace.h>#include <linux/debug_locks.h>#include <linux/irqflags.h>#include <linux/utsname.h>#include <linux/hash.h>#include <linux/ftrace.h>#include <linux/stringify.h>#include <linux/bitops.h>#include <linux/gfp.h>#include <linux/kmemcheck.h>#include <linux/random.h>#include <linux/jhash.h> #include <asm/sections.h> #include "lockdep_internals.h" #define CREATE_TRACE_POINTS#include <trace/events/lock.h> #ifdef CONFIG_LOCKDEP_CROSSRELEASE#include <linux/slab.h>#endif #ifdef CONFIG_PROVE_LOCKINGint prove_locking = 1;module_param(prove_locking, int, 0644);#else#define prove_locking 0#endif #ifdef CONFIG_LOCK_STATint lock_stat = 1;module_param(lock_stat, int, 0644);#else#define lock_stat 0#endif /* * lockdep_lock: protects the lockdep graph, the hashes and the *               class/list/hash allocators. * * This is one of the rare exceptions where it's justified * to use a raw spinlock - we really dont want the spinlock * code to recurse back into the lockdep code... */static arch_spinlock_t lockdep_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; static int graph_lock(void){	arch_spin_lock(&lockdep_lock);	/*	 * Make sure that if another CPU detected a bug while	 * walking the graph we dont change it (while the other	 * CPU is busy printing out stuff with the graph lock	 * dropped already)	 */	if (!debug_locks) {		arch_spin_unlock(&lockdep_lock);		return 0;	}	/* prevent any recursions within lockdep from causing deadlocks */	current->lockdep_recursion++;	return 1;} static inline int graph_unlock(void){	if (debug_locks && !arch_spin_is_locked(&lockdep_lock)) {		/*		 * The lockdep graph lock isn't locked while we expect it to		 * be, we're confused now, bye!		 */		return DEBUG_LOCKS_WARN_ON(1);	} 	current->lockdep_recursion--;	arch_spin_unlock(&lockdep_lock);	return 0;} /* * Turn lock debugging off and return with 0 if it was off already, * and also release the graph lock: */static inline int debug_locks_off_graph_unlock(void){	int ret = debug_locks_off(); 	arch_spin_unlock(&lockdep_lock); 	return ret;} unsigned long nr_list_entries;static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; /* * All data structures here are protected by the global debug_lock. * * Mutex key structs only get allocated, once during bootup, and never * get freed - this significantly simplifies the debugging code. */unsigned long nr_lock_classes;static struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; static inline struct lock_class *hlock_class(struct held_lock *hlock){	if (!hlock->class_idx) {		/*		 * Someone passed in garbage, we give up.		 */		DEBUG_LOCKS_WARN_ON(1);		return NULL;	}	return lock_classes + hlock->class_idx - 1;} #ifdef CONFIG_LOCK_STATstatic DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats); static inline u64 lockstat_clock(void){	return local_clock();} static int lock_point(unsigned long points[], unsigned long ip){	int i; 	for (i = 0; i < LOCKSTAT_POINTS; i++) {		if (points[i] == 0) {			points[i] = ip;			break;		}		if (points[i] == ip)			break;	} 	return i;} static void lock_time_inc(struct lock_time *lt, u64 time){	if (time > lt->max)		lt->max = time; 	if (time < lt->min || !lt->nr)		lt->min = time; 	lt->total += time;	lt->nr++;} static inline void lock_time_add(struct lock_time *src, struct lock_time *dst){	if (!src->nr)		return; 	if (src->max > dst->max)		dst->max = src->max; 	if (src->min < dst->min || !dst->nr)		dst->min = src->min; 	dst->total += src->total;	dst->nr += src->nr;} struct lock_class_stats lock_stats(struct lock_class *class){	struct lock_class_stats stats;	int cpu, i; 	memset(&stats, 0, sizeof(struct lock_class_stats));	for_each_possible_cpu(cpu) {		struct lock_class_stats *pcs =			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; 		for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)			stats.contention_point[i] += pcs->contention_point[i]; 		for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)			stats.contending_point[i] += pcs->contending_point[i]; 		lock_time_add(&pcs->read_waittime, &stats.read_waittime);		lock_time_add(&pcs->write_waittime, &stats.write_waittime); 		lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);		lock_time_add(&pcs->write_holdtime, &stats.write_holdtime); 		for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)			stats.bounces[i] += pcs->bounces[i];	} 	return stats;} void clear_lock_stats(struct lock_class *class){	int cpu; 	for_each_possible_cpu(cpu) {		struct lock_class_stats *cpu_stats =			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; 		memset(cpu_stats, 0, sizeof(struct lock_class_stats));	}	memset(class->contention_point, 0, sizeof(class->contention_point));	memset(class->contending_point, 0, sizeof(class->contending_point));} static struct lock_class_stats *get_lock_stats(struct lock_class *class){	return &get_cpu_var(cpu_lock_stats)[class - lock_classes];} static void put_lock_stats(struct lock_class_stats *stats){	put_cpu_var(cpu_lock_stats);} static void lock_release_holdtime(struct held_lock *hlock){	struct lock_class_stats *stats;	u64 holdtime; 	if (!lock_stat)		return; 	holdtime = lockstat_clock() - hlock->holdtime_stamp; 	stats = get_lock_stats(hlock_class(hlock));	if (hlock->read)		lock_time_inc(&stats->read_holdtime, holdtime);	else		lock_time_inc(&stats->write_holdtime, holdtime);	put_lock_stats(stats);}#elsestatic inline void lock_release_holdtime(struct held_lock *hlock){}#endif /* * We keep a global list of all lock classes. The list only grows, * never shrinks. The list is only accessed with the lockdep * spinlock lock held. */LIST_HEAD(all_lock_classes); /* * The lockdep classes are in a hash-table as well, for fast lookup: */#define CLASSHASH_BITS		(MAX_LOCKDEP_KEYS_BITS - 1)#define CLASSHASH_SIZE		(1UL << CLASSHASH_BITS)#define __classhashfn(key)	hash_long((unsigned long)key, CLASSHASH_BITS)#define classhashentry(key)	(classhash_table + __classhashfn((key))) static struct hlist_head classhash_table[CLASSHASH_SIZE]; /* * We put the lock dependency chains into a hash-table as well, to cache * their existence: */#define CHAINHASH_BITS		(MAX_LOCKDEP_CHAINS_BITS-1)#define CHAINHASH_SIZE		(1UL << CHAINHASH_BITS)#define __chainhashfn(chain)	hash_long(chain, CHAINHASH_BITS)#define chainhashentry(chain)	(chainhash_table + __chainhashfn((chain))) static struct hlist_head chainhash_table[CHAINHASH_SIZE]; /* * The hash key of the lock dependency chains is a hash itself too: * it's a hash of all locks taken up to that lock, including that lock. * It's a 64-bit hash, because it's important for the keys to be * unique. */static inline u64 iterate_chain_key(u64 key, u32 idx){	u32 k0 = key, k1 = key >> 32; 	__jhash_mix(idx, k0, k1); /* Macro that modifies arguments! */ 	return k0 | (u64)k1 << 32;} void lockdep_off(void){	current->lockdep_recursion++;}EXPORT_SYMBOL(lockdep_off); void lockdep_on(void){	current->lockdep_recursion--;}EXPORT_SYMBOL(lockdep_on); /* * Debugging switches: */ #define VERBOSE			0#define VERY_VERBOSE		0 #if VERBOSE# define HARDIRQ_VERBOSE	1# define SOFTIRQ_VERBOSE	1#else# define HARDIRQ_VERBOSE	0# define SOFTIRQ_VERBOSE	0#endif #if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE/* * Quick filtering for interesting events: */static int class_filter(struct lock_class *class){#if 0	/* Example */	if (class->name_version == 1 &&			!strcmp(class->name, "lockname"))		return 1;	if (class->name_version == 1 &&			!strcmp(class->name, "&struct->lockfield"))		return 1;#endif	/* Filter everything else. 1 would be to allow everything else */	return 0;}#endif static int verbose(struct lock_class *class){#if VERBOSE	return class_filter(class);#endif	return 0;} /* * Stack-trace: tightly packed array of stack backtrace * addresses. Protected by the graph_lock. */unsigned long nr_stack_trace_entries;static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES]; static void print_lockdep_off(const char *bug_msg){	printk(KERN_DEBUG "%s\n", bug_msg);	printk(KERN_DEBUG "turning off the locking correctness validator.\n");#ifdef CONFIG_LOCK_STAT	printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");#endif} static int save_trace(struct stack_trace *trace){	trace->nr_entries = 0;	trace->max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;	trace->entries = stack_trace + nr_stack_trace_entries; 	trace->skip = 3; 	save_stack_trace(trace); 	/*	 * Some daft arches put -1 at the end to indicate its a full trace.	 *	 * <rant> this is buggy anyway, since it takes a whole extra entry so a	 * complete trace that maxes out the entries provided will be reported	 * as incomplete, friggin useless </rant>	 */	if (trace->nr_entries != 0 &&	    trace->entries[trace->nr_entries-1] == ULONG_MAX)		trace->nr_entries--; 	trace->max_entries = trace->nr_entries; 	nr_stack_trace_entries += trace->nr_entries; 	if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {		if (!debug_locks_off_graph_unlock())			return 0; 		print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");		dump_stack(); 		return 0;	} 	return 1;} unsigned int nr_hardirq_chains;unsigned int nr_softirq_chains;unsigned int nr_process_chains;unsigned int max_lockdep_depth; #ifdef CONFIG_DEBUG_LOCKDEP/* * Various lockdep statistics: */DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);#endif /* * Locking printouts: */ #define __USAGE(__STATE)						\	[LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W",	\	[LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W",		\	[LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\	[LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R", static const char *usage_str[] ={#define LOCKDEP_STATE(__STATE) __USAGE(__STATE)#include "lockdep_states.h"#undef LOCKDEP_STATE	[LOCK_USED] = "INITIAL USE",}; const char * __get_key_name(struct lockdep_subclass_key *key, char *str){	return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);} static inline unsigned long lock_flag(enum lock_usage_bit bit){	return 1UL << bit;} static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit){	char c = '.'; 	if (class->usage_mask & lock_flag(bit + 2))		c = '+';	if (class->usage_mask & lock_flag(bit)) {		c = '-';		if (class->usage_mask & lock_flag(bit + 2))			c = '?';	} 	return c;} void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS]){	int i = 0; #define LOCKDEP_STATE(__STATE) 						\	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE);	\	usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);#include "lockdep_states.h"

Lines 1–500 of 5152.