// SPDX-License-Identifier: GPL-2.0-only/* * kernel/lockdep.c * * Runtime locking correctness validator * * Started by Ingo Molnar: * *  Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra * * this code maps all the lock dependencies as they occur in a live kernel * and will warn about the following classes of locking bugs: * * - lock inversion scenarios * - circular lock dependencies * - hardirq/softirq safe/unsafe locking bugs * * Bugs are reported even if the current locking scenario does not cause * any deadlock at this point. * * I.e. if anytime in the past two locks were taken in a different order, * even if it happened for another task, even if those were different * locks (but of the same class as this lock), this code will detect it. * * Thanks to Arjan van de Ven for coming up with the initial idea of * mapping lock dependencies runtime. */#define DISABLE_BRANCH_PROFILING#include <linux/mutex.h>#include <linux/sched.h>#include <linux/sched/clock.h>#include <linux/sched/task.h>#include <linux/sched/mm.h>#include <linux/delay.h>#include <linux/module.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/spinlock.h>#include <linux/kallsyms.h>#include <linux/interrupt.h>#include <linux/stacktrace.h>#include <linux/debug_locks.h>#include <linux/irqflags.h>#include <linux/utsname.h>#include <linux/hash.h>#include <linux/ftrace.h>#include <linux/stringify.h>#include <linux/bitmap.h>#include <linux/bitops.h>#include <linux/gfp.h>#include <linux/random.h>#include <linux/jhash.h>#include <linux/nmi.h>#include <linux/rcupdate.h>#include <linux/kprobes.h> #include <asm/sections.h> #include "lockdep_internals.h" #define CREATE_TRACE_POINTS#include <trace/events/lock.h> #ifdef CONFIG_PROVE_LOCKINGint prove_locking = 1;module_param(prove_locking, int, 0644);#else#define prove_locking 0#endif #ifdef CONFIG_LOCK_STATint lock_stat = 1;module_param(lock_stat, int, 0644);#else#define lock_stat 0#endif DEFINE_PER_CPU(unsigned int, lockdep_recursion);EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion); static inline bool lockdep_enabled(void){	if (!debug_locks)		return false; 	if (this_cpu_read(lockdep_recursion))		return false; 	if (current->lockdep_recursion)		return false; 	return true;} /* * lockdep_lock: protects the lockdep graph, the hashes and the *               class/list/hash allocators. * * This is one of the rare exceptions where it's justified * to use a raw spinlock - we really dont want the spinlock * code to recurse back into the lockdep code... */static arch_spinlock_t __lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;static struct task_struct *__owner; static inline void lockdep_lock(void){	DEBUG_LOCKS_WARN_ON(!irqs_disabled()); 	__this_cpu_inc(lockdep_recursion);	arch_spin_lock(&__lock);	__owner = current;} static inline void lockdep_unlock(void){	DEBUG_LOCKS_WARN_ON(!irqs_disabled()); 	if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))		return; 	__owner = NULL;	arch_spin_unlock(&__lock);	__this_cpu_dec(lockdep_recursion);} static inline bool lockdep_assert_locked(void){	return DEBUG_LOCKS_WARN_ON(__owner != current);} static struct task_struct *lockdep_selftest_task_struct;  static int graph_lock(void){	lockdep_lock();	/*	 * Make sure that if another CPU detected a bug while	 * walking the graph we dont change it (while the other	 * CPU is busy printing out stuff with the graph lock	 * dropped already)	 */	if (!debug_locks) {		lockdep_unlock();		return 0;	}	return 1;} static inline void graph_unlock(void){	lockdep_unlock();} /* * Turn lock debugging off and return with 0 if it was off already, * and also release the graph lock: */static inline int debug_locks_off_graph_unlock(void){	int ret = debug_locks_off(); 	lockdep_unlock(); 	return ret;} unsigned long nr_list_entries;static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];static DECLARE_BITMAP(list_entries_in_use, MAX_LOCKDEP_ENTRIES); /* * All data structures here are protected by the global debug_lock. * * nr_lock_classes is the number of elements of lock_classes[] that is * in use. */#define KEYHASH_BITS		(MAX_LOCKDEP_KEYS_BITS - 1)#define KEYHASH_SIZE		(1UL << KEYHASH_BITS)static struct hlist_head lock_keys_hash[KEYHASH_SIZE];unsigned long nr_lock_classes;unsigned long nr_zapped_classes;#ifndef CONFIG_DEBUG_LOCKDEPstatic#endifstruct lock_class lock_classes[MAX_LOCKDEP_KEYS];static DECLARE_BITMAP(lock_classes_in_use, MAX_LOCKDEP_KEYS); static inline struct lock_class *hlock_class(struct held_lock *hlock){	unsigned int class_idx = hlock->class_idx; 	/* Don't re-read hlock->class_idx, can't use READ_ONCE() on bitfield */	barrier(); 	if (!test_bit(class_idx, lock_classes_in_use)) {		/*		 * Someone passed in garbage, we give up.		 */		DEBUG_LOCKS_WARN_ON(1);		return NULL;	} 	/*	 * At this point, if the passed hlock->class_idx is still garbage,	 * we just have to live with it	 */	return lock_classes + class_idx;} #ifdef CONFIG_LOCK_STATstatic DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats); static inline u64 lockstat_clock(void){	return local_clock();} static int lock_point(unsigned long points[], unsigned long ip){	int i; 	for (i = 0; i < LOCKSTAT_POINTS; i++) {		if (points[i] == 0) {			points[i] = ip;			break;		}		if (points[i] == ip)			break;	} 	return i;} static void lock_time_inc(struct lock_time *lt, u64 time){	if (time > lt->max)		lt->max = time; 	if (time < lt->min || !lt->nr)		lt->min = time; 	lt->total += time;	lt->nr++;} static inline void lock_time_add(struct lock_time *src, struct lock_time *dst){	if (!src->nr)		return; 	if (src->max > dst->max)		dst->max = src->max; 	if (src->min < dst->min || !dst->nr)		dst->min = src->min; 	dst->total += src->total;	dst->nr += src->nr;} struct lock_class_stats lock_stats(struct lock_class *class){	struct lock_class_stats stats;	int cpu, i; 	memset(&stats, 0, sizeof(struct lock_class_stats));	for_each_possible_cpu(cpu) {		struct lock_class_stats *pcs =			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; 		for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)			stats.contention_point[i] += pcs->contention_point[i]; 		for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)			stats.contending_point[i] += pcs->contending_point[i]; 		lock_time_add(&pcs->read_waittime, &stats.read_waittime);		lock_time_add(&pcs->write_waittime, &stats.write_waittime); 		lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);		lock_time_add(&pcs->write_holdtime, &stats.write_holdtime); 		for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)			stats.bounces[i] += pcs->bounces[i];	} 	return stats;} void clear_lock_stats(struct lock_class *class){	int cpu; 	for_each_possible_cpu(cpu) {		struct lock_class_stats *cpu_stats =			&per_cpu(cpu_lock_stats, cpu)[class - lock_classes]; 		memset(cpu_stats, 0, sizeof(struct lock_class_stats));	}	memset(class->contention_point, 0, sizeof(class->contention_point));	memset(class->contending_point, 0, sizeof(class->contending_point));} static struct lock_class_stats *get_lock_stats(struct lock_class *class){	return &this_cpu_ptr(cpu_lock_stats)[class - lock_classes];} static void lock_release_holdtime(struct held_lock *hlock){	struct lock_class_stats *stats;	u64 holdtime; 	if (!lock_stat)		return; 	holdtime = lockstat_clock() - hlock->holdtime_stamp; 	stats = get_lock_stats(hlock_class(hlock));	if (hlock->read)		lock_time_inc(&stats->read_holdtime, holdtime);	else		lock_time_inc(&stats->write_holdtime, holdtime);}#elsestatic inline void lock_release_holdtime(struct held_lock *hlock){}#endif /* * We keep a global list of all lock classes. The list is only accessed with * the lockdep spinlock lock held. free_lock_classes is a list with free * elements. These elements are linked together by the lock_entry member in * struct lock_class. */LIST_HEAD(all_lock_classes);static LIST_HEAD(free_lock_classes); /** * struct pending_free - information about data structures about to be freed * @zapped: Head of a list with struct lock_class elements. * @lock_chains_being_freed: Bitmap that indicates which lock_chains[] elements *	are about to be freed. */struct pending_free {	struct list_head zapped;	DECLARE_BITMAP(lock_chains_being_freed, MAX_LOCKDEP_CHAINS);}; /** * struct delayed_free - data structures used for delayed freeing * * A data structure for delayed freeing of data structures that may be * accessed by RCU readers at the time these were freed. * * @rcu_head:  Used to schedule an RCU callback for freeing data structures. * @index:     Index of @pf to which freed data structures are added. * @scheduled: Whether or not an RCU callback has been scheduled. * @pf:        Array with information about data structures about to be freed. */static struct delayed_free {	struct rcu_head		rcu_head;	int			index;	int			scheduled;	struct pending_free	pf[2];} delayed_free; /* * The lockdep classes are in a hash-table as well, for fast lookup: */#define CLASSHASH_BITS		(MAX_LOCKDEP_KEYS_BITS - 1)#define CLASSHASH_SIZE		(1UL << CLASSHASH_BITS)#define __classhashfn(key)	hash_long((unsigned long)key, CLASSHASH_BITS)#define classhashentry(key)	(classhash_table + __classhashfn((key))) static struct hlist_head classhash_table[CLASSHASH_SIZE]; /* * We put the lock dependency chains into a hash-table as well, to cache * their existence: */#define CHAINHASH_BITS		(MAX_LOCKDEP_CHAINS_BITS-1)#define CHAINHASH_SIZE		(1UL << CHAINHASH_BITS)#define __chainhashfn(chain)	hash_long(chain, CHAINHASH_BITS)#define chainhashentry(chain)	(chainhash_table + __chainhashfn((chain))) static struct hlist_head chainhash_table[CHAINHASH_SIZE]; /* * the id of held_lock */static inline u16 hlock_id(struct held_lock *hlock){	BUILD_BUG_ON(MAX_LOCKDEP_KEYS_BITS + 2 > 16); 	return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));} static inline unsigned int chain_hlock_class_idx(u16 hlock_id){	return hlock_id & (MAX_LOCKDEP_KEYS - 1);} /* * The hash key of the lock dependency chains is a hash itself too: * it's a hash of all locks taken up to that lock, including that lock. * It's a 64-bit hash, because it's important for the keys to be * unique. */static inline u64 iterate_chain_key(u64 key, u32 idx){	u32 k0 = key, k1 = key >> 32; 	__jhash_mix(idx, k0, k1); /* Macro that modifies arguments! */ 	return k0 | (u64)k1 << 32;} void lockdep_init_task(struct task_struct *task){	task->lockdep_depth = 0; /* no locks held yet */	task->curr_chain_key = INITIAL_CHAIN_KEY;	task->lockdep_recursion = 0;} static __always_inline void lockdep_recursion_inc(void){	__this_cpu_inc(lockdep_recursion);} static __always_inline void lockdep_recursion_finish(void){	if (WARN_ON_ONCE(__this_cpu_dec_return(lockdep_recursion)))		__this_cpu_write(lockdep_recursion, 0);} void lockdep_set_selftest_task(struct task_struct *task){	lockdep_selftest_task_struct = task;} /* * Debugging switches: */ #define VERBOSE			0#define VERY_VERBOSE		0 #if VERBOSE# define HARDIRQ_VERBOSE	1# define SOFTIRQ_VERBOSE	1#else# define HARDIRQ_VERBOSE	0# define SOFTIRQ_VERBOSE	0#endif #if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE/* * Quick filtering for interesting events: */static int class_filter(struct lock_class *class){#if 0	/* Example */	if (class->name_version == 1 &&			!strcmp(class->name, "lockname"))		return 1;	if (class->name_version == 1 &&			!strcmp(class->name, "&struct->lockfield"))		return 1;#endif	/* Filter everything else. 1 would be to allow everything else */	return 0;}#endif static int verbose(struct lock_class *class){#if VERBOSE	return class_filter(class);#endif	return 0;} static void print_lockdep_off(const char *bug_msg){	printk(KERN_DEBUG "%s\n", bug_msg);	printk(KERN_DEBUG "turning off the locking correctness validator.\n");#ifdef CONFIG_LOCK_STAT	printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");#endif} unsigned long nr_stack_trace_entries; #ifdef CONFIG_PROVE_LOCKING

Lines 1–500 of 6357. Load more