// SPDX-License-Identifier: GPL-2.0/* * Generic infrastructure for lifetime debugging of objects. * * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de> */ #define pr_fmt(fmt) "ODEBUG: " fmt #include <linux/debugobjects.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/sched/task_stack.h>#include <linux/seq_file.h>#include <linux/debugfs.h>#include <linux/slab.h>#include <linux/hash.h>#include <linux/kmemleak.h>#include <linux/cpu.h> #define ODEBUG_HASH_BITS	14#define ODEBUG_HASH_SIZE	(1 << ODEBUG_HASH_BITS) #define ODEBUG_POOL_SIZE	1024#define ODEBUG_POOL_MIN_LEVEL	256#define ODEBUG_POOL_PERCPU_SIZE	64#define ODEBUG_BATCH_SIZE	16 #define ODEBUG_CHUNK_SHIFT	PAGE_SHIFT#define ODEBUG_CHUNK_SIZE	(1 << ODEBUG_CHUNK_SHIFT)#define ODEBUG_CHUNK_MASK	(~(ODEBUG_CHUNK_SIZE - 1)) /* * We limit the freeing of debug objects via workqueue at a maximum * frequency of 10Hz and about 1024 objects for each freeing operation. * So it is freeing at most 10k debug objects per second. */#define ODEBUG_FREE_WORK_MAX	(1024 / ODEBUG_BATCH_SIZE)#define ODEBUG_FREE_WORK_DELAY	DIV_ROUND_UP(HZ, 10) struct debug_bucket {	struct hlist_head	list;	raw_spinlock_t		lock;}; struct obj_pool {	struct hlist_head	objects;	unsigned int		cnt;	unsigned int		min_cnt;	unsigned int		max_cnt;} ____cacheline_aligned;  static DEFINE_PER_CPU_ALIGNED(struct obj_pool, pool_pcpu)  = {	.max_cnt	= ODEBUG_POOL_PERCPU_SIZE,}; static struct debug_bucket	obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj		obj_static_pool[ODEBUG_POOL_SIZE] __initdata; static DEFINE_RAW_SPINLOCK(pool_lock); static struct obj_pool pool_global = {	.min_cnt	= ODEBUG_POOL_MIN_LEVEL,	.max_cnt	= ODEBUG_POOL_SIZE,}; static struct obj_pool pool_to_free = {	.max_cnt	= UINT_MAX,}; static HLIST_HEAD(pool_boot); /* * Because of the presence of percpu free pools, obj_pool_free will * under-count those in the percpu free pools. Similarly, obj_pool_used * will over-count those in the percpu free pools. Adjustments will be * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used * can be off. */static int __data_racy		obj_pool_min_free = ODEBUG_POOL_SIZE;static int			obj_pool_used;static int __data_racy		obj_pool_max_used;static bool			obj_freeing; static int __data_racy			debug_objects_maxchain __read_mostly;static int __data_racy __maybe_unused	debug_objects_maxchecked __read_mostly;static int __data_racy			debug_objects_fixups __read_mostly;static int __data_racy			debug_objects_warnings __read_mostly;static bool __data_racy			debug_objects_enabled __read_mostly					= CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT; static const struct debug_obj_descr	*descr_test  __read_mostly;static struct kmem_cache		*obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */static int __data_racy		debug_objects_allocated;static int __data_racy		debug_objects_freed; static void free_obj_work(struct work_struct *work);static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static int __init enable_object_debug(char *str){	debug_objects_enabled = true;	return 0;}early_param("debug_objects", enable_object_debug); static int __init disable_object_debug(char *str){	debug_objects_enabled = false;	return 0;}early_param("no_debug_objects", disable_object_debug); static const char *obj_states[ODEBUG_STATE_MAX] = {	[ODEBUG_STATE_NONE]		= "none",	[ODEBUG_STATE_INIT]		= "initialized",	[ODEBUG_STATE_INACTIVE]		= "inactive",	[ODEBUG_STATE_ACTIVE]		= "active",	[ODEBUG_STATE_DESTROYED]	= "destroyed",	[ODEBUG_STATE_NOTAVAILABLE]	= "not available",}; static __always_inline unsigned int pool_count(struct obj_pool *pool){	return READ_ONCE(pool->cnt);} static __always_inline bool pool_should_refill(struct obj_pool *pool){	return pool_count(pool) < pool->min_cnt;} static __always_inline bool pool_must_refill(struct obj_pool *pool){	return pool_count(pool) < pool->min_cnt / 2;} static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src){	if (dst->cnt + ODEBUG_BATCH_SIZE > dst->max_cnt || !src->cnt)		return false; 	for (int i = 0; i < ODEBUG_BATCH_SIZE && src->cnt; i++) {		struct hlist_node *node = src->objects.first; 		WRITE_ONCE(src->cnt, src->cnt - 1);		WRITE_ONCE(dst->cnt, dst->cnt + 1); 		hlist_del(node);		hlist_add_head(node, &dst->objects);	}	return true;} static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src){	if (!src->cnt)		return false; 	for (int i = 0; src->cnt && i < ODEBUG_BATCH_SIZE; i++) {		struct hlist_node *node = src->objects.first; 		WRITE_ONCE(src->cnt, src->cnt - 1);		hlist_del(node);		hlist_add_head(node, head);	}	return true;} static struct debug_obj *__alloc_object(struct hlist_head *list){	struct debug_obj *obj; 	if (unlikely(!list->first))		return NULL; 	obj = hlist_entry(list->first, typeof(*obj), node);	hlist_del(&obj->node);	return obj;} static struct debug_obj *pcpu_alloc(void){	struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); 	lockdep_assert_irqs_disabled(); 	for (;;) {		struct debug_obj *obj = __alloc_object(&pcp->objects); 		if (likely(obj)) {			pcp->cnt--;			return obj;		} 		guard(raw_spinlock)(&pool_lock);		if (!pool_move_batch(pcp, &pool_to_free)) {			if (!pool_move_batch(pcp, &pool_global))				return NULL;		}		obj_pool_used += pcp->cnt; 		if (obj_pool_used > obj_pool_max_used)			obj_pool_max_used = obj_pool_used; 		if (pool_global.cnt < obj_pool_min_free)			obj_pool_min_free = pool_global.cnt;	}} static void pcpu_free(struct debug_obj *obj){	struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); 	lockdep_assert_irqs_disabled(); 	hlist_add_head(&obj->node, &pcp->objects);	pcp->cnt++; 	/* Pool full ? */	if (pcp->cnt < ODEBUG_POOL_PERCPU_SIZE)		return; 	/* Remove a batch from the per CPU pool */	guard(raw_spinlock)(&pool_lock);	/* Try to fit the batch into the pool_global first */	if (!pool_move_batch(&pool_global, pcp))		pool_move_batch(&pool_to_free, pcp);	obj_pool_used -= ODEBUG_BATCH_SIZE;} static void free_object_list(struct hlist_head *head){	struct hlist_node *tmp;	struct debug_obj *obj;	int cnt = 0; 	hlist_for_each_entry_safe(obj, tmp, head, node) {		hlist_del(&obj->node);		kmem_cache_free(obj_cache, obj);		cnt++;	}	debug_objects_freed += cnt;} static void fill_pool_from_freelist(void){	static unsigned long state; 	/*	 * Reuse objs from the global obj_to_free list; they will be	 * reinitialized when allocating.	 */	if (!pool_count(&pool_to_free))		return; 	/*	 * Prevent the context from being scheduled or interrupted after	 * setting the state flag;	 */	guard(irqsave)(); 	/*	 * Avoid lock contention on &pool_lock and avoid making the cache	 * line exclusive by testing the bit before attempting to set it.	 */	if (test_bit(0, &state) || test_and_set_bit(0, &state))		return; 	/* Avoid taking the lock when there is no work to do */	while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) {		guard(raw_spinlock)(&pool_lock);		/* Move a batch if possible */		pool_move_batch(&pool_global, &pool_to_free);	}	clear_bit(0, &state);} static void fill_pool(void){	static atomic_t cpus_allocating; 	/*	 * Avoid allocation and lock contention when:	 *   - One other CPU is already allocating	 *   - the global pool has not reached the critical level yet	 */	if (!pool_must_refill(&pool_global) && atomic_read(&cpus_allocating))		return; 	atomic_inc(&cpus_allocating);	while (pool_should_refill(&pool_global)) {		struct debug_obj *new, *last = NULL;		HLIST_HEAD(head);		int cnt; 		for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {			new = kmem_cache_zalloc(obj_cache, __GFP_HIGH | __GFP_NOWARN);			if (!new)				break;			hlist_add_head(&new->node, &head);			if (!last)				last = new;		}		if (!cnt)			break; 		guard(raw_spinlock_irqsave)(&pool_lock);		hlist_splice_init(&head, &last->node, &pool_global.objects);		debug_objects_allocated += cnt;		WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);	}	atomic_dec(&cpus_allocating);} /* * Lookup an object in the hash bucket. */static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b){	struct debug_obj *obj;	int cnt = 0; 	hlist_for_each_entry(obj, &b->list, node) {		cnt++;		if (obj->object == addr)			return obj;	}	if (cnt > debug_objects_maxchain)		debug_objects_maxchain = cnt; 	return NULL;} static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,				      const struct debug_obj_descr *descr){	struct debug_obj *obj; 	if (likely(obj_cache))		obj = pcpu_alloc();	else		obj = __alloc_object(&pool_boot); 	if (likely(obj)) {		obj->object = addr;		obj->descr  = descr;		obj->state  = ODEBUG_STATE_NONE;		obj->astate = 0;		hlist_add_head(&obj->node, &b->list);	}	return obj;} /* workqueue function to free objects. */static void free_obj_work(struct work_struct *work){	bool free = true; 	WRITE_ONCE(obj_freeing, false); 	if (!pool_count(&pool_to_free))		return; 	for (unsigned int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) {		HLIST_HEAD(tofree); 		/* Acquire and drop the lock for each batch */		scoped_guard(raw_spinlock_irqsave, &pool_lock) {			if (!pool_to_free.cnt)				return; 			/* Refill the global pool if possible */			if (pool_move_batch(&pool_global, &pool_to_free)) {				/* Don't free as there seems to be demand */				free = false;			} else if (free) {				pool_pop_batch(&tofree, &pool_to_free);			} else {				return;			}		}		free_object_list(&tofree);	}} static void __free_object(struct debug_obj *obj){	guard(irqsave)();	if (likely(obj_cache))		pcpu_free(obj);	else		hlist_add_head(&obj->node, &pool_boot);} /* * Put the object back into the pool and schedule work to free objects * if necessary. */static void free_object(struct debug_obj *obj){	__free_object(obj);	if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {		WRITE_ONCE(obj_freeing, true);		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);	}} static void put_objects(struct hlist_head *list){	struct hlist_node *tmp;	struct debug_obj *obj; 	/*	 * Using free_object() puts the objects into reuse or schedules	 * them for freeing and it get's all the accounting correct.	 */	hlist_for_each_entry_safe(obj, tmp, list, node) {		hlist_del(&obj->node);		free_object(obj);	}} #ifdef CONFIG_HOTPLUG_CPUstatic int object_cpu_offline(unsigned int cpu){	/* Remote access is safe as the CPU is dead already */	struct obj_pool *pcp = per_cpu_ptr(&pool_pcpu, cpu); 	put_objects(&pcp->objects);	pcp->cnt = 0;	return 0;}#endif /* Out of memory. Free all objects from hash */static void debug_objects_oom(void){	struct debug_bucket *db = obj_hash;	HLIST_HEAD(freelist); 	pr_warn("Out of memory. ODEBUG disabled\n"); 	for (int i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {		scoped_guard(raw_spinlock_irqsave, &db->lock)			hlist_move_list(&db->list, &freelist); 		put_objects(&freelist);	}} /* * We use the pfn of the address for the hash. That way we can check * for freed objects simply by checking the affected bucket. */static struct debug_bucket *get_bucket(unsigned long addr){	unsigned long hash; 	hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);	return &obj_hash[hash];} static void debug_print_object(struct debug_obj *obj, char *msg){	const struct debug_obj_descr *descr = obj->descr;	static int limit; 	/*	 * Don't report if lookup_object_or_alloc() by the current thread	 * failed because lookup_object_or_alloc()/debug_objects_oom() by a	 * concurrent thread turned off debug_objects_enabled and cleared	 * the hash buckets.	 */	if (!debug_objects_enabled)		return; 	if (limit < 5 && descr != descr_test) {		void *hint = descr->debug_hint ?			descr->debug_hint(obj->object) : NULL;		limit++;		WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "				 "object: %p object type: %s hint: %pS\n",			msg, obj_states[obj->state], obj->astate,			obj->object, descr->name, hint);	}	debug_objects_warnings++;} /* * Try to repair the damage, so we have a better chance to get useful * debug output. */static bool

Lines 1–500 of 1387. Load more