// SPDX-License-Identifier: GPL-2.0/* * Generic infrastructure for lifetime debugging of objects. * * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de> */ #define pr_fmt(fmt) "ODEBUG: " fmt #include <linux/debugobjects.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/sched/task_stack.h>#include <linux/seq_file.h>#include <linux/debugfs.h>#include <linux/slab.h>#include <linux/hash.h>#include <linux/kmemleak.h>#include <linux/cpu.h> #define ODEBUG_HASH_BITS 14#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) #define ODEBUG_POOL_SIZE 1024#define ODEBUG_POOL_MIN_LEVEL 256#define ODEBUG_POOL_PERCPU_SIZE 64#define ODEBUG_BATCH_SIZE 16 #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)#define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1)) /* * We limit the freeing of debug objects via workqueue at a maximum * frequency of 10Hz and about 1024 objects for each freeing operation. * So it is freeing at most 10k debug objects per second. */#define ODEBUG_FREE_WORK_MAX 1024#define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10) struct debug_bucket { struct hlist_head list; raw_spinlock_t lock;}; /* * Debug object percpu free list * Access is protected by disabling irq */struct debug_percpu_free { struct hlist_head free_objs; int obj_free;}; static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool); static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; static DEFINE_RAW_SPINLOCK(pool_lock); static HLIST_HEAD(obj_pool);static HLIST_HEAD(obj_to_free); /* * Because of the presence of percpu free pools, obj_pool_free will * under-count those in the percpu free pools. Similarly, obj_pool_used * will over-count those in the percpu free pools. Adjustments will be * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used * can be off. */static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE;static int obj_pool_used;static int __data_racy obj_pool_max_used;static bool obj_freeing;/* The number of objs on the global free list */static int obj_nr_tofree; static int __data_racy debug_objects_maxchain __read_mostly;static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;static int __data_racy debug_objects_fixups __read_mostly;static int __data_racy debug_objects_warnings __read_mostly;static int __data_racy debug_objects_enabled __read_mostly = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;static int debug_objects_pool_size __ro_after_init = ODEBUG_POOL_SIZE;static int debug_objects_pool_min_level __ro_after_init = ODEBUG_POOL_MIN_LEVEL; static const struct debug_obj_descr *descr_test __read_mostly;static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */static int __data_racy debug_objects_allocated;static int __data_racy debug_objects_freed; static void free_obj_work(struct work_struct *work);static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static int __init enable_object_debug(char *str){ debug_objects_enabled = 1; return 0;} static int __init disable_object_debug(char *str){ debug_objects_enabled = 0; return 0;} early_param("debug_objects", enable_object_debug);early_param("no_debug_objects", disable_object_debug); static const char *obj_states[ODEBUG_STATE_MAX] = { [ODEBUG_STATE_NONE] = "none", [ODEBUG_STATE_INIT] = "initialized", [ODEBUG_STATE_INACTIVE] = "inactive", [ODEBUG_STATE_ACTIVE] = "active", [ODEBUG_STATE_DESTROYED] = "destroyed", [ODEBUG_STATE_NOTAVAILABLE] = "not available",}; static void fill_pool(void){ gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; struct debug_obj *obj; unsigned long flags; if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) return; /* * Reuse objs from the global obj_to_free list; they will be * reinitialized when allocating. * * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with * the WRITE_ONCE() in pool_lock critical sections. */ if (READ_ONCE(obj_nr_tofree)) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have * won the race and freed the global free list already. */ while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); hlist_add_head(&obj->node, &obj_pool); WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } if (unlikely(!obj_cache)) return; while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { struct debug_obj *new[ODEBUG_BATCH_SIZE]; int cnt; for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { new[cnt] = kmem_cache_zalloc(obj_cache, gfp); if (!new[cnt]) break; } if (!cnt) return; raw_spin_lock_irqsave(&pool_lock, flags); while (cnt) { hlist_add_head(&new[--cnt]->node, &obj_pool); debug_objects_allocated++; WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); }} /* * Lookup an object in the hash bucket. */static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b){ struct debug_obj *obj; int cnt = 0; hlist_for_each_entry(obj, &b->list, node) { cnt++; if (obj->object == addr) return obj; } if (cnt > debug_objects_maxchain) debug_objects_maxchain = cnt; return NULL;} /* * Allocate a new object from the hlist */static struct debug_obj *__alloc_object(struct hlist_head *list){ struct debug_obj *obj = NULL; if (list->first) { obj = hlist_entry(list->first, typeof(*obj), node); hlist_del(&obj->node); } return obj;} static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr){ struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool); struct debug_obj *obj; if (likely(obj_cache)) { obj = __alloc_object(&percpu_pool->free_objs); if (obj) { percpu_pool->obj_free--; goto init_obj; } } raw_spin_lock(&pool_lock); obj = __alloc_object(&obj_pool); if (obj) { obj_pool_used++; WRITE_ONCE(obj_pool_free, obj_pool_free - 1); /* * Looking ahead, allocate one batch of debug objects and * put them into the percpu free pool. */ if (likely(obj_cache)) { int i; for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { struct debug_obj *obj2; obj2 = __alloc_object(&obj_pool); if (!obj2) break; hlist_add_head(&obj2->node, &percpu_pool->free_objs); percpu_pool->obj_free++; obj_pool_used++; WRITE_ONCE(obj_pool_free, obj_pool_free - 1); } } if (obj_pool_used > obj_pool_max_used) obj_pool_max_used = obj_pool_used; if (obj_pool_free < obj_pool_min_free) obj_pool_min_free = obj_pool_free; } raw_spin_unlock(&pool_lock); init_obj: if (obj) { obj->object = addr; obj->descr = descr; obj->state = ODEBUG_STATE_NONE; obj->astate = 0; hlist_add_head(&obj->node, &b->list); } return obj;} /* * workqueue function to free objects. * * To reduce contention on the global pool_lock, the actual freeing of * debug objects will be delayed if the pool_lock is busy. */static void free_obj_work(struct work_struct *work){ struct hlist_node *tmp; struct debug_obj *obj; unsigned long flags; HLIST_HEAD(tofree); WRITE_ONCE(obj_freeing, false); if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return; if (obj_pool_free >= debug_objects_pool_size) goto free_objs; /* * The objs on the pool list might be allocated before the work is * run, so recheck if pool list it full or not, if not fill pool * list from the global free list. As it is likely that a workload * may be gearing up to use more and more objects, don't free any * of them until the next round. */ while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); hlist_add_head(&obj->node, &obj_pool); WRITE_ONCE(obj_pool_free, obj_pool_free + 1); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); return; free_objs: /* * Pool list is already full and there are still objs on the free * list. Move remaining free objs to a temporary list to free the * memory outside the pool_lock held region. */ if (obj_nr_tofree) { hlist_move_list(&obj_to_free, &tofree); debug_objects_freed += obj_nr_tofree; WRITE_ONCE(obj_nr_tofree, 0); } raw_spin_unlock_irqrestore(&pool_lock, flags); hlist_for_each_entry_safe(obj, tmp, &tofree, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); }} static void __free_object(struct debug_obj *obj){ struct debug_obj *objs[ODEBUG_BATCH_SIZE]; struct debug_percpu_free *percpu_pool; int lookahead_count = 0; unsigned long flags; bool work; local_irq_save(flags); if (!obj_cache) goto free_to_obj_pool; /* * Try to free it into the percpu pool first. */ percpu_pool = this_cpu_ptr(&percpu_obj_pool); if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) { hlist_add_head(&obj->node, &percpu_pool->free_objs); percpu_pool->obj_free++; local_irq_restore(flags); return; } /* * As the percpu pool is full, look ahead and pull out a batch * of objects from the percpu pool and free them as well. */ for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) { objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs); if (!objs[lookahead_count]) break; percpu_pool->obj_free--; } free_to_obj_pool: raw_spin_lock(&pool_lock); work = (obj_pool_free > debug_objects_pool_size) && obj_cache && (obj_nr_tofree < ODEBUG_FREE_WORK_MAX); obj_pool_used--; if (work) { WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); hlist_add_head(&obj->node, &obj_to_free); if (lookahead_count) { WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, &obj_to_free); } } if ((obj_pool_free > debug_objects_pool_size) && (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) { int i; /* * Free one more batch of objects from obj_pool. */ for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { obj = __alloc_object(&obj_pool); hlist_add_head(&obj->node, &obj_to_free); WRITE_ONCE(obj_pool_free, obj_pool_free - 1); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); } } } else { WRITE_ONCE(obj_pool_free, obj_pool_free + 1); hlist_add_head(&obj->node, &obj_pool); if (lookahead_count) { WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, &obj_pool); } } } raw_spin_unlock(&pool_lock); local_irq_restore(flags);} /* * Put the object back into the pool and schedule work to free objects * if necessary. */static void free_object(struct debug_obj *obj){ __free_object(obj); if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); }} #ifdef CONFIG_HOTPLUG_CPUstatic int object_cpu_offline(unsigned int cpu){ struct debug_percpu_free *percpu_pool; struct hlist_node *tmp; struct debug_obj *obj; unsigned long flags; /* Remote access is safe as the CPU is dead already */ percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu); hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); } raw_spin_lock_irqsave(&pool_lock, flags); obj_pool_used -= percpu_pool->obj_free; debug_objects_freed += percpu_pool->obj_free; raw_spin_unlock_irqrestore(&pool_lock, flags); percpu_pool->obj_free = 0; return 0;}#endif /* * We run out of memory. That means we probably have tons of objects * allocated. */static void debug_objects_oom(void){ struct debug_bucket *db = obj_hash; struct hlist_node *tmp; HLIST_HEAD(freelist); struct debug_obj *obj; unsigned long flags; int i; pr_warn("Out of memory. ODEBUG disabled\n"); for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) { raw_spin_lock_irqsave(&db->lock, flags); hlist_move_list(&db->list, &freelist); raw_spin_unlock_irqrestore(&db->lock, flags); /* Now free them */ hlist_for_each_entry_safe(obj, tmp, &freelist, node) { hlist_del(&obj->node); free_object(obj); } }} /* * We use the pfn of the address for the hash. That way we can check * for freed objects simply by checking the affected bucket. */static struct debug_bucket *get_bucket(unsigned long addr){ unsigned long hash; hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS); return &obj_hash[hash];} static void debug_print_object(struct debug_obj *obj, char *msg){ const struct debug_obj_descr *descr = obj->descr; static int limit; Lines 1–500 of 1406. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Generic infrastructure for lifetime debugging of objects. * * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de> */ #define pr_fmt(fmt) "ODEBUG: " fmt #include <linux/debugobjects.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/sched/task_stack.h> #include <linux/seq_file.h> #include <linux/debugfs.h> #include <linux/slab.h> #include <linux/hash.h> #include <linux/kmemleak.h> #include <linux/cpu.h> #define ODEBUG_HASH_BITS 14 #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) #define ODEBUG_POOL_SIZE 1024 #define ODEBUG_POOL_MIN_LEVEL 256 #define ODEBUG_POOL_PERCPU_SIZE 64 #define ODEBUG_BATCH_SIZE 16 #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT) #define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1)) /* * We limit the freeing of debug objects via workqueue at a maximum * frequency of 10Hz and about 1024 objects for each freeing operation. * So it is freeing at most 10k debug objects per second. */ #define ODEBUG_FREE_WORK_MAX 1024 #define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10) struct debug_bucket { struct hlist_head list; raw_spinlock_t lock; }; /* * Debug object percpu free list * Access is protected by disabling irq */ struct debug_percpu_free { struct hlist_head free_objs; int obj_free; }; static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool); static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; static DEFINE_RAW_SPINLOCK(pool_lock); static HLIST_HEAD(obj_pool); static HLIST_HEAD(obj_to_free); /* * Because of the presence of percpu free pools, obj_pool_free will * under-count those in the percpu free pools. Similarly, obj_pool_used * will over-count those in the percpu free pools. Adjustments will be * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used * can be off. */ static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE; static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE; static int obj_pool_used; static int __data_racy obj_pool_max_used; static bool obj_freeing; /* The number of objs on the global free list */ static int obj_nr_tofree; static int __data_racy debug_objects_maxchain __read_mostly; static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly; static int __data_racy debug_objects_fixups __read_mostly; static int __data_racy debug_objects_warnings __read_mostly; static int __data_racy debug_objects_enabled __read_mostly = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT; static int debug_objects_pool_size __ro_after_init = ODEBUG_POOL_SIZE; static int debug_objects_pool_min_level __ro_after_init = ODEBUG_POOL_MIN_LEVEL; static const struct debug_obj_descr *descr_test __read_mostly; static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */ static int __data_racy debug_objects_allocated; static int __data_racy debug_objects_freed; static void free_obj_work(struct work_struct *work); static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static int __init enable_object_debug(char *str) { debug_objects_enabled = 1; return 0; } static int __init disable_object_debug(char *str) { debug_objects_enabled = 0; return 0; } early_param("debug_objects", enable_object_debug); early_param("no_debug_objects", disable_object_debug); static const char *obj_states[ODEBUG_STATE_MAX] = { [ODEBUG_STATE_NONE] = "none", [ODEBUG_STATE_INIT] = "initialized", [ODEBUG_STATE_INACTIVE] = "inactive", [ODEBUG_STATE_ACTIVE] = "active", [ODEBUG_STATE_DESTROYED] = "destroyed", [ODEBUG_STATE_NOTAVAILABLE] = "not available", }; static void fill_pool(void) { gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; struct debug_obj *obj; unsigned long flags; if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level)) return; /* * Reuse objs from the global obj_to_free list; they will be * reinitialized when allocating. * * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with * the WRITE_ONCE() in pool_lock critical sections. */ if (READ_ONCE(obj_nr_tofree)) { raw_spin_lock_irqsave(&pool_lock, flags); /* * Recheck with the lock held as the worker thread might have * won the race and freed the global free list already. */ while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); hlist_add_head(&obj->node, &obj_pool); WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } if (unlikely(!obj_cache)) return; while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) { struct debug_obj *new[ODEBUG_BATCH_SIZE]; int cnt; for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { new[cnt] = kmem_cache_zalloc(obj_cache, gfp); if (!new[cnt]) break; } if (!cnt) return; raw_spin_lock_irqsave(&pool_lock, flags); while (cnt) { hlist_add_head(&new[--cnt]->node, &obj_pool); debug_objects_allocated++; WRITE_ONCE(obj_pool_free, obj_pool_free + 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); } } /* * Lookup an object in the hash bucket. */ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) { struct debug_obj *obj; int cnt = 0; hlist_for_each_entry(obj, &b->list, node) { cnt++; if (obj->object == addr) return obj; } if (cnt > debug_objects_maxchain) debug_objects_maxchain = cnt; return NULL; } /* * Allocate a new object from the hlist */ static struct debug_obj *__alloc_object(struct hlist_head *list) { struct debug_obj *obj = NULL; if (list->first) { obj = hlist_entry(list->first, typeof(*obj), node); hlist_del(&obj->node); } return obj; } static struct debug_obj * alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr) { struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool); struct debug_obj *obj; if (likely(obj_cache)) { obj = __alloc_object(&percpu_pool->free_objs); if (obj) { percpu_pool->obj_free--; goto init_obj; } } raw_spin_lock(&pool_lock); obj = __alloc_object(&obj_pool); if (obj) { obj_pool_used++; WRITE_ONCE(obj_pool_free, obj_pool_free - 1); /* * Looking ahead, allocate one batch of debug objects and * put them into the percpu free pool. */ if (likely(obj_cache)) { int i; for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { struct debug_obj *obj2; obj2 = __alloc_object(&obj_pool); if (!obj2) break; hlist_add_head(&obj2->node, &percpu_pool->free_objs); percpu_pool->obj_free++; obj_pool_used++; WRITE_ONCE(obj_pool_free, obj_pool_free - 1); } } if (obj_pool_used > obj_pool_max_used) obj_pool_max_used = obj_pool_used; if (obj_pool_free < obj_pool_min_free) obj_pool_min_free = obj_pool_free; } raw_spin_unlock(&pool_lock); init_obj: if (obj) { obj->object = addr; obj->descr = descr; obj->state = ODEBUG_STATE_NONE; obj->astate = 0; hlist_add_head(&obj->node, &b->list); } return obj; } /* * workqueue function to free objects. * * To reduce contention on the global pool_lock, the actual freeing of * debug objects will be delayed if the pool_lock is busy. */ static void free_obj_work(struct work_struct *work) { struct hlist_node *tmp; struct debug_obj *obj; unsigned long flags; HLIST_HEAD(tofree); WRITE_ONCE(obj_freeing, false); if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return; if (obj_pool_free >= debug_objects_pool_size) goto free_objs; /* * The objs on the pool list might be allocated before the work is * run, so recheck if pool list it full or not, if not fill pool * list from the global free list. As it is likely that a workload * may be gearing up to use more and more objects, don't free any * of them until the next round. */ while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) { obj = hlist_entry(obj_to_free.first, typeof(*obj), node); hlist_del(&obj->node); hlist_add_head(&obj->node, &obj_pool); WRITE_ONCE(obj_pool_free, obj_pool_free + 1); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1); } raw_spin_unlock_irqrestore(&pool_lock, flags); return; free_objs: /* * Pool list is already full and there are still objs on the free * list. Move remaining free objs to a temporary list to free the * memory outside the pool_lock held region. */ if (obj_nr_tofree) { hlist_move_list(&obj_to_free, &tofree); debug_objects_freed += obj_nr_tofree; WRITE_ONCE(obj_nr_tofree, 0); } raw_spin_unlock_irqrestore(&pool_lock, flags); hlist_for_each_entry_safe(obj, tmp, &tofree, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); } } static void __free_object(struct debug_obj *obj) { struct debug_obj *objs[ODEBUG_BATCH_SIZE]; struct debug_percpu_free *percpu_pool; int lookahead_count = 0; unsigned long flags; bool work; local_irq_save(flags); if (!obj_cache) goto free_to_obj_pool; /* * Try to free it into the percpu pool first. */ percpu_pool = this_cpu_ptr(&percpu_obj_pool); if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) { hlist_add_head(&obj->node, &percpu_pool->free_objs); percpu_pool->obj_free++; local_irq_restore(flags); return; } /* * As the percpu pool is full, look ahead and pull out a batch * of objects from the percpu pool and free them as well. */ for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) { objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs); if (!objs[lookahead_count]) break; percpu_pool->obj_free--; } free_to_obj_pool: raw_spin_lock(&pool_lock); work = (obj_pool_free > debug_objects_pool_size) && obj_cache && (obj_nr_tofree < ODEBUG_FREE_WORK_MAX); obj_pool_used--; if (work) { WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); hlist_add_head(&obj->node, &obj_to_free); if (lookahead_count) { WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, &obj_to_free); } } if ((obj_pool_free > debug_objects_pool_size) && (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) { int i; /* * Free one more batch of objects from obj_pool. */ for (i = 0; i < ODEBUG_BATCH_SIZE; i++) { obj = __alloc_object(&obj_pool); hlist_add_head(&obj->node, &obj_to_free); WRITE_ONCE(obj_pool_free, obj_pool_free - 1); WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1); } } } else { WRITE_ONCE(obj_pool_free, obj_pool_free + 1); hlist_add_head(&obj->node, &obj_pool); if (lookahead_count) { WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count); obj_pool_used -= lookahead_count; while (lookahead_count) { hlist_add_head(&objs[--lookahead_count]->node, &obj_pool); } } } raw_spin_unlock(&pool_lock); local_irq_restore(flags); } /* * Put the object back into the pool and schedule work to free objects * if necessary. */ static void free_object(struct debug_obj *obj) { __free_object(obj); if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) { WRITE_ONCE(obj_freeing, true); schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY); } } #ifdef CONFIG_HOTPLUG_CPU static int object_cpu_offline(unsigned int cpu) { struct debug_percpu_free *percpu_pool; struct hlist_node *tmp; struct debug_obj *obj; unsigned long flags; /* Remote access is safe as the CPU is dead already */ percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu); hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); } raw_spin_lock_irqsave(&pool_lock, flags); obj_pool_used -= percpu_pool->obj_free; debug_objects_freed += percpu_pool->obj_free; raw_spin_unlock_irqrestore(&pool_lock, flags); percpu_pool->obj_free = 0; return 0; } #endif /* * We run out of memory. That means we probably have tons of objects * allocated. */ static void debug_objects_oom(void) { struct debug_bucket *db = obj_hash; struct hlist_node *tmp; HLIST_HEAD(freelist); struct debug_obj *obj; unsigned long flags; int i; pr_warn("Out of memory. ODEBUG disabled\n"); for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) { raw_spin_lock_irqsave(&db->lock, flags); hlist_move_list(&db->list, &freelist); raw_spin_unlock_irqrestore(&db->lock, flags); /* Now free them */ hlist_for_each_entry_safe(obj, tmp, &freelist, node) { hlist_del(&obj->node); free_object(obj); } } } /* * We use the pfn of the address for the hash. That way we can check * for freed objects simply by checking the affected bucket. */ static struct debug_bucket *get_bucket(unsigned long addr) { unsigned long hash; hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS); return &obj_hash[hash]; } static void debug_print_object(struct debug_obj *obj, char *msg) { const struct debug_obj_descr *descr = obj->descr; static int limit;