// SPDX-License-Identifier: GPL-2.0/* * Generic infrastructure for lifetime debugging of objects. * * Copyright (C) 2008, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> */ #define pr_fmt(fmt) "ODEBUG: " fmt #include <linux/cpu.h>#include <linux/debugobjects.h>#include <linux/debugfs.h>#include <linux/hash.h>#include <linux/kmemleak.h>#include <linux/sched.h>#include <linux/sched/loadavg.h>#include <linux/sched/task_stack.h>#include <linux/seq_file.h>#include <linux/slab.h>#include <linux/static_key.h> #define ODEBUG_HASH_BITS 14#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) /* Must be power of two */#define ODEBUG_BATCH_SIZE 16 /* Initial values. Must all be a multiple of batch size */#define ODEBUG_POOL_SIZE (64 * ODEBUG_BATCH_SIZE)#define ODEBUG_POOL_MIN_LEVEL (ODEBUG_POOL_SIZE / 4) #define ODEBUG_POOL_PERCPU_SIZE (8 * ODEBUG_BATCH_SIZE) #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)#define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1)) /* * We limit the freeing of debug objects via workqueue at a maximum * frequency of 10Hz and about 1024 objects for each freeing operation. * So it is freeing at most 10k debug objects per second. */#define ODEBUG_FREE_WORK_MAX (1024 / ODEBUG_BATCH_SIZE)#define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10) struct debug_bucket { struct hlist_head list; raw_spinlock_t lock;}; struct pool_stats { unsigned int cur_used; unsigned int max_used; unsigned int min_fill;}; struct obj_pool { struct hlist_head objects; unsigned int cnt; unsigned int min_cnt; unsigned int max_cnt; struct pool_stats stats;} ____cacheline_aligned; static DEFINE_PER_CPU_ALIGNED(struct obj_pool, pool_pcpu) = { .max_cnt = ODEBUG_POOL_PERCPU_SIZE,}; static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; static DEFINE_RAW_SPINLOCK(pool_lock); static struct obj_pool pool_global = { .min_cnt = ODEBUG_POOL_MIN_LEVEL, .max_cnt = ODEBUG_POOL_SIZE, .stats = { .min_fill = ODEBUG_POOL_SIZE, },}; static struct obj_pool pool_to_free = { .max_cnt = UINT_MAX,}; static HLIST_HEAD(pool_boot); static unsigned long avg_usage;static bool obj_freeing; static int __data_racy debug_objects_maxchain __read_mostly;static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;static int __data_racy debug_objects_fixups __read_mostly;static int __data_racy debug_objects_warnings __read_mostly;static bool __data_racy debug_objects_enabled __read_mostly = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT; static const struct debug_obj_descr *descr_test __read_mostly;static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */static int __data_racy debug_objects_allocated;static int __data_racy debug_objects_freed; static void free_obj_work(struct work_struct *work);static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static DEFINE_STATIC_KEY_FALSE(obj_cache_enabled); static int __init enable_object_debug(char *str){ debug_objects_enabled = true; return 0;}early_param("debug_objects", enable_object_debug); static int __init disable_object_debug(char *str){ debug_objects_enabled = false; return 0;}early_param("no_debug_objects", disable_object_debug); static const char *obj_states[ODEBUG_STATE_MAX] = { [ODEBUG_STATE_NONE] = "none", [ODEBUG_STATE_INIT] = "initialized", [ODEBUG_STATE_INACTIVE] = "inactive", [ODEBUG_STATE_ACTIVE] = "active", [ODEBUG_STATE_DESTROYED] = "destroyed", [ODEBUG_STATE_NOTAVAILABLE] = "not available",}; static __always_inline unsigned int pool_count(struct obj_pool *pool){ return READ_ONCE(pool->cnt);} static __always_inline bool pool_should_refill(struct obj_pool *pool){ return pool_count(pool) < pool->min_cnt;} static __always_inline bool pool_must_refill(struct obj_pool *pool){ return pool_count(pool) < pool->min_cnt / 2;} static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src){ struct hlist_node *last, *next_batch, *first_batch; struct debug_obj *obj; if (dst->cnt >= dst->max_cnt || !src->cnt) return false; first_batch = src->objects.first; obj = hlist_entry(first_batch, typeof(*obj), node); last = obj->batch_last; next_batch = last->next; /* Move the next batch to the front of the source pool */ src->objects.first = next_batch; if (next_batch) next_batch->pprev = &src->objects.first; /* Add the extracted batch to the destination pool */ last->next = dst->objects.first; if (last->next) last->next->pprev = &last->next; first_batch->pprev = &dst->objects.first; dst->objects.first = first_batch; WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE); return true;} static bool pool_push_batch(struct obj_pool *dst, struct hlist_head *head){ struct hlist_node *last; struct debug_obj *obj; if (dst->cnt >= dst->max_cnt) return false; obj = hlist_entry(head->first, typeof(*obj), node); last = obj->batch_last; hlist_splice_init(head, last, &dst->objects); WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE); return true;} static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src){ struct hlist_node *last, *next; struct debug_obj *obj; if (!src->cnt) return false; /* Move the complete list to the head */ hlist_move_list(&src->objects, head); obj = hlist_entry(head->first, typeof(*obj), node); last = obj->batch_last; next = last->next; /* Disconnect the batch from the list */ last->next = NULL; /* Move the node after last back to the source pool. */ src->objects.first = next; if (next) next->pprev = &src->objects.first; WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); return true;} static struct debug_obj *__alloc_object(struct hlist_head *list){ struct debug_obj *obj; if (unlikely(!list->first)) return NULL; obj = hlist_entry(list->first, typeof(*obj), node); hlist_del(&obj->node); return obj;} static void pcpu_refill_stats(void){ struct pool_stats *stats = &pool_global.stats; WRITE_ONCE(stats->cur_used, stats->cur_used + ODEBUG_BATCH_SIZE); if (stats->cur_used > stats->max_used) stats->max_used = stats->cur_used; if (pool_global.cnt < stats->min_fill) stats->min_fill = pool_global.cnt;} static struct debug_obj *pcpu_alloc(void){ struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); lockdep_assert_irqs_disabled(); for (;;) { struct debug_obj *obj = __alloc_object(&pcp->objects); if (likely(obj)) { pcp->cnt--; /* * If this emptied a batch try to refill from the * free pool. Don't do that if this was the top-most * batch as pcpu_free() expects the per CPU pool * to be less than ODEBUG_POOL_PERCPU_SIZE. */ if (unlikely(pcp->cnt < (ODEBUG_POOL_PERCPU_SIZE - ODEBUG_BATCH_SIZE) && !(pcp->cnt % ODEBUG_BATCH_SIZE))) { /* * Don't try to allocate from the regular pool here * to not exhaust it prematurely. */ if (pool_count(&pool_to_free)) { guard(raw_spinlock)(&pool_lock); pool_move_batch(pcp, &pool_to_free); pcpu_refill_stats(); } } return obj; } guard(raw_spinlock)(&pool_lock); if (!pool_move_batch(pcp, &pool_to_free)) { if (!pool_move_batch(pcp, &pool_global)) return NULL; } pcpu_refill_stats(); }} static void pcpu_free(struct debug_obj *obj){ struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); struct debug_obj *first; lockdep_assert_irqs_disabled(); if (!(pcp->cnt % ODEBUG_BATCH_SIZE)) { obj->batch_last = &obj->node; } else { first = hlist_entry(pcp->objects.first, typeof(*first), node); obj->batch_last = first->batch_last; } hlist_add_head(&obj->node, &pcp->objects); pcp->cnt++; /* Pool full ? */ if (pcp->cnt < ODEBUG_POOL_PERCPU_SIZE) return; /* Remove a batch from the per CPU pool */ guard(raw_spinlock)(&pool_lock); /* Try to fit the batch into the pool_global first */ if (!pool_move_batch(&pool_global, pcp)) pool_move_batch(&pool_to_free, pcp); WRITE_ONCE(pool_global.stats.cur_used, pool_global.stats.cur_used - ODEBUG_BATCH_SIZE);} static void free_object_list(struct hlist_head *head){ struct hlist_node *tmp; struct debug_obj *obj; int cnt = 0; hlist_for_each_entry_safe(obj, tmp, head, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); cnt++; } debug_objects_freed += cnt;} static void fill_pool_from_freelist(void){ static unsigned long state; /* * Reuse objs from the global obj_to_free list; they will be * reinitialized when allocating. */ if (!pool_count(&pool_to_free)) return; /* * Prevent the context from being scheduled or interrupted after * setting the state flag; */ guard(irqsave)(); /* * Avoid lock contention on &pool_lock and avoid making the cache * line exclusive by testing the bit before attempting to set it. */ if (test_bit(0, &state) || test_and_set_bit(0, &state)) return; /* Avoid taking the lock when there is no work to do */ while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) { guard(raw_spinlock)(&pool_lock); /* Move a batch if possible */ pool_move_batch(&pool_global, &pool_to_free); } clear_bit(0, &state);} static bool kmem_alloc_batch(struct hlist_head *head, struct kmem_cache *cache, gfp_t gfp){ struct hlist_node *last = NULL; struct debug_obj *obj; for (int cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { obj = kmem_cache_zalloc(cache, gfp); if (!obj) { free_object_list(head); return false; } debug_objects_allocated++; if (!last) last = &obj->node; obj->batch_last = last; hlist_add_head(&obj->node, head); } return true;} static void fill_pool(void){ static atomic_t cpus_allocating; /* * Avoid allocation and lock contention when: * - One other CPU is already allocating * - the global pool has not reached the critical level yet */ if (!pool_must_refill(&pool_global) && atomic_read(&cpus_allocating)) return; atomic_inc(&cpus_allocating); while (pool_should_refill(&pool_global)) { gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; HLIST_HEAD(head); /* * Allow reclaim only in preemptible context and during * early boot. If not preemptible, the caller might hold * locks causing a deadlock in the allocator. * * If the reclaim flag is not set during early boot then * allocations, which happen before deferred page * initialization has completed, will fail. * * In preemptible context the flag is harmless and not a * performance issue as that's usually invoked from slow * path initialization context. */ if (preemptible() || system_state < SYSTEM_SCHEDULING) gfp |= __GFP_KSWAPD_RECLAIM; if (!kmem_alloc_batch(&head, obj_cache, gfp)) break; guard(raw_spinlock_irqsave)(&pool_lock); if (!pool_push_batch(&pool_global, &head)) pool_push_batch(&pool_to_free, &head); } atomic_dec(&cpus_allocating);} /* * Lookup an object in the hash bucket. */static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b){ struct debug_obj *obj; int cnt = 0; hlist_for_each_entry(obj, &b->list, node) { cnt++; if (obj->object == addr) return obj; } if (cnt > debug_objects_maxchain) debug_objects_maxchain = cnt; return NULL;} static void calc_usage(void){ static DEFINE_RAW_SPINLOCK(avg_lock); static unsigned long avg_period; unsigned long cur, now = jiffies; if (!time_after_eq(now, READ_ONCE(avg_period))) return; if (!raw_spin_trylock(&avg_lock)) return; WRITE_ONCE(avg_period, now + msecs_to_jiffies(10)); cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX; WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur)); raw_spin_unlock(&avg_lock);} static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr){ struct debug_obj *obj; calc_usage(); if (static_branch_likely(&obj_cache_enabled)) obj = pcpu_alloc(); else obj = __alloc_object(&pool_boot); if (likely(obj)) { obj->object = addr; obj->descr = descr; obj->state = ODEBUG_STATE_NONE; obj->astate = 0; hlist_add_head(&obj->node, &b->list); } return obj;} /* workqueue function to free objects. */static void free_obj_work(struct work_struct *work){ static unsigned long last_use_avg; unsigned long cur_used, last_used, delta; unsigned int max_free = 0; WRITE_ONCE(obj_freeing, false); /* Rate limit freeing based on current use average */ cur_used = READ_ONCE(avg_usage); last_used = last_use_avg;Lines 1–500 of 1568. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Generic infrastructure for lifetime debugging of objects. * * Copyright (C) 2008, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> */ #define pr_fmt(fmt) "ODEBUG: " fmt #include <linux/cpu.h> #include <linux/debugobjects.h> #include <linux/debugfs.h> #include <linux/hash.h> #include <linux/kmemleak.h> #include <linux/sched.h> #include <linux/sched/loadavg.h> #include <linux/sched/task_stack.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/static_key.h> #define ODEBUG_HASH_BITS 14 #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS) /* Must be power of two */ #define ODEBUG_BATCH_SIZE 16 /* Initial values. Must all be a multiple of batch size */ #define ODEBUG_POOL_SIZE (64 * ODEBUG_BATCH_SIZE) #define ODEBUG_POOL_MIN_LEVEL (ODEBUG_POOL_SIZE / 4) #define ODEBUG_POOL_PERCPU_SIZE (8 * ODEBUG_BATCH_SIZE) #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT) #define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1)) /* * We limit the freeing of debug objects via workqueue at a maximum * frequency of 10Hz and about 1024 objects for each freeing operation. * So it is freeing at most 10k debug objects per second. */ #define ODEBUG_FREE_WORK_MAX (1024 / ODEBUG_BATCH_SIZE) #define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10) struct debug_bucket { struct hlist_head list; raw_spinlock_t lock; }; struct pool_stats { unsigned int cur_used; unsigned int max_used; unsigned int min_fill; }; struct obj_pool { struct hlist_head objects; unsigned int cnt; unsigned int min_cnt; unsigned int max_cnt; struct pool_stats stats; } ____cacheline_aligned; static DEFINE_PER_CPU_ALIGNED(struct obj_pool, pool_pcpu) = { .max_cnt = ODEBUG_POOL_PERCPU_SIZE, }; static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE]; static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; static DEFINE_RAW_SPINLOCK(pool_lock); static struct obj_pool pool_global = { .min_cnt = ODEBUG_POOL_MIN_LEVEL, .max_cnt = ODEBUG_POOL_SIZE, .stats = { .min_fill = ODEBUG_POOL_SIZE, }, }; static struct obj_pool pool_to_free = { .max_cnt = UINT_MAX, }; static HLIST_HEAD(pool_boot); static unsigned long avg_usage; static bool obj_freeing; static int __data_racy debug_objects_maxchain __read_mostly; static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly; static int __data_racy debug_objects_fixups __read_mostly; static int __data_racy debug_objects_warnings __read_mostly; static bool __data_racy debug_objects_enabled __read_mostly = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT; static const struct debug_obj_descr *descr_test __read_mostly; static struct kmem_cache *obj_cache __ro_after_init; /* * Track numbers of kmem_cache_alloc()/free() calls done. */ static int __data_racy debug_objects_allocated; static int __data_racy debug_objects_freed; static void free_obj_work(struct work_struct *work); static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work); static DEFINE_STATIC_KEY_FALSE(obj_cache_enabled); static int __init enable_object_debug(char *str) { debug_objects_enabled = true; return 0; } early_param("debug_objects", enable_object_debug); static int __init disable_object_debug(char *str) { debug_objects_enabled = false; return 0; } early_param("no_debug_objects", disable_object_debug); static const char *obj_states[ODEBUG_STATE_MAX] = { [ODEBUG_STATE_NONE] = "none", [ODEBUG_STATE_INIT] = "initialized", [ODEBUG_STATE_INACTIVE] = "inactive", [ODEBUG_STATE_ACTIVE] = "active", [ODEBUG_STATE_DESTROYED] = "destroyed", [ODEBUG_STATE_NOTAVAILABLE] = "not available", }; static __always_inline unsigned int pool_count(struct obj_pool *pool) { return READ_ONCE(pool->cnt); } static __always_inline bool pool_should_refill(struct obj_pool *pool) { return pool_count(pool) < pool->min_cnt; } static __always_inline bool pool_must_refill(struct obj_pool *pool) { return pool_count(pool) < pool->min_cnt / 2; } static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src) { struct hlist_node *last, *next_batch, *first_batch; struct debug_obj *obj; if (dst->cnt >= dst->max_cnt || !src->cnt) return false; first_batch = src->objects.first; obj = hlist_entry(first_batch, typeof(*obj), node); last = obj->batch_last; next_batch = last->next; /* Move the next batch to the front of the source pool */ src->objects.first = next_batch; if (next_batch) next_batch->pprev = &src->objects.first; /* Add the extracted batch to the destination pool */ last->next = dst->objects.first; if (last->next) last->next->pprev = &last->next; first_batch->pprev = &dst->objects.first; dst->objects.first = first_batch; WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE); return true; } static bool pool_push_batch(struct obj_pool *dst, struct hlist_head *head) { struct hlist_node *last; struct debug_obj *obj; if (dst->cnt >= dst->max_cnt) return false; obj = hlist_entry(head->first, typeof(*obj), node); last = obj->batch_last; hlist_splice_init(head, last, &dst->objects); WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE); return true; } static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src) { struct hlist_node *last, *next; struct debug_obj *obj; if (!src->cnt) return false; /* Move the complete list to the head */ hlist_move_list(&src->objects, head); obj = hlist_entry(head->first, typeof(*obj), node); last = obj->batch_last; next = last->next; /* Disconnect the batch from the list */ last->next = NULL; /* Move the node after last back to the source pool. */ src->objects.first = next; if (next) next->pprev = &src->objects.first; WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE); return true; } static struct debug_obj *__alloc_object(struct hlist_head *list) { struct debug_obj *obj; if (unlikely(!list->first)) return NULL; obj = hlist_entry(list->first, typeof(*obj), node); hlist_del(&obj->node); return obj; } static void pcpu_refill_stats(void) { struct pool_stats *stats = &pool_global.stats; WRITE_ONCE(stats->cur_used, stats->cur_used + ODEBUG_BATCH_SIZE); if (stats->cur_used > stats->max_used) stats->max_used = stats->cur_used; if (pool_global.cnt < stats->min_fill) stats->min_fill = pool_global.cnt; } static struct debug_obj *pcpu_alloc(void) { struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); lockdep_assert_irqs_disabled(); for (;;) { struct debug_obj *obj = __alloc_object(&pcp->objects); if (likely(obj)) { pcp->cnt--; /* * If this emptied a batch try to refill from the * free pool. Don't do that if this was the top-most * batch as pcpu_free() expects the per CPU pool * to be less than ODEBUG_POOL_PERCPU_SIZE. */ if (unlikely(pcp->cnt < (ODEBUG_POOL_PERCPU_SIZE - ODEBUG_BATCH_SIZE) && !(pcp->cnt % ODEBUG_BATCH_SIZE))) { /* * Don't try to allocate from the regular pool here * to not exhaust it prematurely. */ if (pool_count(&pool_to_free)) { guard(raw_spinlock)(&pool_lock); pool_move_batch(pcp, &pool_to_free); pcpu_refill_stats(); } } return obj; } guard(raw_spinlock)(&pool_lock); if (!pool_move_batch(pcp, &pool_to_free)) { if (!pool_move_batch(pcp, &pool_global)) return NULL; } pcpu_refill_stats(); } } static void pcpu_free(struct debug_obj *obj) { struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu); struct debug_obj *first; lockdep_assert_irqs_disabled(); if (!(pcp->cnt % ODEBUG_BATCH_SIZE)) { obj->batch_last = &obj->node; } else { first = hlist_entry(pcp->objects.first, typeof(*first), node); obj->batch_last = first->batch_last; } hlist_add_head(&obj->node, &pcp->objects); pcp->cnt++; /* Pool full ? */ if (pcp->cnt < ODEBUG_POOL_PERCPU_SIZE) return; /* Remove a batch from the per CPU pool */ guard(raw_spinlock)(&pool_lock); /* Try to fit the batch into the pool_global first */ if (!pool_move_batch(&pool_global, pcp)) pool_move_batch(&pool_to_free, pcp); WRITE_ONCE(pool_global.stats.cur_used, pool_global.stats.cur_used - ODEBUG_BATCH_SIZE); } static void free_object_list(struct hlist_head *head) { struct hlist_node *tmp; struct debug_obj *obj; int cnt = 0; hlist_for_each_entry_safe(obj, tmp, head, node) { hlist_del(&obj->node); kmem_cache_free(obj_cache, obj); cnt++; } debug_objects_freed += cnt; } static void fill_pool_from_freelist(void) { static unsigned long state; /* * Reuse objs from the global obj_to_free list; they will be * reinitialized when allocating. */ if (!pool_count(&pool_to_free)) return; /* * Prevent the context from being scheduled or interrupted after * setting the state flag; */ guard(irqsave)(); /* * Avoid lock contention on &pool_lock and avoid making the cache * line exclusive by testing the bit before attempting to set it. */ if (test_bit(0, &state) || test_and_set_bit(0, &state)) return; /* Avoid taking the lock when there is no work to do */ while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) { guard(raw_spinlock)(&pool_lock); /* Move a batch if possible */ pool_move_batch(&pool_global, &pool_to_free); } clear_bit(0, &state); } static bool kmem_alloc_batch(struct hlist_head *head, struct kmem_cache *cache, gfp_t gfp) { struct hlist_node *last = NULL; struct debug_obj *obj; for (int cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) { obj = kmem_cache_zalloc(cache, gfp); if (!obj) { free_object_list(head); return false; } debug_objects_allocated++; if (!last) last = &obj->node; obj->batch_last = last; hlist_add_head(&obj->node, head); } return true; } static void fill_pool(void) { static atomic_t cpus_allocating; /* * Avoid allocation and lock contention when: * - One other CPU is already allocating * - the global pool has not reached the critical level yet */ if (!pool_must_refill(&pool_global) && atomic_read(&cpus_allocating)) return; atomic_inc(&cpus_allocating); while (pool_should_refill(&pool_global)) { gfp_t gfp = __GFP_HIGH | __GFP_NOWARN; HLIST_HEAD(head); /* * Allow reclaim only in preemptible context and during * early boot. If not preemptible, the caller might hold * locks causing a deadlock in the allocator. * * If the reclaim flag is not set during early boot then * allocations, which happen before deferred page * initialization has completed, will fail. * * In preemptible context the flag is harmless and not a * performance issue as that's usually invoked from slow * path initialization context. */ if (preemptible() || system_state < SYSTEM_SCHEDULING) gfp |= __GFP_KSWAPD_RECLAIM; if (!kmem_alloc_batch(&head, obj_cache, gfp)) break; guard(raw_spinlock_irqsave)(&pool_lock); if (!pool_push_batch(&pool_global, &head)) pool_push_batch(&pool_to_free, &head); } atomic_dec(&cpus_allocating); } /* * Lookup an object in the hash bucket. */ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) { struct debug_obj *obj; int cnt = 0; hlist_for_each_entry(obj, &b->list, node) { cnt++; if (obj->object == addr) return obj; } if (cnt > debug_objects_maxchain) debug_objects_maxchain = cnt; return NULL; } static void calc_usage(void) { static DEFINE_RAW_SPINLOCK(avg_lock); static unsigned long avg_period; unsigned long cur, now = jiffies; if (!time_after_eq(now, READ_ONCE(avg_period))) return; if (!raw_spin_trylock(&avg_lock)) return; WRITE_ONCE(avg_period, now + msecs_to_jiffies(10)); cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX; WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur)); raw_spin_unlock(&avg_lock); } static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr) { struct debug_obj *obj; calc_usage(); if (static_branch_likely(&obj_cache_enabled)) obj = pcpu_alloc(); else obj = __alloc_object(&pool_boot); if (likely(obj)) { obj->object = addr; obj->descr = descr; obj->state = ODEBUG_STATE_NONE; obj->astate = 0; hlist_add_head(&obj->node, &b->list); } return obj; } /* workqueue function to free objects. */ static void free_obj_work(struct work_struct *work) { static unsigned long last_use_avg; unsigned long cur_used, last_used, delta; unsigned int max_free = 0; WRITE_ONCE(obj_freeing, false); /* Rate limit freeing based on current use average */ cur_used = READ_ONCE(avg_usage); last_used = last_use_avg;