// SPDX-License-Identifier: GPL-2.0-only/* * Stack depot - a stack trace storage that avoids duplication. * * Internally, stack depot maintains a hash table of unique stacktraces. The * stack traces themselves are stored contiguously one after another in a set * of separate page allocations. * * Author: Alexander Potapenko <glider@google.com> * Copyright (C) 2016 Google, Inc. * * Based on the code by Dmitry Chernenkov. */ #define pr_fmt(fmt) "stackdepot: " fmt #include <linux/debugfs.h>#include <linux/gfp.h>#include <linux/jhash.h>#include <linux/kernel.h>#include <linux/kmsan.h>#include <linux/list.h>#include <linux/mm.h>#include <linux/mutex.h>#include <linux/poison.h>#include <linux/printk.h>#include <linux/rculist.h>#include <linux/rcupdate.h>#include <linux/refcount.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/stacktrace.h>#include <linux/stackdepot.h>#include <linux/string.h>#include <linux/types.h>#include <linux/memblock.h>#include <linux/kasan-enabled.h> /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); static bool stack_depot_disabled;static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT);static bool __stack_depot_early_init_passed __initdata; /* Use one hash table bucket per 16 KB of memory. */#define STACK_HASH_TABLE_SCALE 14/* Limit the number of buckets between 4K and 1M. */#define STACK_BUCKET_NUMBER_ORDER_MIN 12#define STACK_BUCKET_NUMBER_ORDER_MAX 20/* Initial seed for jhash2. */#define STACK_HASH_SEED 0x9747b28c /* Hash table of stored stack records. */static struct list_head *stack_table;/* Fixed order of the number of table buckets. Used when KASAN is enabled. */static unsigned int stack_bucket_number_order;/* Hash mask for indexing the table. */static unsigned int stack_hash_mask; /* The lock must be held when performing pool or freelist modifications. */static DEFINE_RAW_SPINLOCK(pool_lock);/* Array of memory regions that store stack records. */static void **stack_pools __pt_guarded_by(&pool_lock);/* Newly allocated pool that is not yet added to stack_pools. */static void *new_pool;/* Number of pools in stack_pools. */static int pools_num;/* Offset to the unused space in the currently used pool. */static size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE;/* Freelist of stack records within stack_pools. */static __guarded_by(&pool_lock) LIST_HEAD(free_stacks); /* Statistics counters for debugfs. */enum depot_counter_id { DEPOT_COUNTER_REFD_ALLOCS, DEPOT_COUNTER_REFD_FREES, DEPOT_COUNTER_REFD_INUSE, DEPOT_COUNTER_FREELIST_SIZE, DEPOT_COUNTER_PERSIST_COUNT, DEPOT_COUNTER_PERSIST_BYTES, DEPOT_COUNTER_COUNT,};static long counters[DEPOT_COUNTER_COUNT];static const char *const counter_names[] = { [DEPOT_COUNTER_REFD_ALLOCS] = "refcounted_allocations", [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes",};static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); static int __init disable_stack_depot(char *str){ return kstrtobool(str, &stack_depot_disabled);}early_param("stack_depot_disable", disable_stack_depot); static int __init parse_max_pools(char *str){ const long long limit = (1LL << (DEPOT_POOL_INDEX_BITS)) - 1; unsigned int max_pools; int rv; rv = kstrtouint(str, 0, &max_pools); if (rv) return rv; if (max_pools < 1024) { pr_err("stack_depot_max_pools below 1024, using default of %u\n", stack_max_pools); goto out; } if (max_pools > limit) { pr_err("stack_depot_max_pools exceeds %lld, using default of %u\n", limit, stack_max_pools); goto out; } stack_max_pools = max_pools;out: return 0;}early_param("stack_depot_max_pools", parse_max_pools); void __init stack_depot_request_early_init(void){ /* Too late to request early init now. */ WARN_ON(__stack_depot_early_init_passed); __stack_depot_early_init_requested = true;} /* Initialize list_head's within the hash table. */static void init_stack_table(unsigned long entries){ unsigned long i; for (i = 0; i < entries; i++) INIT_LIST_HEAD(&stack_table[i]);} /* Allocates a hash table via memblock. Can only be used during early boot. */int __init stack_depot_early_init(void){ unsigned long entries = 0; /* This function must be called only once, from mm_init(). */ if (WARN_ON(__stack_depot_early_init_passed)) return 0; __stack_depot_early_init_passed = true; /* * Print disabled message even if early init has not been requested: * stack_depot_init() will not print one. */ if (stack_depot_disabled) { pr_info("disabled\n"); return 0; } /* * If KASAN is enabled, use the maximum order: KASAN is frequently used * in fuzzing scenarios, which leads to a large number of different * stack traces being stored in stack depot. */ if (kasan_enabled() && !stack_bucket_number_order) stack_bucket_number_order = STACK_BUCKET_NUMBER_ORDER_MAX; /* * Check if early init has been requested after setting * stack_bucket_number_order: stack_depot_init() uses its value. */ if (!__stack_depot_early_init_requested) return 0; /* * If stack_bucket_number_order is not set, leave entries as 0 to rely * on the automatic calculations performed by alloc_large_system_hash(). */ if (stack_bucket_number_order) entries = 1UL << stack_bucket_number_order; pr_info("allocating hash table via alloc_large_system_hash\n"); stack_table = alloc_large_system_hash("stackdepot", sizeof(struct list_head), entries, STACK_HASH_TABLE_SCALE, HASH_EARLY, NULL, &stack_hash_mask, 1UL << STACK_BUCKET_NUMBER_ORDER_MIN, 1UL << STACK_BUCKET_NUMBER_ORDER_MAX); if (!stack_table) { pr_err("hash table allocation failed, disabling\n"); stack_depot_disabled = true; return -ENOMEM; } if (!entries) { /* * Obtain the number of entries that was calculated by * alloc_large_system_hash(). */ entries = stack_hash_mask + 1; } init_stack_table(entries); pr_info("allocating space for %u stack pools via memblock\n", stack_max_pools); stack_pools = memblock_alloc(stack_max_pools * sizeof(void *), PAGE_SIZE); if (!stack_pools) { pr_err("stack pools allocation failed, disabling\n"); memblock_free(stack_table, entries * sizeof(struct list_head)); stack_depot_disabled = true; return -ENOMEM; } return 0;} /* Allocates a hash table via kvcalloc. Can be used after boot. */int stack_depot_init(void){ static DEFINE_MUTEX(stack_depot_init_mutex); unsigned long entries; int ret = 0; mutex_lock(&stack_depot_init_mutex); if (stack_depot_disabled || stack_table) goto out_unlock; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order * if assigned, and rely on automatic scaling otherwise. */ if (stack_bucket_number_order) { entries = 1UL << stack_bucket_number_order; } else { int scale = STACK_HASH_TABLE_SCALE; entries = nr_free_buffer_pages(); entries = roundup_pow_of_two(entries); if (scale > PAGE_SHIFT) entries >>= (scale - PAGE_SHIFT); else entries <<= (PAGE_SHIFT - scale); } if (entries < 1UL << STACK_BUCKET_NUMBER_ORDER_MIN) entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MIN; if (entries > 1UL << STACK_BUCKET_NUMBER_ORDER_MAX) entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX; pr_info("allocating hash table of %lu entries via kvcalloc\n", entries); stack_table = kvzalloc_objs(struct list_head, entries); if (!stack_table) { pr_err("hash table allocation failed, disabling\n"); stack_depot_disabled = true; ret = -ENOMEM; goto out_unlock; } stack_hash_mask = entries - 1; init_stack_table(entries); pr_info("allocating space for %u stack pools via kvcalloc\n", stack_max_pools); stack_pools = kvcalloc(stack_max_pools, sizeof(void *), GFP_KERNEL); if (!stack_pools) { pr_err("stack pools allocation failed, disabling\n"); kvfree(stack_table); stack_depot_disabled = true; ret = -ENOMEM; } out_unlock: mutex_unlock(&stack_depot_init_mutex); return ret;}EXPORT_SYMBOL_GPL(stack_depot_init); /* * Initializes new stack pool, and updates the list of pools. */static bool depot_init_pool(void **prealloc) __must_hold(&pool_lock){ lockdep_assert_held(&pool_lock); if (unlikely(pools_num >= stack_max_pools)) { /* Bail out if we reached the pool limit. */ WARN_ON_ONCE(pools_num > stack_max_pools); /* should never happen */ WARN_ON_ONCE(!new_pool); /* to avoid unnecessary pre-allocation */ WARN_ONCE(1, "Stack depot reached limit capacity"); return false; } if (!new_pool && *prealloc) { /* We have preallocated memory, use it. */ WRITE_ONCE(new_pool, *prealloc); *prealloc = NULL; } if (!new_pool) return false; /* new_pool and *prealloc are NULL */ /* Save reference to the pool to be used by depot_fetch_stack(). */ stack_pools[pools_num] = new_pool; /* * Stack depot tries to keep an extra pool allocated even before it runs * out of space in the currently used pool. * * To indicate that a new preallocation is needed new_pool is reset to * NULL; do not reset to NULL if we have reached the maximum number of * pools. */ if (pools_num < stack_max_pools) WRITE_ONCE(new_pool, NULL); else WRITE_ONCE(new_pool, STACK_DEPOT_POISON); /* Pairs with concurrent READ_ONCE() in depot_fetch_stack(). */ WRITE_ONCE(pools_num, pools_num + 1); ASSERT_EXCLUSIVE_WRITER(pools_num); pool_offset = 0; return true;} /* Keeps the preallocated memory to be used for a new stack depot pool. */static void depot_keep_new_pool(void **prealloc) __must_hold(&pool_lock){ lockdep_assert_held(&pool_lock); /* * If a new pool is already saved or the maximum number of * pools is reached, do not use the preallocated memory. */ if (new_pool) return; WRITE_ONCE(new_pool, *prealloc); *prealloc = NULL;} /* * Try to initialize a new stack record from the current pool, a cached pool, or * the current pre-allocation. */static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size) __must_hold(&pool_lock){ struct stack_record *stack; void *current_pool; u32 pool_index; lockdep_assert_held(&pool_lock); if (pool_offset + size > DEPOT_POOL_SIZE) { if (!depot_init_pool(prealloc)) return NULL; } if (WARN_ON_ONCE(pools_num < 1)) return NULL; pool_index = pools_num - 1; current_pool = stack_pools[pool_index]; if (WARN_ON_ONCE(!current_pool)) return NULL; stack = current_pool + pool_offset; /* Pre-initialize handle once. */ stack->handle.pool_index_plus_1 = pool_index + 1; stack->handle.offset = pool_offset >> DEPOT_STACK_ALIGN; stack->handle.extra = 0; INIT_LIST_HEAD(&stack->hash_list); pool_offset += size; return stack;} /* Try to find next free usable entry from the freelist. */static struct stack_record *depot_pop_free(void) __must_hold(&pool_lock){ struct stack_record *stack; lockdep_assert_held(&pool_lock); if (list_empty(&free_stacks)) return NULL; /* * We maintain the invariant that the elements in front are least * recently used, and are therefore more likely to be associated with an * RCU grace period in the past. Consequently it is sufficient to only * check the first entry. */ stack = list_first_entry(&free_stacks, struct stack_record, free_list); if (!poll_state_synchronize_rcu(stack->rcu_state)) return NULL; list_del(&stack->free_list); counters[DEPOT_COUNTER_FREELIST_SIZE]--; return stack;} static inline size_t depot_stack_record_size(struct stack_record *s, unsigned int nr_entries){ const size_t used = flex_array_size(s, entries, nr_entries); const size_t unused = sizeof(s->entries) - used; WARN_ON_ONCE(sizeof(s->entries) < used); return ALIGN(sizeof(struct stack_record) - unused, 1 << DEPOT_STACK_ALIGN);} /* Allocates a new stack in a stack depot pool. */static struct stack_record *depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc) __must_hold(&pool_lock){ struct stack_record *stack = NULL; size_t record_size; lockdep_assert_held(&pool_lock); /* This should already be checked by public API entry points. */ if (WARN_ON_ONCE(!nr_entries)) return NULL; /* Limit number of saved frames to CONFIG_STACKDEPOT_MAX_FRAMES. */ if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; if (flags & STACK_DEPOT_FLAG_GET) { /* * Evictable entries have to allocate the max. size so they may * safely be re-used by differently sized allocations. */ record_size = depot_stack_record_size(stack, CONFIG_STACKDEPOT_MAX_FRAMES); stack = depot_pop_free(); } else { record_size = depot_stack_record_size(stack, nr_entries); } if (!stack) { stack = depot_pop_free_pool(prealloc, record_size); if (!stack) return NULL; } /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); if (flags & STACK_DEPOT_FLAG_GET) { refcount_set(&stack->count, 1); counters[DEPOT_COUNTER_REFD_ALLOCS]++; counters[DEPOT_COUNTER_REFD_INUSE]++; } else { /* Warn on attempts to switch to refcounting this entry. */ refcount_set(&stack->count, REFCOUNT_SATURATED); counters[DEPOT_COUNTER_PERSIST_COUNT]++; counters[DEPOT_COUNTER_PERSIST_BYTES] += record_size; } /* * Let KMSAN know the stored stack record is initialized. This shall * prevent false positive reports if instrumented code accesses it. */ kmsan_unpoison_memory(stack, record_size); return stack;} static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle) __must_not_hold(&pool_lock){ const int pools_num_cached = READ_ONCE(pools_num); union handle_parts parts = { .handle = handle }; void *pool; u32 pool_index = parts.pool_index_plus_1 - 1; size_t offset = parts.offset << DEPOT_STACK_ALIGN;Lines 1–500 of 878. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Stack depot - a stack trace storage that avoids duplication. * * Internally, stack depot maintains a hash table of unique stacktraces. The * stack traces themselves are stored contiguously one after another in a set * of separate page allocations. * * Author: Alexander Potapenko <glider@google.com> * Copyright (C) 2016 Google, Inc. * * Based on the code by Dmitry Chernenkov. */ #define pr_fmt(fmt) "stackdepot: " fmt #include <linux/debugfs.h> #include <linux/gfp.h> #include <linux/jhash.h> #include <linux/kernel.h> #include <linux/kmsan.h> #include <linux/list.h> #include <linux/mm.h> #include <linux/mutex.h> #include <linux/poison.h> #include <linux/printk.h> #include <linux/rculist.h> #include <linux/rcupdate.h> #include <linux/refcount.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/stacktrace.h> #include <linux/stackdepot.h> #include <linux/string.h> #include <linux/types.h> #include <linux/memblock.h> #include <linux/kasan-enabled.h> /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); static bool stack_depot_disabled; static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; /* Use one hash table bucket per 16 KB of memory. */ #define STACK_HASH_TABLE_SCALE 14 /* Limit the number of buckets between 4K and 1M. */ #define STACK_BUCKET_NUMBER_ORDER_MIN 12 #define STACK_BUCKET_NUMBER_ORDER_MAX 20 /* Initial seed for jhash2. */ #define STACK_HASH_SEED 0x9747b28c /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ static unsigned int stack_bucket_number_order; /* Hash mask for indexing the table. */ static unsigned int stack_hash_mask; /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); /* Array of memory regions that store stack records. */ static void **stack_pools __pt_guarded_by(&pool_lock); /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; /* Number of pools in stack_pools. */ static int pools_num; /* Offset to the unused space in the currently used pool. */ static size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ static __guarded_by(&pool_lock) LIST_HEAD(free_stacks); /* Statistics counters for debugfs. */ enum depot_counter_id { DEPOT_COUNTER_REFD_ALLOCS, DEPOT_COUNTER_REFD_FREES, DEPOT_COUNTER_REFD_INUSE, DEPOT_COUNTER_FREELIST_SIZE, DEPOT_COUNTER_PERSIST_COUNT, DEPOT_COUNTER_PERSIST_BYTES, DEPOT_COUNTER_COUNT, }; static long counters[DEPOT_COUNTER_COUNT]; static const char *const counter_names[] = { [DEPOT_COUNTER_REFD_ALLOCS] = "refcounted_allocations", [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); } early_param("stack_depot_disable", disable_stack_depot); static int __init parse_max_pools(char *str) { const long long limit = (1LL << (DEPOT_POOL_INDEX_BITS)) - 1; unsigned int max_pools; int rv; rv = kstrtouint(str, 0, &max_pools); if (rv) return rv; if (max_pools < 1024) { pr_err("stack_depot_max_pools below 1024, using default of %u\n", stack_max_pools); goto out; } if (max_pools > limit) { pr_err("stack_depot_max_pools exceeds %lld, using default of %u\n", limit, stack_max_pools); goto out; } stack_max_pools = max_pools; out: return 0; } early_param("stack_depot_max_pools", parse_max_pools); void __init stack_depot_request_early_init(void) { /* Too late to request early init now. */ WARN_ON(__stack_depot_early_init_passed); __stack_depot_early_init_requested = true; } /* Initialize list_head's within the hash table. */ static void init_stack_table(unsigned long entries) { unsigned long i; for (i = 0; i < entries; i++) INIT_LIST_HEAD(&stack_table[i]); } /* Allocates a hash table via memblock. Can only be used during early boot. */ int __init stack_depot_early_init(void) { unsigned long entries = 0; /* This function must be called only once, from mm_init(). */ if (WARN_ON(__stack_depot_early_init_passed)) return 0; __stack_depot_early_init_passed = true; /* * Print disabled message even if early init has not been requested: * stack_depot_init() will not print one. */ if (stack_depot_disabled) { pr_info("disabled\n"); return 0; } /* * If KASAN is enabled, use the maximum order: KASAN is frequently used * in fuzzing scenarios, which leads to a large number of different * stack traces being stored in stack depot. */ if (kasan_enabled() && !stack_bucket_number_order) stack_bucket_number_order = STACK_BUCKET_NUMBER_ORDER_MAX; /* * Check if early init has been requested after setting * stack_bucket_number_order: stack_depot_init() uses its value. */ if (!__stack_depot_early_init_requested) return 0; /* * If stack_bucket_number_order is not set, leave entries as 0 to rely * on the automatic calculations performed by alloc_large_system_hash(). */ if (stack_bucket_number_order) entries = 1UL << stack_bucket_number_order; pr_info("allocating hash table via alloc_large_system_hash\n"); stack_table = alloc_large_system_hash("stackdepot", sizeof(struct list_head), entries, STACK_HASH_TABLE_SCALE, HASH_EARLY, NULL, &stack_hash_mask, 1UL << STACK_BUCKET_NUMBER_ORDER_MIN, 1UL << STACK_BUCKET_NUMBER_ORDER_MAX); if (!stack_table) { pr_err("hash table allocation failed, disabling\n"); stack_depot_disabled = true; return -ENOMEM; } if (!entries) { /* * Obtain the number of entries that was calculated by * alloc_large_system_hash(). */ entries = stack_hash_mask + 1; } init_stack_table(entries); pr_info("allocating space for %u stack pools via memblock\n", stack_max_pools); stack_pools = memblock_alloc(stack_max_pools * sizeof(void *), PAGE_SIZE); if (!stack_pools) { pr_err("stack pools allocation failed, disabling\n"); memblock_free(stack_table, entries * sizeof(struct list_head)); stack_depot_disabled = true; return -ENOMEM; } return 0; } /* Allocates a hash table via kvcalloc. Can be used after boot. */ int stack_depot_init(void) { static DEFINE_MUTEX(stack_depot_init_mutex); unsigned long entries; int ret = 0; mutex_lock(&stack_depot_init_mutex); if (stack_depot_disabled || stack_table) goto out_unlock; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order * if assigned, and rely on automatic scaling otherwise. */ if (stack_bucket_number_order) { entries = 1UL << stack_bucket_number_order; } else { int scale = STACK_HASH_TABLE_SCALE; entries = nr_free_buffer_pages(); entries = roundup_pow_of_two(entries); if (scale > PAGE_SHIFT) entries >>= (scale - PAGE_SHIFT); else entries <<= (PAGE_SHIFT - scale); } if (entries < 1UL << STACK_BUCKET_NUMBER_ORDER_MIN) entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MIN; if (entries > 1UL << STACK_BUCKET_NUMBER_ORDER_MAX) entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX; pr_info("allocating hash table of %lu entries via kvcalloc\n", entries); stack_table = kvzalloc_objs(struct list_head, entries); if (!stack_table) { pr_err("hash table allocation failed, disabling\n"); stack_depot_disabled = true; ret = -ENOMEM; goto out_unlock; } stack_hash_mask = entries - 1; init_stack_table(entries); pr_info("allocating space for %u stack pools via kvcalloc\n", stack_max_pools); stack_pools = kvcalloc(stack_max_pools, sizeof(void *), GFP_KERNEL); if (!stack_pools) { pr_err("stack pools allocation failed, disabling\n"); kvfree(stack_table); stack_depot_disabled = true; ret = -ENOMEM; } out_unlock: mutex_unlock(&stack_depot_init_mutex); return ret; } EXPORT_SYMBOL_GPL(stack_depot_init); /* * Initializes new stack pool, and updates the list of pools. */ static bool depot_init_pool(void **prealloc) __must_hold(&pool_lock) { lockdep_assert_held(&pool_lock); if (unlikely(pools_num >= stack_max_pools)) { /* Bail out if we reached the pool limit. */ WARN_ON_ONCE(pools_num > stack_max_pools); /* should never happen */ WARN_ON_ONCE(!new_pool); /* to avoid unnecessary pre-allocation */ WARN_ONCE(1, "Stack depot reached limit capacity"); return false; } if (!new_pool && *prealloc) { /* We have preallocated memory, use it. */ WRITE_ONCE(new_pool, *prealloc); *prealloc = NULL; } if (!new_pool) return false; /* new_pool and *prealloc are NULL */ /* Save reference to the pool to be used by depot_fetch_stack(). */ stack_pools[pools_num] = new_pool; /* * Stack depot tries to keep an extra pool allocated even before it runs * out of space in the currently used pool. * * To indicate that a new preallocation is needed new_pool is reset to * NULL; do not reset to NULL if we have reached the maximum number of * pools. */ if (pools_num < stack_max_pools) WRITE_ONCE(new_pool, NULL); else WRITE_ONCE(new_pool, STACK_DEPOT_POISON); /* Pairs with concurrent READ_ONCE() in depot_fetch_stack(). */ WRITE_ONCE(pools_num, pools_num + 1); ASSERT_EXCLUSIVE_WRITER(pools_num); pool_offset = 0; return true; } /* Keeps the preallocated memory to be used for a new stack depot pool. */ static void depot_keep_new_pool(void **prealloc) __must_hold(&pool_lock) { lockdep_assert_held(&pool_lock); /* * If a new pool is already saved or the maximum number of * pools is reached, do not use the preallocated memory. */ if (new_pool) return; WRITE_ONCE(new_pool, *prealloc); *prealloc = NULL; } /* * Try to initialize a new stack record from the current pool, a cached pool, or * the current pre-allocation. */ static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size) __must_hold(&pool_lock) { struct stack_record *stack; void *current_pool; u32 pool_index; lockdep_assert_held(&pool_lock); if (pool_offset + size > DEPOT_POOL_SIZE) { if (!depot_init_pool(prealloc)) return NULL; } if (WARN_ON_ONCE(pools_num < 1)) return NULL; pool_index = pools_num - 1; current_pool = stack_pools[pool_index]; if (WARN_ON_ONCE(!current_pool)) return NULL; stack = current_pool + pool_offset; /* Pre-initialize handle once. */ stack->handle.pool_index_plus_1 = pool_index + 1; stack->handle.offset = pool_offset >> DEPOT_STACK_ALIGN; stack->handle.extra = 0; INIT_LIST_HEAD(&stack->hash_list); pool_offset += size; return stack; } /* Try to find next free usable entry from the freelist. */ static struct stack_record *depot_pop_free(void) __must_hold(&pool_lock) { struct stack_record *stack; lockdep_assert_held(&pool_lock); if (list_empty(&free_stacks)) return NULL; /* * We maintain the invariant that the elements in front are least * recently used, and are therefore more likely to be associated with an * RCU grace period in the past. Consequently it is sufficient to only * check the first entry. */ stack = list_first_entry(&free_stacks, struct stack_record, free_list); if (!poll_state_synchronize_rcu(stack->rcu_state)) return NULL; list_del(&stack->free_list); counters[DEPOT_COUNTER_FREELIST_SIZE]--; return stack; } static inline size_t depot_stack_record_size(struct stack_record *s, unsigned int nr_entries) { const size_t used = flex_array_size(s, entries, nr_entries); const size_t unused = sizeof(s->entries) - used; WARN_ON_ONCE(sizeof(s->entries) < used); return ALIGN(sizeof(struct stack_record) - unused, 1 << DEPOT_STACK_ALIGN); } /* Allocates a new stack in a stack depot pool. */ static struct stack_record * depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc) __must_hold(&pool_lock) { struct stack_record *stack = NULL; size_t record_size; lockdep_assert_held(&pool_lock); /* This should already be checked by public API entry points. */ if (WARN_ON_ONCE(!nr_entries)) return NULL; /* Limit number of saved frames to CONFIG_STACKDEPOT_MAX_FRAMES. */ if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; if (flags & STACK_DEPOT_FLAG_GET) { /* * Evictable entries have to allocate the max. size so they may * safely be re-used by differently sized allocations. */ record_size = depot_stack_record_size(stack, CONFIG_STACKDEPOT_MAX_FRAMES); stack = depot_pop_free(); } else { record_size = depot_stack_record_size(stack, nr_entries); } if (!stack) { stack = depot_pop_free_pool(prealloc, record_size); if (!stack) return NULL; } /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); if (flags & STACK_DEPOT_FLAG_GET) { refcount_set(&stack->count, 1); counters[DEPOT_COUNTER_REFD_ALLOCS]++; counters[DEPOT_COUNTER_REFD_INUSE]++; } else { /* Warn on attempts to switch to refcounting this entry. */ refcount_set(&stack->count, REFCOUNT_SATURATED); counters[DEPOT_COUNTER_PERSIST_COUNT]++; counters[DEPOT_COUNTER_PERSIST_BYTES] += record_size; } /* * Let KMSAN know the stored stack record is initialized. This shall * prevent false positive reports if instrumented code accesses it. */ kmsan_unpoison_memory(stack, record_size); return stack; } static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle) __must_not_hold(&pool_lock) { const int pools_num_cached = READ_ONCE(pools_num); union handle_parts parts = { .handle = handle }; void *pool; u32 pool_index = parts.pool_index_plus_1 - 1; size_t offset = parts.offset << DEPOT_STACK_ALIGN;