last_use_avg = cur_used; 	if (!pool_count(&pool_to_free))		return; 	if (cur_used <= last_used) {		delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX;		max_free = min(delta, ODEBUG_FREE_WORK_MAX);	} 	for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) {		HLIST_HEAD(tofree); 		/* Acquire and drop the lock for each batch */		scoped_guard(raw_spinlock_irqsave, &pool_lock) {			if (!pool_to_free.cnt)				return; 			/* Refill the global pool if possible */			if (pool_move_batch(&pool_global, &pool_to_free)) {				/* Don't free as there seems to be demand */				max_free = 0;			} else if (max_free) {				pool_pop_batch(&tofree, &pool_to_free);				max_free--;			} else {				return;			}		}		free_object_list(&tofree);	}} static void __free_object(struct debug_obj *obj){	guard(irqsave)();	if (static_branch_likely(&obj_cache_enabled))		pcpu_free(obj);	else		hlist_add_head(&obj->node, &pool_boot);} /* * Put the object back into the pool and schedule work to free objects * if necessary. */static void free_object(struct debug_obj *obj){	__free_object(obj);	if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {		WRITE_ONCE(obj_freeing, true);		schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);	}} static void put_objects(struct hlist_head *list){	struct hlist_node *tmp;	struct debug_obj *obj; 	/*	 * Using free_object() puts the objects into reuse or schedules	 * them for freeing and it get's all the accounting correct.	 */	hlist_for_each_entry_safe(obj, tmp, list, node) {		hlist_del(&obj->node);		free_object(obj);	}} #ifdef CONFIG_HOTPLUG_CPUstatic int object_cpu_offline(unsigned int cpu){	/* Remote access is safe as the CPU is dead already */	struct obj_pool *pcp = per_cpu_ptr(&pool_pcpu, cpu); 	put_objects(&pcp->objects);	pcp->cnt = 0;	return 0;}#endif /* Out of memory. Free all objects from hash */static void debug_objects_oom(void){	struct debug_bucket *db = obj_hash;	HLIST_HEAD(freelist); 	pr_warn("Out of memory. ODEBUG disabled\n"); 	for (int i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {		scoped_guard(raw_spinlock_irqsave, &db->lock)			hlist_move_list(&db->list, &freelist); 		put_objects(&freelist);	}} /* * We use the pfn of the address for the hash. That way we can check * for freed objects simply by checking the affected bucket. */static struct debug_bucket *get_bucket(unsigned long addr){	unsigned long hash; 	hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);	return &obj_hash[hash];} static void debug_print_object(struct debug_obj *obj, char *msg){	const struct debug_obj_descr *descr = obj->descr;	static int limit; 	/*	 * Don't report if lookup_object_or_alloc() by the current thread	 * failed because lookup_object_or_alloc()/debug_objects_oom() by a	 * concurrent thread turned off debug_objects_enabled and cleared	 * the hash buckets.	 */	if (!debug_objects_enabled)		return; 	if (limit < 5 && descr != descr_test) {		void *hint = descr->debug_hint ?			descr->debug_hint(obj->object) : NULL;		limit++;		WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "				 "object: %p object type: %s hint: %pS\n",			msg, obj_states[obj->state], obj->astate,			obj->object, descr->name, hint);	}	debug_objects_warnings++;} /* * Try to repair the damage, so we have a better chance to get useful * debug output. */static booldebug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),		   void * addr, enum debug_obj_state state){	if (fixup && fixup(addr, state)) {		debug_objects_fixups++;		return true;	}	return false;} static void debug_object_is_on_stack(void *addr, int onstack){	int is_on_stack;	static int limit; 	if (limit > 4)		return; 	is_on_stack = object_is_on_stack(addr);	if (is_on_stack == onstack)		return; 	limit++;	if (is_on_stack)		pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,			 task_stack_page(current));	else		pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,			 task_stack_page(current)); 	WARN_ON(1);} static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket *b,						const struct debug_obj_descr *descr,						bool onstack, bool alloc_ifstatic){	struct debug_obj *obj = lookup_object(addr, b);	enum debug_obj_state state = ODEBUG_STATE_NONE; 	if (likely(obj))		return obj; 	/*	 * debug_object_init() unconditionally allocates untracked	 * objects. It does not matter whether it is a static object or	 * not.	 *	 * debug_object_assert_init() and debug_object_activate() allow	 * allocation only if the descriptor callback confirms that the	 * object is static and considered initialized. For non-static	 * objects the allocation needs to be done from the fixup callback.	 */	if (unlikely(alloc_ifstatic)) {		if (!descr->is_static_object || !descr->is_static_object(addr))			return ERR_PTR(-ENOENT);		/* Statically allocated objects are considered initialized */		state = ODEBUG_STATE_INIT;	} 	obj = alloc_object(addr, b, descr);	if (likely(obj)) {		obj->state = state;		debug_object_is_on_stack(addr, onstack);		return obj;	} 	/* Out of memory. Do the cleanup outside of the locked region */	debug_objects_enabled = false;	return NULL;} static inline bool debug_objects_is_pi_blocked_on(void){#ifdef CONFIG_RT_MUTEXES	return current->pi_blocked_on != NULL;#else	return false;#endif} static inline bool can_fill_pool(void){	/*	 * On !RT enabled kernels there are no restrictions and spinlock_t and	 * raw_spinlock_t are the same types.	 */	if (!IS_ENABLED(CONFIG_PREEMPT_RT))		return true; 	/*	 * On RT enabled kernels, the task must not be blocked on a lock as	 * that could corrupt the PI state when blocking on a lock in the	 * allocation path.	 */	if (debug_objects_is_pi_blocked_on())		return false; 	/*	 * On RT enabled kernels the pool refill should happen in preemptible	 * context.	 */	if (preemptible())		return true; 	/*	 * Though during system boot before scheduling is set up, preemption is	 * disabled and the pool can get exhausted. Before scheduling is active	 * a task cannot be blocked on a sleeping lock, but it might hold a lock	 * and if interrupted then hard interrupt context might run into a lock	 * inversion. So exclude hard interrupt context from allocations before	 * scheduling is active.	 */	return system_state < SYSTEM_SCHEDULING && !in_hardirq();} static void debug_objects_fill_pool(void){	if (!static_branch_likely(&obj_cache_enabled))		return; 	if (likely(!pool_should_refill(&pool_global)))		return; 	/* Try reusing objects from obj_to_free_list */	fill_pool_from_freelist(); 	if (likely(!pool_should_refill(&pool_global)))		return; 	if (can_fill_pool()) {		/*		 * Annotate away the spinlock_t inside raw_spinlock_t warning		 * by temporarily raising the wait-type to LD_WAIT_CONFIG, matching		 * the preemptible() condition in can_fill_pool().		 */		static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_CONFIG);		lock_map_acquire_try(&fill_pool_map);		fill_pool();		lock_map_release(&fill_pool_map);	}} static void__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack){	struct debug_obj *obj, o;	struct debug_bucket *db;	unsigned long flags; 	debug_objects_fill_pool(); 	db = get_bucket((unsigned long) addr); 	raw_spin_lock_irqsave(&db->lock, flags); 	obj = lookup_object_or_alloc(addr, db, descr, onstack, false);	if (unlikely(!obj)) {		raw_spin_unlock_irqrestore(&db->lock, flags);		debug_objects_oom();		return;	} 	switch (obj->state) {	case ODEBUG_STATE_NONE:	case ODEBUG_STATE_INIT:	case ODEBUG_STATE_INACTIVE:		obj->state = ODEBUG_STATE_INIT;		raw_spin_unlock_irqrestore(&db->lock, flags);		return;	default:		break;	} 	o = *obj;	raw_spin_unlock_irqrestore(&db->lock, flags);	debug_print_object(&o, "init"); 	if (o.state == ODEBUG_STATE_ACTIVE)		debug_object_fixup(descr->fixup_init, addr, o.state);} /** * debug_object_init - debug checks when an object is initialized * @addr:	address of the object * @descr:	pointer to an object specific debug description structure */void debug_object_init(void *addr, const struct debug_obj_descr *descr){	if (!debug_objects_enabled)		return; 	__debug_object_init(addr, descr, 0);}EXPORT_SYMBOL_GPL(debug_object_init); /** * debug_object_init_on_stack - debug checks when an object on stack is *				initialized * @addr:	address of the object * @descr:	pointer to an object specific debug description structure */void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr){	if (!debug_objects_enabled)		return; 	__debug_object_init(addr, descr, 1);}EXPORT_SYMBOL_GPL(debug_object_init_on_stack); /** * debug_object_activate - debug checks when an object is activated * @addr:	address of the object * @descr:	pointer to an object specific debug description structure * Returns 0 for success, -EINVAL for check failed. */int debug_object_activate(void *addr, const struct debug_obj_descr *descr){	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };	struct debug_bucket *db;	struct debug_obj *obj;	unsigned long flags; 	if (!debug_objects_enabled)		return 0; 	debug_objects_fill_pool(); 	db = get_bucket((unsigned long) addr); 	raw_spin_lock_irqsave(&db->lock, flags); 	obj = lookup_object_or_alloc(addr, db, descr, false, true);	if (unlikely(!obj)) {		raw_spin_unlock_irqrestore(&db->lock, flags);		debug_objects_oom();		return 0;	} else if (likely(!IS_ERR(obj))) {		switch (obj->state) {		case ODEBUG_STATE_ACTIVE:		case ODEBUG_STATE_DESTROYED:			o = *obj;			break;		case ODEBUG_STATE_INIT:		case ODEBUG_STATE_INACTIVE:			obj->state = ODEBUG_STATE_ACTIVE;			fallthrough;		default:			raw_spin_unlock_irqrestore(&db->lock, flags);			return 0;		}	} 	raw_spin_unlock_irqrestore(&db->lock, flags); 	/*	 * lookup_object_or_alloc() might have raced with a concurrent	 * allocation failure which disabled debug objects.	 */	if (!debug_objects_enabled)		return 0; 	debug_print_object(&o, "activate"); 	switch (o.state) {	case ODEBUG_STATE_ACTIVE:	case ODEBUG_STATE_NOTAVAILABLE:		if (debug_object_fixup(descr->fixup_activate, addr, o.state))			return 0;		fallthrough;	default:		return -EINVAL;	}}EXPORT_SYMBOL_GPL(debug_object_activate); /** * debug_object_deactivate - debug checks when an object is deactivated * @addr:	address of the object * @descr:	pointer to an object specific debug description structure */void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr){	struct debug_obj o = { .object = addr, .state = ODEBUG_STATE_NOTAVAILABLE, .descr = descr };	struct debug_bucket *db;	struct debug_obj *obj;	unsigned long flags; 	if (!debug_objects_enabled)		return; 	db = get_bucket((unsigned long) addr); 	raw_spin_lock_irqsave(&db->lock, flags); 	obj = lookup_object(addr, db);	if (obj) {		switch (obj->state) {		case ODEBUG_STATE_DESTROYED:			break;		case ODEBUG_STATE_INIT:		case ODEBUG_STATE_INACTIVE:		case ODEBUG_STATE_ACTIVE:			if (obj->astate)				break;			obj->state = ODEBUG_STATE_INACTIVE;			fallthrough;		default:			raw_spin_unlock_irqrestore(&db->lock, flags);			return;		}		o = *obj;	} 	raw_spin_unlock_irqrestore(&db->lock, flags);	debug_print_object(&o, "deactivate");}EXPORT_SYMBOL_GPL(debug_object_deactivate); /** * debug_object_destroy - debug checks when an object is destroyed * @addr:	address of the object * @descr:	pointer to an object specific debug description structure */void debug_object_destroy(void *addr, const struct debug_obj_descr *descr){	struct debug_obj *obj, o;	struct debug_bucket *db;	unsigned long flags; 	if (!debug_objects_enabled)		return; 	db = get_bucket((unsigned long) addr); 	raw_spin_lock_irqsave(&db->lock, flags); 	obj = lookup_object(addr, db);	if (!obj) {		raw_spin_unlock_irqrestore(&db->lock, flags);		return;	} 	switch (obj->state) {	case ODEBUG_STATE_ACTIVE:	case ODEBUG_STATE_DESTROYED:		break;	case ODEBUG_STATE_NONE:	case ODEBUG_STATE_INIT:	case ODEBUG_STATE_INACTIVE:		obj->state = ODEBUG_STATE_DESTROYED;		fallthrough;	default:		raw_spin_unlock_irqrestore(&db->lock, flags);		return;	} 	o = *obj;

Lines 501–1000 of 1585. Previous lines Load more