History

debugobjects: Prepare kmem_cache allocations for batching

Allocate a batch and then push it into the pool. Utilize the debug_obj::last_node pointer for keeping track of the batch boundary.

Thomas Gleixner committed 2 years ago

Commit aebbfe01 parent 74fe1ad1 file · +49 −31Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+49−31
@@ -164,6 +164,22 @@ static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src) · 163 unmodified lines
164 return true;164 return true;
165}165}
166 166
167static bool pool_push_batch(struct obj_pool *dst, struct hlist_head *head)
168{
169 struct hlist_node *last;
170 struct debug_obj *obj;
171
172 if (dst->cnt >= dst->max_cnt)
173 return false;
174
175 obj = hlist_entry(head->first, typeof(*obj), node);
176 last = obj->batch_last;
177
178 hlist_splice_init(head, last, &dst->objects);
179 WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE);
180 return true;
181}
182
167static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src)183static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src)
168{184{
169 if (!src->cnt)185 if (!src->cnt)
@@ -288,6 +304,28 @@ static void fill_pool_from_freelist(void) · 118 unmodified lines
288 clear_bit(0, &state);304 clear_bit(0, &state);
289}305}
290 306
307static bool kmem_alloc_batch(struct hlist_head *head, struct kmem_cache *cache, gfp_t gfp)
308{
309 struct hlist_node *last = NULL;
310 struct debug_obj *obj;
311
312 for (int cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
313 obj = kmem_cache_zalloc(cache, gfp);
314 if (!obj) {
315 free_object_list(head);
316 return false;
317 }
318 debug_objects_allocated++;
319
320 if (!last)
321 last = &obj->node;
322 obj->batch_last = last;
323
324 hlist_add_head(&obj->node, head);
325 }
326 return true;
327}
328
291static void fill_pool(void)329static void fill_pool(void)
292{330{
293 static atomic_t cpus_allocating;331 static atomic_t cpus_allocating;
@@ -302,25 +340,14 @@ static void fill_pool(void) · 8 unmodified lines
302 340
303 atomic_inc(&cpus_allocating);341 atomic_inc(&cpus_allocating);
304 while (pool_should_refill(&pool_global)) {342 while (pool_should_refill(&pool_global)) {
305 struct debug_obj *new, *last = NULL;
306 HLIST_HEAD(head);343 HLIST_HEAD(head);
307 int cnt;
308 344
309 for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {345 if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN))
310 new = kmem_cache_zalloc(obj_cache, __GFP_HIGH | __GFP_NOWARN);
311 if (!new)
312 break;
313 hlist_add_head(&new->node, &head);
314 if (!last)
315 last = new;
316 }
317 if (!cnt)
318 break;346 break;
319 347
320 guard(raw_spinlock_irqsave)(&pool_lock);348 guard(raw_spinlock_irqsave)(&pool_lock);
321 hlist_splice_init(&head, &last->node, &pool_global.objects);349 if (!pool_push_batch(&pool_global, &head))
322 debug_objects_allocated += cnt;350 pool_push_batch(&pool_to_free, &head);
323 WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);
324 }351 }
325 atomic_dec(&cpus_allocating);352 atomic_dec(&cpus_allocating);
326}353}
@@ -1302,26 +1329,18 @@ void __init debug_objects_early_init(void) · 975 unmodified lines
1302static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache)1329static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache)
1303{1330{
1304 struct debug_bucket *db = obj_hash;1331 struct debug_bucket *db = obj_hash;
1305 struct debug_obj *obj, *new;
1306 struct hlist_node *tmp;1332 struct hlist_node *tmp;
1333 struct debug_obj *obj;
1307 HLIST_HEAD(objects);1334 HLIST_HEAD(objects);
1308 int i;1335 int i;
1309 1336
1310 for (i = 0; i < ODEBUG_POOL_SIZE; i++) {1337 for (i = 0; i < ODEBUG_POOL_SIZE; i += ODEBUG_BATCH_SIZE) {
1311 obj = kmem_cache_zalloc(cache, GFP_KERNEL);1338 if (!kmem_alloc_batch(&objects, cache, GFP_KERNEL))
1312 if (!obj)
1313 goto free;1339 goto free;
1314 hlist_add_head(&obj->node, &objects);1340 pool_push_batch(&pool_global, &objects);
1315 }1341 }
1316 1342
1317 debug_objects_allocated = ODEBUG_POOL_SIZE;1343 /* Disconnect the boot pool. */
1318 pool_global.cnt = ODEBUG_POOL_SIZE;
1319
1320 /*
1321 * Move the allocated objects to the global pool and disconnect the
1322 * boot pool.
1323 */
1324 hlist_move_list(&objects, &pool_global.objects);
1325 pool_boot.first = NULL;1344 pool_boot.first = NULL;
1326 1345
1327 /* Replace the active object references */1346 /* Replace the active object references */
@@ -1329,9 +1348,8 @@ static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache · 1 unmodified line
1329 hlist_move_list(&db->list, &objects);1348 hlist_move_list(&db->list, &objects);
1330 1349
1331 hlist_for_each_entry(obj, &objects, node) {1350 hlist_for_each_entry(obj, &objects, node) {
1332 new = hlist_entry(pool_global.objects.first, typeof(*obj), node);1351 struct debug_obj *new = pcpu_alloc();
1333 hlist_del(&new->node);1352
1334 pool_global.cnt--;
1335 /* copy object data */1353 /* copy object data */
1336 *new = *obj;1354 *new = *obj;
1337 hlist_add_head(&new->node, &db->list);1355 hlist_add_head(&new->node, &db->list);
@@ -1340,7 +1358,7 @@ static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache · 2 unmodified lines
1340 return true;1358 return true;
1341free:1359free:
1342 /* Can't use free_object_list() as the cache is not populated yet */1360 /* Can't use free_object_list() as the cache is not populated yet */
1343 hlist_for_each_entry_safe(obj, tmp, &objects, node) {1361 hlist_for_each_entry_safe(obj, tmp, &pool_global.objects, node) {
1344 hlist_del(&obj->node);1362 hlist_del(&obj->node);
1345 kmem_cache_free(cache, obj);1363 kmem_cache_free(cache, obj);
1346 }1364 }