History

debugobjects: Move pools into a datastructure

The contention on the global pool lock can be reduced by strict batch processing where batches of objects are moved from one list head to another instead of moving them object by object. This also reduces the cache footprint because it avoids the list walk and dirties at maximum three cache lines instead of potentially up to eighteen. To prepare for that, move the hlist head and related counters into a struct. No functional change.

Thomas Gleixner committed 2 years ago

Commit e18328f1 parent d8c6cd31 file · +78 −62Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+78−62
@@ -52,6 +52,11 @@ struct debug_percpu_free { · 51 unmodified lines
52 int obj_free;52 int obj_free;
53};53};
54 54
55struct obj_pool {
56 struct hlist_head objects;
57 unsigned int cnt;
58} ____cacheline_aligned;
59
55static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);60static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
56 61
57static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];62static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
@@ -60,8 +65,8 @@ static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; · 2 unmodified lines
60 65
61static DEFINE_RAW_SPINLOCK(pool_lock);66static DEFINE_RAW_SPINLOCK(pool_lock);
62 67
63static HLIST_HEAD(obj_pool);68static struct obj_pool pool_global;
64static HLIST_HEAD(obj_to_free);69static struct obj_pool pool_to_free;
65 70
66/*71/*
67 * Because of the presence of percpu free pools, obj_pool_free will72 * Because of the presence of percpu free pools, obj_pool_free will
@@ -71,12 +76,9 @@ static HLIST_HEAD(obj_to_free); · 3 unmodified lines
71 * can be off.76 * can be off.
72 */77 */
73static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;78static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;
74static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE;
75static int obj_pool_used;79static int obj_pool_used;
76static int __data_racy obj_pool_max_used;80static int __data_racy obj_pool_max_used;
77static bool obj_freeing;81static bool obj_freeing;
78/* The number of objs on the global free list */
79static int obj_nr_tofree;
80 82
81static int __data_racy debug_objects_maxchain __read_mostly;83static int __data_racy debug_objects_maxchain __read_mostly;
82static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;84static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;
@@ -124,6 +126,21 @@ static const char *obj_states[ODEBUG_STATE_MAX] = { · 41 unmodified lines
124 [ODEBUG_STATE_NOTAVAILABLE] = "not available",126 [ODEBUG_STATE_NOTAVAILABLE] = "not available",
125};127};
126 128
129static __always_inline unsigned int pool_count(struct obj_pool *pool)
130{
131 return READ_ONCE(pool->cnt);
132}
133
134static inline bool pool_global_should_refill(void)
135{
136 return READ_ONCE(pool_global.cnt) < debug_objects_pool_min_level;
137}
138
139static inline bool pool_global_must_refill(void)
140{
141 return READ_ONCE(pool_global.cnt) < (debug_objects_pool_min_level / 2);
142}
143
127static void free_object_list(struct hlist_head *head)144static void free_object_list(struct hlist_head *head)
128{145{
129 struct hlist_node *tmp;146 struct hlist_node *tmp;
@@ -146,11 +163,8 @@ static void fill_pool_from_freelist(void) · 16 unmodified lines
146 /*163 /*
147 * Reuse objs from the global obj_to_free list; they will be164 * Reuse objs from the global obj_to_free list; they will be
148 * reinitialized when allocating.165 * reinitialized when allocating.
149 *
150 * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with
151 * the WRITE_ONCE() in pool_lock critical sections.
152 */166 */
153 if (!READ_ONCE(obj_nr_tofree))167 if (!pool_count(&pool_to_free))
154 return;168 return;
155 169
156 /*170 /*
@@ -171,12 +185,12 @@ static void fill_pool_from_freelist(void) · 14 unmodified lines
171 * Recheck with the lock held as the worker thread might have185 * Recheck with the lock held as the worker thread might have
172 * won the race and freed the global free list already.186 * won the race and freed the global free list already.
173 */187 */
174 while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) {188 while (pool_to_free.cnt && (pool_global.cnt < debug_objects_pool_min_level)) {
175 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);189 obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
176 hlist_del(&obj->node);190 hlist_del(&obj->node);
177 WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);191 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
178 hlist_add_head(&obj->node, &obj_pool);192 hlist_add_head(&obj->node, &pool_global.objects);
179 WRITE_ONCE(obj_pool_free, obj_pool_free + 1);193 WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
180 }194 }
181 clear_bit(0, &state);195 clear_bit(0, &state);
182}196}
@@ -190,12 +204,11 @@ static void fill_pool(void) · 7 unmodified lines
190 * - One other CPU is already allocating204 * - One other CPU is already allocating
191 * - the global pool has not reached the critical level yet205 * - the global pool has not reached the critical level yet
192 */206 */
193 if (READ_ONCE(obj_pool_free) > (debug_objects_pool_min_level / 2) &&207 if (!pool_global_must_refill() && atomic_read(&cpus_allocating))
194 atomic_read(&cpus_allocating))
195 return;208 return;
196 209
197 atomic_inc(&cpus_allocating);210 atomic_inc(&cpus_allocating);
198 while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {211 while (pool_global_should_refill()) {
199 struct debug_obj *new, *last = NULL;212 struct debug_obj *new, *last = NULL;
200 HLIST_HEAD(head);213 HLIST_HEAD(head);
201 int cnt;214 int cnt;
@@ -212,9 +225,9 @@ static void fill_pool(void) · 10 unmodified lines
212 break;225 break;
213 226
214 guard(raw_spinlock_irqsave)(&pool_lock);227 guard(raw_spinlock_irqsave)(&pool_lock);
215 hlist_splice_init(&head, &last->node, &obj_pool);228 hlist_splice_init(&head, &last->node, &pool_global.objects);
216 debug_objects_allocated += cnt;229 debug_objects_allocated += cnt;
217 WRITE_ONCE(obj_pool_free, obj_pool_free + cnt);230 WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);
218 }231 }
219 atomic_dec(&cpus_allocating);232 atomic_dec(&cpus_allocating);
220}233}
@@ -268,10 +281,10 @@ alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *d · 47 unmodified lines
268 }281 }
269 282
270 raw_spin_lock(&pool_lock);283 raw_spin_lock(&pool_lock);
271 obj = __alloc_object(&obj_pool);284 obj = __alloc_object(&pool_global.objects);
272 if (obj) {285 if (obj) {
273 obj_pool_used++;286 obj_pool_used++;
274 WRITE_ONCE(obj_pool_free, obj_pool_free - 1);287 WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
275 288
276 /*289 /*
277 * Looking ahead, allocate one batch of debug objects and290 * Looking ahead, allocate one batch of debug objects and
@@ -283,22 +296,21 @@ alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *d · 5 unmodified lines
283 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {296 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
284 struct debug_obj *obj2;297 struct debug_obj *obj2;
285 298
286 obj2 = __alloc_object(&obj_pool);299 obj2 = __alloc_object(&pool_global.objects);
287 if (!obj2)300 if (!obj2)
288 break;301 break;
289 hlist_add_head(&obj2->node,302 hlist_add_head(&obj2->node, &percpu_pool->free_objs);
290 &percpu_pool->free_objs);
291 percpu_pool->obj_free++;303 percpu_pool->obj_free++;
292 obj_pool_used++;304 obj_pool_used++;
293 WRITE_ONCE(obj_pool_free, obj_pool_free - 1);305 WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
294 }306 }
295 }307 }
296 308
297 if (obj_pool_used > obj_pool_max_used)309 if (obj_pool_used > obj_pool_max_used)
298 obj_pool_max_used = obj_pool_used;310 obj_pool_max_used = obj_pool_used;
299 311
300 if (obj_pool_free < obj_pool_min_free)312 if (pool_global.cnt < obj_pool_min_free)
301 obj_pool_min_free = obj_pool_free;313 obj_pool_min_free = pool_global.cnt;
302 }314 }
303 raw_spin_unlock(&pool_lock);315 raw_spin_unlock(&pool_lock);
304 316
@@ -329,7 +341,7 @@ static void free_obj_work(struct work_struct *work) · 24 unmodified lines
329 if (!raw_spin_trylock_irqsave(&pool_lock, flags))341 if (!raw_spin_trylock_irqsave(&pool_lock, flags))
330 return;342 return;
331 343
332 if (obj_pool_free >= debug_objects_pool_size)344 if (pool_global.cnt >= debug_objects_pool_size)
333 goto free_objs;345 goto free_objs;
334 346
335 /*347 /*
@@ -339,12 +351,12 @@ static void free_obj_work(struct work_struct *work) · 3 unmodified lines
339 * may be gearing up to use more and more objects, don't free any351 * may be gearing up to use more and more objects, don't free any
340 * of them until the next round.352 * of them until the next round.
341 */353 */
342 while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {354 while (pool_to_free.cnt && pool_global.cnt < debug_objects_pool_size) {
343 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);355 obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
344 hlist_del(&obj->node);356 hlist_del(&obj->node);
345 hlist_add_head(&obj->node, &obj_pool);357 hlist_add_head(&obj->node, &pool_global.objects);
346 WRITE_ONCE(obj_pool_free, obj_pool_free + 1);358 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
347 WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);359 WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
348 }360 }
349 raw_spin_unlock_irqrestore(&pool_lock, flags);361 raw_spin_unlock_irqrestore(&pool_lock, flags);
350 return;362 return;
@@ -355,9 +367,9 @@ free_objs: · 4 unmodified lines
355 * list. Move remaining free objs to a temporary list to free the367 * list. Move remaining free objs to a temporary list to free the
356 * memory outside the pool_lock held region.368 * memory outside the pool_lock held region.
357 */369 */
358 if (obj_nr_tofree) {370 if (pool_to_free.cnt) {
359 hlist_move_list(&obj_to_free, &tofree);371 hlist_move_list(&pool_to_free.objects, &tofree);
360 WRITE_ONCE(obj_nr_tofree, 0);372 WRITE_ONCE(pool_to_free.cnt, 0);
361 }373 }
362 raw_spin_unlock_irqrestore(&pool_lock, flags);374 raw_spin_unlock_irqrestore(&pool_lock, flags);
363 375
@@ -400,45 +412,45 @@ static void __free_object(struct debug_obj *obj) · 36 unmodified lines
400 412
401free_to_obj_pool:413free_to_obj_pool:
402 raw_spin_lock(&pool_lock);414 raw_spin_lock(&pool_lock);
403 work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&415 work = (pool_global.cnt > debug_objects_pool_size) && obj_cache &&
404 (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);416 (pool_to_free.cnt < ODEBUG_FREE_WORK_MAX);
405 obj_pool_used--;417 obj_pool_used--;
406 418
407 if (work) {419 if (work) {
408 WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);420 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
409 hlist_add_head(&obj->node, &obj_to_free);421 hlist_add_head(&obj->node, &pool_to_free.objects);
410 if (lookahead_count) {422 if (lookahead_count) {
411 WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);423 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + lookahead_count);
412 obj_pool_used -= lookahead_count;424 obj_pool_used -= lookahead_count;
413 while (lookahead_count) {425 while (lookahead_count) {
414 hlist_add_head(&objs[--lookahead_count]->node,426 hlist_add_head(&objs[--lookahead_count]->node,
415 &obj_to_free);427 &pool_to_free.objects);
416 }428 }
417 }429 }
418 430
419 if ((obj_pool_free > debug_objects_pool_size) &&431 if ((pool_global.cnt > debug_objects_pool_size) &&
420 (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {432 (pool_to_free.cnt < ODEBUG_FREE_WORK_MAX)) {
421 int i;433 int i;
422 434
423 /*435 /*
424 * Free one more batch of objects from obj_pool.436 * Free one more batch of objects from obj_pool.
425 */437 */
426 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {438 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
427 obj = __alloc_object(&obj_pool);439 obj = __alloc_object(&pool_global.objects);
428 hlist_add_head(&obj->node, &obj_to_free);440 hlist_add_head(&obj->node, &pool_to_free.objects);
429 WRITE_ONCE(obj_pool_free, obj_pool_free - 1);441 WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
430 WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);442 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
431 }443 }
432 }444 }
433 } else {445 } else {
434 WRITE_ONCE(obj_pool_free, obj_pool_free + 1);446 WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
435 hlist_add_head(&obj->node, &obj_pool);447 hlist_add_head(&obj->node, &pool_global.objects);
436 if (lookahead_count) {448 if (lookahead_count) {
437 WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);449 WRITE_ONCE(pool_global.cnt, pool_global.cnt + lookahead_count);
438 obj_pool_used -= lookahead_count;450 obj_pool_used -= lookahead_count;
439 while (lookahead_count) {451 while (lookahead_count) {
440 hlist_add_head(&objs[--lookahead_count]->node,452 hlist_add_head(&objs[--lookahead_count]->node,
441 &obj_pool);453 &pool_global.objects);
442 }454 }
443 }455 }
444 }456 }
@@ -453,7 +465,7 @@ free_to_obj_pool: · 8 unmodified lines
453static void free_object(struct debug_obj *obj)465static void free_object(struct debug_obj *obj)
454{466{
455 __free_object(obj);467 __free_object(obj);
456 if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {468 if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
457 WRITE_ONCE(obj_freeing, true);469 WRITE_ONCE(obj_freeing, true);
458 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);470 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
459 }471 }
@@ -622,13 +634,13 @@ static void debug_objects_fill_pool(void) · 162 unmodified lines
622 if (unlikely(!obj_cache))634 if (unlikely(!obj_cache))
623 return;635 return;
624 636
625 if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))637 if (likely(!pool_global_should_refill()))
626 return;638 return;
627 639
628 /* Try reusing objects from obj_to_free_list */640 /* Try reusing objects from obj_to_free_list */
629 fill_pool_from_freelist();641 fill_pool_from_freelist();
630 642
631 if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))643 if (likely(!pool_global_should_refill()))
632 return;644 return;
633 645
634 /*646 /*
@@ -1040,7 +1052,7 @@ repeat: · 405 unmodified lines
1040 debug_objects_maxchecked = objs_checked;1052 debug_objects_maxchecked = objs_checked;
1041 1053
1042 /* Schedule work to actually kmem_cache_free() objects */1054 /* Schedule work to actually kmem_cache_free() objects */
1043 if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {1055 if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
1044 WRITE_ONCE(obj_freeing, true);1056 WRITE_ONCE(obj_freeing, true);
1045 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);1057 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
1046 }1058 }
@@ -1066,12 +1078,12 @@ static int debug_stats_show(struct seq_file *m, void *v) · 19 unmodified lines
1066 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);1078 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
1067 seq_printf(m, "warnings :%d\n", debug_objects_warnings);1079 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
1068 seq_printf(m, "fixups :%d\n", debug_objects_fixups);1080 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
1069 seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);1081 seq_printf(m, "pool_free :%d\n", pool_count(&pool_global) + obj_percpu_free);
1070 seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);1082 seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
1071 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);1083 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
1072 seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);1084 seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
1073 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);1085 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1074 seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));1086 seq_printf(m, "on_free_list :%d\n", pool_count(&pool_to_free));
1075 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);1087 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
1076 seq_printf(m, "objs_freed :%d\n", debug_objects_freed);1088 seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
1077 return 0;1089 return 0;
@@ -1330,7 +1342,9 @@ void __init debug_objects_early_init(void) · 252 unmodified lines
1330 raw_spin_lock_init(&obj_hash[i].lock);1342 raw_spin_lock_init(&obj_hash[i].lock);
1331 1343
1332 for (i = 0; i < ODEBUG_POOL_SIZE; i++)1344 for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1333 hlist_add_head(&obj_static_pool[i].node, &obj_pool);1345 hlist_add_head(&obj_static_pool[i].node, &pool_global.objects);
1346
1347 pool_global.cnt = ODEBUG_POOL_SIZE;
1334}1348}
1335 1349
1336/*1350/*
@@ -1354,21 +1368,23 @@ static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache · 17 unmodified lines
1354 hlist_add_head(&obj->node, &objects);1368 hlist_add_head(&obj->node, &objects);
1355 }1369 }
1356 1370
1357 debug_objects_allocated += i;1371 debug_objects_allocated = ODEBUG_POOL_SIZE;
1372 pool_global.cnt = ODEBUG_POOL_SIZE;
1358 1373
1359 /*1374 /*
1360 * Replace the statically allocated objects list with the allocated1375 * Replace the statically allocated objects list with the allocated
1361 * objects list.1376 * objects list.
1362 */1377 */
1363 hlist_move_list(&objects, &obj_pool);1378 hlist_move_list(&objects, &pool_global.objects);
1364 1379
1365 /* Replace the active object references */1380 /* Replace the active object references */
1366 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {1381 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1367 hlist_move_list(&db->list, &objects);1382 hlist_move_list(&db->list, &objects);
1368 1383
1369 hlist_for_each_entry(obj, &objects, node) {1384 hlist_for_each_entry(obj, &objects, node) {
1370 new = hlist_entry(obj_pool.first, typeof(*obj), node);1385 new = hlist_entry(pool_global.objects.first, typeof(*obj), node);
1371 hlist_del(&new->node);1386 hlist_del(&new->node);
1387 pool_global.cnt--;
1372 /* copy object data */1388 /* copy object data */
1373 *new = *obj;1389 *new = *obj;
1374 hlist_add_head(&new->node, &db->list);1390 hlist_add_head(&new->node, &db->list);