History

debugobjects: Move pools into a datastructure

The contention on the global pool lock can be reduced by strict batch processing where batches of objects are moved from one list head to another instead of moving them object by object. This also reduces the cache footprint because it avoids the list walk and dirties at maximum three cache lines instead of potentially up to eighteen. To prepare for that, move the hlist head and related counters into a struct. No functional change.

Thomas Gleixner committed 2 years ago

Commit e18328f1 parent d8c6cd31 file · +78 −62Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+78−62
@@ -52,6 +52,11 @@ struct debug_percpu_free { · 51 unmodified lines
5252 int obj_free;
5353 };
5454
55+struct obj_pool {
56+ struct hlist_head objects;
57+ unsigned int cnt;
58+} ____cacheline_aligned;
59+
5560 static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
5661
5762 static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
@@ -60,8 +65,8 @@ static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata; · 2 unmodified lines
6065
6166 static DEFINE_RAW_SPINLOCK(pool_lock);
6267
63−static HLIST_HEAD(obj_pool);
64−static HLIST_HEAD(obj_to_free);
68+static struct obj_pool pool_global;
69+static struct obj_pool pool_to_free;
6570
6671 /*
6772 * Because of the presence of percpu free pools, obj_pool_free will
@@ -71,12 +76,9 @@ static HLIST_HEAD(obj_to_free); · 3 unmodified lines
7176 * can be off.
7277 */
7378 static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;
74−static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE;
7579 static int obj_pool_used;
7680 static int __data_racy obj_pool_max_used;
7781 static bool obj_freeing;
78−/* The number of objs on the global free list */
79−static int obj_nr_tofree;
8082
8183 static int __data_racy debug_objects_maxchain __read_mostly;
8284 static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;
@@ -124,6 +126,21 @@ static const char *obj_states[ODEBUG_STATE_MAX] = { · 41 unmodified lines
124126 [ODEBUG_STATE_NOTAVAILABLE] = "not available",
125127 };
126128
129+static __always_inline unsigned int pool_count(struct obj_pool *pool)
130+{
131+ return READ_ONCE(pool->cnt);
132+}
133+
134+static inline bool pool_global_should_refill(void)
135+{
136+ return READ_ONCE(pool_global.cnt) < debug_objects_pool_min_level;
137+}
138+
139+static inline bool pool_global_must_refill(void)
140+{
141+ return READ_ONCE(pool_global.cnt) < (debug_objects_pool_min_level / 2);
142+}
143+
127144 static void free_object_list(struct hlist_head *head)
128145 {
129146 struct hlist_node *tmp;
@@ -146,11 +163,8 @@ static void fill_pool_from_freelist(void) · 16 unmodified lines
146163 /*
147164 * Reuse objs from the global obj_to_free list; they will be
148165 * reinitialized when allocating.
149− *
150− * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with
151− * the WRITE_ONCE() in pool_lock critical sections.
152166 */
153− if (!READ_ONCE(obj_nr_tofree))
167+ if (!pool_count(&pool_to_free))
154168 return;
155169
156170 /*
@@ -171,12 +185,12 @@ static void fill_pool_from_freelist(void) · 14 unmodified lines
171185 * Recheck with the lock held as the worker thread might have
172186 * won the race and freed the global free list already.
173187 */
174− while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) {
175− obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
188+ while (pool_to_free.cnt && (pool_global.cnt < debug_objects_pool_min_level)) {
189+ obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
176190 hlist_del(&obj->node);
177− WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
178− hlist_add_head(&obj->node, &obj_pool);
179− WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
191+ WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
192+ hlist_add_head(&obj->node, &pool_global.objects);
193+ WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
180194 }
181195 clear_bit(0, &state);
182196 }
@@ -190,12 +204,11 @@ static void fill_pool(void) · 7 unmodified lines
190204 * - One other CPU is already allocating
191205 * - the global pool has not reached the critical level yet
192206 */
193− if (READ_ONCE(obj_pool_free) > (debug_objects_pool_min_level / 2) &&
194− atomic_read(&cpus_allocating))
207+ if (!pool_global_must_refill() && atomic_read(&cpus_allocating))
195208 return;
196209
197210 atomic_inc(&cpus_allocating);
198− while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
211+ while (pool_global_should_refill()) {
199212 struct debug_obj *new, *last = NULL;
200213 HLIST_HEAD(head);
201214 int cnt;
@@ -212,9 +225,9 @@ static void fill_pool(void) · 10 unmodified lines
212225 break;
213226
214227 guard(raw_spinlock_irqsave)(&pool_lock);
215− hlist_splice_init(&head, &last->node, &obj_pool);
228+ hlist_splice_init(&head, &last->node, &pool_global.objects);
216229 debug_objects_allocated += cnt;
217− WRITE_ONCE(obj_pool_free, obj_pool_free + cnt);
230+ WRITE_ONCE(pool_global.cnt, pool_global.cnt + cnt);
218231 }
219232 atomic_dec(&cpus_allocating);
220233 }
@@ -268,10 +281,10 @@ alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *d · 47 unmodified lines
268281 }
269282
270283 raw_spin_lock(&pool_lock);
271− obj = __alloc_object(&obj_pool);
284+ obj = __alloc_object(&pool_global.objects);
272285 if (obj) {
273286 obj_pool_used++;
274− WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
287+ WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
275288
276289 /*
277290 * Looking ahead, allocate one batch of debug objects and
@@ -283,22 +296,21 @@ alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *d · 5 unmodified lines
283296 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
284297 struct debug_obj *obj2;
285298
286− obj2 = __alloc_object(&obj_pool);
299+ obj2 = __alloc_object(&pool_global.objects);
287300 if (!obj2)
288301 break;
289− hlist_add_head(&obj2->node,
290− &percpu_pool->free_objs);
302+ hlist_add_head(&obj2->node, &percpu_pool->free_objs);
291303 percpu_pool->obj_free++;
292304 obj_pool_used++;
293− WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
305+ WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
294306 }
295307 }
296308
297309 if (obj_pool_used > obj_pool_max_used)
298310 obj_pool_max_used = obj_pool_used;
299311
300− if (obj_pool_free < obj_pool_min_free)
301− obj_pool_min_free = obj_pool_free;
312+ if (pool_global.cnt < obj_pool_min_free)
313+ obj_pool_min_free = pool_global.cnt;
302314 }
303315 raw_spin_unlock(&pool_lock);
304316
@@ -329,7 +341,7 @@ static void free_obj_work(struct work_struct *work) · 24 unmodified lines
329341 if (!raw_spin_trylock_irqsave(&pool_lock, flags))
330342 return;
331343
332− if (obj_pool_free >= debug_objects_pool_size)
344+ if (pool_global.cnt >= debug_objects_pool_size)
333345 goto free_objs;
334346
335347 /*
@@ -339,12 +351,12 @@ static void free_obj_work(struct work_struct *work) · 3 unmodified lines
339351 * may be gearing up to use more and more objects, don't free any
340352 * of them until the next round.
341353 */
342− while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
343− obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
354+ while (pool_to_free.cnt && pool_global.cnt < debug_objects_pool_size) {
355+ obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
344356 hlist_del(&obj->node);
345− hlist_add_head(&obj->node, &obj_pool);
346− WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
347− WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
357+ hlist_add_head(&obj->node, &pool_global.objects);
358+ WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
359+ WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
348360 }
349361 raw_spin_unlock_irqrestore(&pool_lock, flags);
350362 return;
@@ -355,9 +367,9 @@ free_objs: · 4 unmodified lines
355367 * list. Move remaining free objs to a temporary list to free the
356368 * memory outside the pool_lock held region.
357369 */
358− if (obj_nr_tofree) {
359− hlist_move_list(&obj_to_free, &tofree);
360− WRITE_ONCE(obj_nr_tofree, 0);
370+ if (pool_to_free.cnt) {
371+ hlist_move_list(&pool_to_free.objects, &tofree);
372+ WRITE_ONCE(pool_to_free.cnt, 0);
361373 }
362374 raw_spin_unlock_irqrestore(&pool_lock, flags);
363375
@@ -400,45 +412,45 @@ static void __free_object(struct debug_obj *obj) · 36 unmodified lines
400412
401413 free_to_obj_pool:
402414 raw_spin_lock(&pool_lock);
403− work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
404− (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
415+ work = (pool_global.cnt > debug_objects_pool_size) && obj_cache &&
416+ (pool_to_free.cnt < ODEBUG_FREE_WORK_MAX);
405417 obj_pool_used--;
406418
407419 if (work) {
408− WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
409− hlist_add_head(&obj->node, &obj_to_free);
420+ WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
421+ hlist_add_head(&obj->node, &pool_to_free.objects);
410422 if (lookahead_count) {
411− WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
423+ WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + lookahead_count);
412424 obj_pool_used -= lookahead_count;
413425 while (lookahead_count) {
414426 hlist_add_head(&objs[--lookahead_count]->node,
415− &obj_to_free);
427+ &pool_to_free.objects);
416428 }
417429 }
418430
419− if ((obj_pool_free > debug_objects_pool_size) &&
420− (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
431+ if ((pool_global.cnt > debug_objects_pool_size) &&
432+ (pool_to_free.cnt < ODEBUG_FREE_WORK_MAX)) {
421433 int i;
422434
423435 /*
424436 * Free one more batch of objects from obj_pool.
425437 */
426438 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
427− obj = __alloc_object(&obj_pool);
428− hlist_add_head(&obj->node, &obj_to_free);
429− WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
430− WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
439+ obj = __alloc_object(&pool_global.objects);
440+ hlist_add_head(&obj->node, &pool_to_free.objects);
441+ WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
442+ WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt + 1);
431443 }
432444 }
433445 } else {
434− WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
435− hlist_add_head(&obj->node, &obj_pool);
446+ WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
447+ hlist_add_head(&obj->node, &pool_global.objects);
436448 if (lookahead_count) {
437− WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
449+ WRITE_ONCE(pool_global.cnt, pool_global.cnt + lookahead_count);
438450 obj_pool_used -= lookahead_count;
439451 while (lookahead_count) {
440452 hlist_add_head(&objs[--lookahead_count]->node,
441− &obj_pool);
453+ &pool_global.objects);
442454 }
443455 }
444456 }
@@ -453,7 +465,7 @@ free_to_obj_pool: · 8 unmodified lines
453465 static void free_object(struct debug_obj *obj)
454466 {
455467 __free_object(obj);
456− if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
468+ if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
457469 WRITE_ONCE(obj_freeing, true);
458470 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
459471 }
@@ -622,13 +634,13 @@ static void debug_objects_fill_pool(void) · 162 unmodified lines
622634 if (unlikely(!obj_cache))
623635 return;
624636
625− if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
637+ if (likely(!pool_global_should_refill()))
626638 return;
627639
628640 /* Try reusing objects from obj_to_free_list */
629641 fill_pool_from_freelist();
630642
631− if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
643+ if (likely(!pool_global_should_refill()))
632644 return;
633645
634646 /*
@@ -1040,7 +1052,7 @@ repeat: · 405 unmodified lines
10401052 debug_objects_maxchecked = objs_checked;
10411053
10421054 /* Schedule work to actually kmem_cache_free() objects */
1043− if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
1055+ if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
10441056 WRITE_ONCE(obj_freeing, true);
10451057 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
10461058 }
@@ -1066,12 +1078,12 @@ static int debug_stats_show(struct seq_file *m, void *v) · 19 unmodified lines
10661078 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
10671079 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
10681080 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
1069− seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
1081+ seq_printf(m, "pool_free :%d\n", pool_count(&pool_global) + obj_percpu_free);
10701082 seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
10711083 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
10721084 seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
10731085 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1074− seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));
1086+ seq_printf(m, "on_free_list :%d\n", pool_count(&pool_to_free));
10751087 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
10761088 seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
10771089 return 0;
@@ -1330,7 +1342,9 @@ void __init debug_objects_early_init(void) · 252 unmodified lines
13301342 raw_spin_lock_init(&obj_hash[i].lock);
13311343
13321344 for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1333− hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1345+ hlist_add_head(&obj_static_pool[i].node, &pool_global.objects);
1346+
1347+ pool_global.cnt = ODEBUG_POOL_SIZE;
13341348 }
13351349
13361350 /*
@@ -1354,21 +1368,23 @@ static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache · 17 unmodified lines
13541368 hlist_add_head(&obj->node, &objects);
13551369 }
13561370
1357− debug_objects_allocated += i;
1371+ debug_objects_allocated = ODEBUG_POOL_SIZE;
1372+ pool_global.cnt = ODEBUG_POOL_SIZE;
13581373
13591374 /*
13601375 * Replace the statically allocated objects list with the allocated
13611376 * objects list.
13621377 */
1363− hlist_move_list(&objects, &obj_pool);
1378+ hlist_move_list(&objects, &pool_global.objects);
13641379
13651380 /* Replace the active object references */
13661381 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
13671382 hlist_move_list(&db->list, &objects);
13681383
13691384 hlist_for_each_entry(obj, &objects, node) {
1370− new = hlist_entry(obj_pool.first, typeof(*obj), node);
1385+ new = hlist_entry(pool_global.objects.first, typeof(*obj), node);
13711386 hlist_del(&new->node);
1387+ pool_global.cnt--;
13721388 /* copy object data */
13731389 *new = *obj;
13741390 hlist_add_head(&new->node, &db->list);