History

debugobjects: Track object usage to avoid premature freeing of objects

The freelist is freed at a constant rate independent of the actual usage requirements. That's bad in scenarios where usage comes in bursts. The end of a burst puts the objects on the free list and freeing proceeds even when the next burst which requires objects started again. Keep track of the usage with a exponentially wheighted moving average and take that into account in the worker function which frees objects from the free list. This further reduces the kmem_cache allocation/free rate for a full kernel compile: kmem_cache_alloc() kmem_cache_free() Baseline: 225k 173k Usage: 170k 117k

Thomas Gleixner committed 2 years ago

Commit ff8d5231 parent 13f9ca71 file · +40 −5Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+40−5
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include <linux/hash.h>13#include <linux/hash.h>
14#include <linux/kmemleak.h>14#include <linux/kmemleak.h>
15#include <linux/sched.h>15#include <linux/sched.h>
16#include <linux/sched/loadavg.h>
16#include <linux/sched/task_stack.h>17#include <linux/sched/task_stack.h>
17#include <linux/seq_file.h>18#include <linux/seq_file.h>
18#include <linux/slab.h>19#include <linux/slab.h>
@@ -86,6 +87,7 @@ static struct obj_pool pool_to_free = { · 67 unmodified lines
86 87
87static HLIST_HEAD(pool_boot);88static HLIST_HEAD(pool_boot);
88 89
90static unsigned long avg_usage;
89static bool obj_freeing;91static bool obj_freeing;
90 92
91static int __data_racy debug_objects_maxchain __read_mostly;93static int __data_racy debug_objects_maxchain __read_mostly;
@@ -427,11 +429,31 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 335 unmodified lines
427 return NULL;429 return NULL;
428}430}
429 431
432static void calc_usage(void)
433{
434 static DEFINE_RAW_SPINLOCK(avg_lock);
435 static unsigned long avg_period;
436 unsigned long cur, now = jiffies;
437
438 if (!time_after_eq(now, READ_ONCE(avg_period)))
439 return;
440
441 if (!raw_spin_trylock(&avg_lock))
442 return;
443
444 WRITE_ONCE(avg_period, now + msecs_to_jiffies(10));
445 cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX;
446 WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur));
447 raw_spin_unlock(&avg_lock);
448}
449
430static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,450static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,
431 const struct debug_obj_descr *descr)451 const struct debug_obj_descr *descr)
432{452{
433 struct debug_obj *obj;453 struct debug_obj *obj;
434 454
455 calc_usage();
456
435 if (static_branch_likely(&obj_cache_enabled))457 if (static_branch_likely(&obj_cache_enabled))
436 obj = pcpu_alloc();458 obj = pcpu_alloc();
437 else459 else
@@ -450,14 +472,26 @@ static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, · 12 unmodified lines
450/* workqueue function to free objects. */472/* workqueue function to free objects. */
451static void free_obj_work(struct work_struct *work)473static void free_obj_work(struct work_struct *work)
452{474{
453 bool free = true;475 static unsigned long last_use_avg;
476 unsigned long cur_used, last_used, delta;
477 unsigned int max_free = 0;
454 478
455 WRITE_ONCE(obj_freeing, false);479 WRITE_ONCE(obj_freeing, false);
456 480
481 /* Rate limit freeing based on current use average */
482 cur_used = READ_ONCE(avg_usage);
483 last_used = last_use_avg;
484 last_use_avg = cur_used;
485
457 if (!pool_count(&pool_to_free))486 if (!pool_count(&pool_to_free))
458 return;487 return;
459 488
460 for (unsigned int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) {489 if (cur_used <= last_used) {
490 delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX;
491 max_free = min(delta, ODEBUG_FREE_WORK_MAX);
492 }
493
494 for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) {
461 HLIST_HEAD(tofree);495 HLIST_HEAD(tofree);
462 496
463 /* Acquire and drop the lock for each batch */497 /* Acquire and drop the lock for each batch */
@@ -468,9 +502,10 @@ static void free_obj_work(struct work_struct *work) · 4 unmodified lines
468 /* Refill the global pool if possible */502 /* Refill the global pool if possible */
469 if (pool_move_batch(&pool_global, &pool_to_free)) {503 if (pool_move_batch(&pool_global, &pool_to_free)) {
470 /* Don't free as there seems to be demand */504 /* Don't free as there seems to be demand */
471 free = false;505 max_free = 0;
472 } else if (free) {506 } else if (max_free) {
473 pool_pop_batch(&tofree, &pool_to_free);507 pool_pop_batch(&tofree, &pool_to_free);
508 max_free--;
474 } else {509 } else {
475 return;510 return;
476 }511 }
@@ -1110,7 +1145,7 @@ static int debug_stats_show(struct seq_file *m, void *v) · 633 unmodified lines
1110 for_each_possible_cpu(cpu)1145 for_each_possible_cpu(cpu)
1111 pcp_free += per_cpu(pool_pcpu.cnt, cpu);1146 pcp_free += per_cpu(pool_pcpu.cnt, cpu);
1112 1147
1113 pool_used = data_race(pool_global.stats.cur_used);1148 pool_used = READ_ONCE(pool_global.stats.cur_used);
1114 pcp_free = min(pool_used, pcp_free);1149 pcp_free = min(pool_used, pcp_free);
1115 pool_used -= pcp_free;1150 pool_used -= pcp_free;
1116 1151