The freelist is freed at a constant rate independent of the actual usage requirements. That's bad in scenarios where usage comes in bursts. The end of a burst puts the objects on the free list and freeing proceeds even when the next burst which requires objects started again. Keep track of the usage with a exponentially wheighted moving average and take that into account in the worker function which frees objects from the free list. This further reduces the kmem_cache allocation/free rate for a full kernel compile: kmem_cache_alloc() kmem_cache_free() Baseline: 225k 173k Usage: 170k 117k
Showing the changes to lib/debugobjects.c (1 of 1 file). Show all
Unified · Split
| @@ -13,6 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | #include <linux/hash.h> |
| 14 | 14 | #include <linux/kmemleak.h> |
| 15 | 15 | #include <linux/sched.h> |
| 16 | +#include <linux/sched/loadavg.h> | |
| 16 | 17 | #include <linux/sched/task_stack.h> |
| 17 | 18 | #include <linux/seq_file.h> |
| 18 | 19 | #include <linux/slab.h> |
| @@ -86,6 +87,7 @@ static struct obj_pool pool_to_free = { · 67 unmodified lines | ||
| 86 | 87 | |
| 87 | 88 | static HLIST_HEAD(pool_boot); |
| 88 | 89 | |
| 90 | +static unsigned long avg_usage; | |
| 89 | 91 | static bool obj_freeing; |
| 90 | 92 | |
| 91 | 93 | static int __data_racy debug_objects_maxchain __read_mostly; |
| @@ -427,11 +429,31 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 335 unmodified lines | ||
| 427 | 429 | return NULL; |
| 428 | 430 | } |
| 429 | 431 | |
| 432 | +static void calc_usage(void) | |
| 433 | +{ | |
| 434 | + static DEFINE_RAW_SPINLOCK(avg_lock); | |
| 435 | + static unsigned long avg_period; | |
| 436 | + unsigned long cur, now = jiffies; | |
| 437 | + | |
| 438 | + if (!time_after_eq(now, READ_ONCE(avg_period))) | |
| 439 | + return; | |
| 440 | + | |
| 441 | + if (!raw_spin_trylock(&avg_lock)) | |
| 442 | + return; | |
| 443 | + | |
| 444 | + WRITE_ONCE(avg_period, now + msecs_to_jiffies(10)); | |
| 445 | + cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX; | |
| 446 | + WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur)); | |
| 447 | + raw_spin_unlock(&avg_lock); | |
| 448 | +} | |
| 449 | + | |
| 430 | 450 | static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, |
| 431 | 451 | const struct debug_obj_descr *descr) |
| 432 | 452 | { |
| 433 | 453 | struct debug_obj *obj; |
| 434 | 454 | |
| 455 | + calc_usage(); | |
| 456 | + | |
| 435 | 457 | if (static_branch_likely(&obj_cache_enabled)) |
| 436 | 458 | obj = pcpu_alloc(); |
| 437 | 459 | else |
| @@ -450,14 +472,26 @@ static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b, · 12 unmodified lines | ||
| 450 | 472 | /* workqueue function to free objects. */ |
| 451 | 473 | static void free_obj_work(struct work_struct *work) |
| 452 | 474 | { |
| 453 | − bool free = true; | |
| 475 | + static unsigned long last_use_avg; | |
| 476 | + unsigned long cur_used, last_used, delta; | |
| 477 | + unsigned int max_free = 0; | |
| 454 | 478 | |
| 455 | 479 | WRITE_ONCE(obj_freeing, false); |
| 456 | 480 | |
| 481 | + /* Rate limit freeing based on current use average */ | |
| 482 | + cur_used = READ_ONCE(avg_usage); | |
| 483 | + last_used = last_use_avg; | |
| 484 | + last_use_avg = cur_used; | |
| 485 | + | |
| 457 | 486 | if (!pool_count(&pool_to_free)) |
| 458 | 487 | return; |
| 459 | 488 | |
| 460 | − for (unsigned int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { | |
| 489 | + if (cur_used <= last_used) { | |
| 490 | + delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX; | |
| 491 | + max_free = min(delta, ODEBUG_FREE_WORK_MAX); | |
| 492 | + } | |
| 493 | + | |
| 494 | + for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) { | |
| 461 | 495 | HLIST_HEAD(tofree); |
| 462 | 496 | |
| 463 | 497 | /* Acquire and drop the lock for each batch */ |
| @@ -468,9 +502,10 @@ static void free_obj_work(struct work_struct *work) · 4 unmodified lines | ||
| 468 | 502 | /* Refill the global pool if possible */ |
| 469 | 503 | if (pool_move_batch(&pool_global, &pool_to_free)) { |
| 470 | 504 | /* Don't free as there seems to be demand */ |
| 471 | − free = false; | |
| 472 | − } else if (free) { | |
| 505 | + max_free = 0; | |
| 506 | + } else if (max_free) { | |
| 473 | 507 | pool_pop_batch(&tofree, &pool_to_free); |
| 508 | + max_free--; | |
| 474 | 509 | } else { |
| 475 | 510 | return; |
| 476 | 511 | } |
| @@ -1110,7 +1145,7 @@ static int debug_stats_show(struct seq_file *m, void *v) · 633 unmodified lines | ||
| 1110 | 1145 | for_each_possible_cpu(cpu) |
| 1111 | 1146 | pcp_free += per_cpu(pool_pcpu.cnt, cpu); |
| 1112 | 1147 | |
| 1113 | − pool_used = data_race(pool_global.stats.cur_used); | |
| 1148 | + pool_used = READ_ONCE(pool_global.stats.cur_used); | |
| 1114 | 1149 | pcp_free = min(pool_used, pcp_free); |
| 1115 | 1150 | pool_used -= pcp_free; |
| 1116 | 1151 | |