History

debugobjects: Add percpu free pools

When a multi-threaded workload does a lot of small memory object allocations and deallocations, it may cause the allocation and freeing of many debug objects. This will make the global pool_lock a bottleneck in the performance of the workload. Since interrupts are disabled when acquiring the pool_lock, it may even cause hard lockups to happen. To reduce contention of the global pool_lock, add a percpu debug object free pool that can be used to buffer some of the debug object allocation and freeing requests without acquiring the pool_lock. Each CPU will now have a percpu free pool that can hold up to a maximum of 64 debug objects. Allocation and freeing requests will go to the percpu free pool first. If that fails, the pool_lock will be taken and the global free pool will be used. The presence or absence of obj_cache is used as a marker to see if the percpu cache should be used.

Waiman Long committed 7 years ago

Commit d86998b1 parent fecb0d91 file · +91 −24Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+91−24
@@ -25,6 +25,7 @@ · 24 unmodified lines
25 25
26#define ODEBUG_POOL_SIZE 102426#define ODEBUG_POOL_SIZE 1024
27#define ODEBUG_POOL_MIN_LEVEL 25627#define ODEBUG_POOL_MIN_LEVEL 256
28#define ODEBUG_POOL_PERCPU_SIZE 64
28 29
29#define ODEBUG_CHUNK_SHIFT PAGE_SHIFT30#define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
30#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)31#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
@@ -35,6 +36,17 @@ struct debug_bucket { · 4 unmodified lines
35 raw_spinlock_t lock;36 raw_spinlock_t lock;
36};37};
37 38
39/*
40 * Debug object percpu free list
41 * Access is protected by disabling irq
42 */
43struct debug_percpu_free {
44 struct hlist_head free_objs;
45 int obj_free;
46};
47
48static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
49
38static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];50static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
39 51
40static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;52static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
@@ -44,13 +56,19 @@ static DEFINE_RAW_SPINLOCK(pool_lock); · 3 unmodified lines
44static HLIST_HEAD(obj_pool);56static HLIST_HEAD(obj_pool);
45static HLIST_HEAD(obj_to_free);57static HLIST_HEAD(obj_to_free);
46 58
59/*
60 * Because of the presence of percpu free pools, obj_pool_free will
61 * under-count those in the percpu free pools. Similarly, obj_pool_used
62 * will over-count those in the percpu free pools. Adjustments will be
63 * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
64 * can be off.
65 */
47static int obj_pool_min_free = ODEBUG_POOL_SIZE;66static int obj_pool_min_free = ODEBUG_POOL_SIZE;
48static int obj_pool_free = ODEBUG_POOL_SIZE;67static int obj_pool_free = ODEBUG_POOL_SIZE;
49static int obj_pool_used;68static int obj_pool_used;
50static int obj_pool_max_used;69static int obj_pool_max_used;
51/* The number of objs on the global free list */70/* The number of objs on the global free list */
52static int obj_nr_tofree;71static int obj_nr_tofree;
53static struct kmem_cache *obj_cache;
54 72
55static int debug_objects_maxchain __read_mostly;73static int debug_objects_maxchain __read_mostly;
56static int __maybe_unused debug_objects_maxchecked __read_mostly;74static int __maybe_unused debug_objects_maxchecked __read_mostly;
@@ -63,6 +81,7 @@ static int debug_objects_pool_size __read_mostly · 6 unmodified lines
63static int debug_objects_pool_min_level __read_mostly81static int debug_objects_pool_min_level __read_mostly
64 = ODEBUG_POOL_MIN_LEVEL;82 = ODEBUG_POOL_MIN_LEVEL;
65static struct debug_obj_descr *descr_test __read_mostly;83static struct debug_obj_descr *descr_test __read_mostly;
84static struct kmem_cache *obj_cache __read_mostly;
66 85
67/*86/*
68 * Track numbers of kmem_cache_alloc()/free() calls done.87 * Track numbers of kmem_cache_alloc()/free() calls done.
@@ -162,6 +181,21 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 93 unmodified lines
162 return NULL;181 return NULL;
163}182}
164 183
184/*
185 * Allocate a new object from the hlist
186 */
187static struct debug_obj *__alloc_object(struct hlist_head *list)
188{
189 struct debug_obj *obj = NULL;
190
191 if (list->first) {
192 obj = hlist_entry(list->first, typeof(*obj), node);
193 hlist_del(&obj->node);
194 }
195
196 return obj;
197}
198
165/*199/*
166 * Allocate a new object. If the pool is empty, switch off the debugger.200 * Allocate a new object. If the pool is empty, switch off the debugger.
167 * Must be called with interrupts disabled.201 * Must be called with interrupts disabled.
@@ -169,20 +203,21 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 1 unmodified line
169static struct debug_obj *203static struct debug_obj *
170alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)204alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
171{205{
172 struct debug_obj *obj = NULL;206 struct debug_percpu_free *percpu_pool;
173 207 struct debug_obj *obj;
174 raw_spin_lock(&pool_lock);
175 if (obj_pool.first) {
176 obj = hlist_entry(obj_pool.first, typeof(*obj), node);
177
178 obj->object = addr;
179 obj->descr = descr;
180 obj->state = ODEBUG_STATE_NONE;
181 obj->astate = 0;
182 hlist_del(&obj->node);
183 208
184 hlist_add_head(&obj->node, &b->list);209 if (likely(obj_cache)) {
210 percpu_pool = this_cpu_ptr(&percpu_obj_pool);
211 obj = __alloc_object(&percpu_pool->free_objs);
212 if (obj) {
213 percpu_pool->obj_free--;
214 goto init_obj;
215 }
216 }
185 217
218 raw_spin_lock(&pool_lock);
219 obj = __alloc_object(&obj_pool);
220 if (obj) {
186 obj_pool_used++;221 obj_pool_used++;
187 if (obj_pool_used > obj_pool_max_used)222 if (obj_pool_used > obj_pool_max_used)
188 obj_pool_max_used = obj_pool_used;223 obj_pool_max_used = obj_pool_used;
@@ -193,6 +228,14 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) · 4 unmodified lines
193 }228 }
194 raw_spin_unlock(&pool_lock);229 raw_spin_unlock(&pool_lock);
195 230
231init_obj:
232 if (obj) {
233 obj->object = addr;
234 obj->descr = descr;
235 obj->state = ODEBUG_STATE_NONE;
236 obj->astate = 0;
237 hlist_add_head(&obj->node, &b->list);
238 }
196 return obj;239 return obj;
197}240}
198 241
@@ -247,8 +290,21 @@ static bool __free_object(struct debug_obj *obj) · 48 unmodified lines
247{290{
248 unsigned long flags;291 unsigned long flags;
249 bool work;292 bool work;
293 struct debug_percpu_free *percpu_pool;
250 294
251 raw_spin_lock_irqsave(&pool_lock, flags);295 local_irq_save(flags);
296 /*
297 * Try to free it into the percpu pool first.
298 */
299 percpu_pool = this_cpu_ptr(&percpu_obj_pool);
300 if (obj_cache && percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
301 hlist_add_head(&obj->node, &percpu_pool->free_objs);
302 percpu_pool->obj_free++;
303 local_irq_restore(flags);
304 return false;
305 }
306
307 raw_spin_lock(&pool_lock);
252 work = (obj_pool_free > debug_objects_pool_size) && obj_cache;308 work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
253 obj_pool_used--;309 obj_pool_used--;
254 310
@@ -259,7 +315,8 @@ static bool __free_object(struct debug_obj *obj) · 4 unmodified lines
259 obj_pool_free++;315 obj_pool_free++;
260 hlist_add_head(&obj->node, &obj_pool);316 hlist_add_head(&obj->node, &obj_pool);
261 }317 }
262 raw_spin_unlock_irqrestore(&pool_lock, flags);318 raw_spin_unlock(&pool_lock);
319 local_irq_restore(flags);
263 return work;320 return work;
264}321}
265 322
@@ -822,13 +879,19 @@ void debug_check_no_obj_freed(const void *address, unsigned long size) · 556 unmodified lines
822 879
823static int debug_stats_show(struct seq_file *m, void *v)880static int debug_stats_show(struct seq_file *m, void *v)
824{881{
882 int cpu, obj_percpu_free = 0;
883
884 for_each_possible_cpu(cpu)
885 obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
886
825 seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);887 seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
826 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);888 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
827 seq_printf(m, "warnings :%d\n", debug_objects_warnings);889 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
828 seq_printf(m, "fixups :%d\n", debug_objects_fixups);890 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
829 seq_printf(m, "pool_free :%d\n", obj_pool_free);891 seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free);
892 seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
830 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);893 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831 seq_printf(m, "pool_used :%d\n", obj_pool_used);894 seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
832 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);895 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833 seq_printf(m, "on_free_list :%d\n", obj_nr_tofree);896 seq_printf(m, "on_free_list :%d\n", obj_nr_tofree);
834 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);897 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
@@ -1165,9 +1228,20 @@ free: · 330 unmodified lines
1165 */1228 */
1166void __init debug_objects_mem_init(void)1229void __init debug_objects_mem_init(void)
1167{1230{
1231 int cpu;
1232
1168 if (!debug_objects_enabled)1233 if (!debug_objects_enabled)
1169 return;1234 return;
1170 1235
1236 /*
1237 * Initialize the percpu object pools
1238 *
1239 * Initialization is not strictly necessary, but was done for
1240 * completeness.
1241 */
1242 for_each_possible_cpu(cpu)
1243 INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1244
1171 obj_cache = kmem_cache_create("debug_objects_cache",1245 obj_cache = kmem_cache_create("debug_objects_cache",
1172 sizeof (struct debug_obj), 0,1246 sizeof (struct debug_obj), 0,
1173 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,1247 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
@@ -1179,11 +1253,4 @@ void __init debug_objects_mem_init(void) · 5 unmodified lines
1179 pr_warn("out of memory.\n");1253 pr_warn("out of memory.\n");
1180 } else1254 } else
1181 debug_objects_selftest();1255 debug_objects_selftest();
1182
1183 /*
1184 * Increase the thresholds for allocating and freeing objects
1185 * according to the number of possible CPUs available in the system.
1186 */
1187 debug_objects_pool_size += num_possible_cpus() * 32;
1188 debug_objects_pool_min_level += num_possible_cpus() * 4;
1189}1256}