History

debugobjects: Add percpu free pools

When a multi-threaded workload does a lot of small memory object allocations and deallocations, it may cause the allocation and freeing of many debug objects. This will make the global pool_lock a bottleneck in the performance of the workload. Since interrupts are disabled when acquiring the pool_lock, it may even cause hard lockups to happen. To reduce contention of the global pool_lock, add a percpu debug object free pool that can be used to buffer some of the debug object allocation and freeing requests without acquiring the pool_lock. Each CPU will now have a percpu free pool that can hold up to a maximum of 64 debug objects. Allocation and freeing requests will go to the percpu free pool first. If that fails, the pool_lock will be taken and the global free pool will be used. The presence or absence of obj_cache is used as a marker to see if the percpu cache should be used.

Waiman Long committed 7 years ago

Commit d86998b1 parent fecb0d91 file · +91 −24Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+91−24
@@ -25,6 +25,7 @@ · 24 unmodified lines
2525
2626 #define ODEBUG_POOL_SIZE 1024
2727 #define ODEBUG_POOL_MIN_LEVEL 256
28+#define ODEBUG_POOL_PERCPU_SIZE 64
2829
2930 #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
3031 #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
@@ -35,6 +36,17 @@ struct debug_bucket { · 4 unmodified lines
3536 raw_spinlock_t lock;
3637 };
3738
39+/*
40+ * Debug object percpu free list
41+ * Access is protected by disabling irq
42+ */
43+struct debug_percpu_free {
44+ struct hlist_head free_objs;
45+ int obj_free;
46+};
47+
48+static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
49+
3850 static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
3951
4052 static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
@@ -44,13 +56,19 @@ static DEFINE_RAW_SPINLOCK(pool_lock); · 3 unmodified lines
4456 static HLIST_HEAD(obj_pool);
4557 static HLIST_HEAD(obj_to_free);
4658
59+/*
60+ * Because of the presence of percpu free pools, obj_pool_free will
61+ * under-count those in the percpu free pools. Similarly, obj_pool_used
62+ * will over-count those in the percpu free pools. Adjustments will be
63+ * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
64+ * can be off.
65+ */
4766 static int obj_pool_min_free = ODEBUG_POOL_SIZE;
4867 static int obj_pool_free = ODEBUG_POOL_SIZE;
4968 static int obj_pool_used;
5069 static int obj_pool_max_used;
5170 /* The number of objs on the global free list */
5271 static int obj_nr_tofree;
53−static struct kmem_cache *obj_cache;
5472
5573 static int debug_objects_maxchain __read_mostly;
5674 static int __maybe_unused debug_objects_maxchecked __read_mostly;
@@ -63,6 +81,7 @@ static int debug_objects_pool_size __read_mostly · 6 unmodified lines
6381 static int debug_objects_pool_min_level __read_mostly
6482 = ODEBUG_POOL_MIN_LEVEL;
6583 static struct debug_obj_descr *descr_test __read_mostly;
84+static struct kmem_cache *obj_cache __read_mostly;
6685
6786 /*
6887 * Track numbers of kmem_cache_alloc()/free() calls done.
@@ -162,6 +181,21 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 93 unmodified lines
162181 return NULL;
163182 }
164183
184+/*
185+ * Allocate a new object from the hlist
186+ */
187+static struct debug_obj *__alloc_object(struct hlist_head *list)
188+{
189+ struct debug_obj *obj = NULL;
190+
191+ if (list->first) {
192+ obj = hlist_entry(list->first, typeof(*obj), node);
193+ hlist_del(&obj->node);
194+ }
195+
196+ return obj;
197+}
198+
165199 /*
166200 * Allocate a new object. If the pool is empty, switch off the debugger.
167201 * Must be called with interrupts disabled.
@@ -169,20 +203,21 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 1 unmodified line
169203 static struct debug_obj *
170204 alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
171205 {
172− struct debug_obj *obj = NULL;
173−
174− raw_spin_lock(&pool_lock);
175− if (obj_pool.first) {
176− obj = hlist_entry(obj_pool.first, typeof(*obj), node);
177−
178− obj->object = addr;
179− obj->descr = descr;
180− obj->state = ODEBUG_STATE_NONE;
181− obj->astate = 0;
182− hlist_del(&obj->node);
206+ struct debug_percpu_free *percpu_pool;
207+ struct debug_obj *obj;
183208
184− hlist_add_head(&obj->node, &b->list);
209+ if (likely(obj_cache)) {
210+ percpu_pool = this_cpu_ptr(&percpu_obj_pool);
211+ obj = __alloc_object(&percpu_pool->free_objs);
212+ if (obj) {
213+ percpu_pool->obj_free--;
214+ goto init_obj;
215+ }
216+ }
185217
218+ raw_spin_lock(&pool_lock);
219+ obj = __alloc_object(&obj_pool);
220+ if (obj) {
186221 obj_pool_used++;
187222 if (obj_pool_used > obj_pool_max_used)
188223 obj_pool_max_used = obj_pool_used;
@@ -193,6 +228,14 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) · 4 unmodified lines
193228 }
194229 raw_spin_unlock(&pool_lock);
195230
231+init_obj:
232+ if (obj) {
233+ obj->object = addr;
234+ obj->descr = descr;
235+ obj->state = ODEBUG_STATE_NONE;
236+ obj->astate = 0;
237+ hlist_add_head(&obj->node, &b->list);
238+ }
196239 return obj;
197240 }
198241
@@ -247,8 +290,21 @@ static bool __free_object(struct debug_obj *obj) · 48 unmodified lines
247290 {
248291 unsigned long flags;
249292 bool work;
293+ struct debug_percpu_free *percpu_pool;
250294
251− raw_spin_lock_irqsave(&pool_lock, flags);
295+ local_irq_save(flags);
296+ /*
297+ * Try to free it into the percpu pool first.
298+ */
299+ percpu_pool = this_cpu_ptr(&percpu_obj_pool);
300+ if (obj_cache && percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
301+ hlist_add_head(&obj->node, &percpu_pool->free_objs);
302+ percpu_pool->obj_free++;
303+ local_irq_restore(flags);
304+ return false;
305+ }
306+
307+ raw_spin_lock(&pool_lock);
252308 work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
253309 obj_pool_used--;
254310
@@ -259,7 +315,8 @@ static bool __free_object(struct debug_obj *obj) · 4 unmodified lines
259315 obj_pool_free++;
260316 hlist_add_head(&obj->node, &obj_pool);
261317 }
262− raw_spin_unlock_irqrestore(&pool_lock, flags);
318+ raw_spin_unlock(&pool_lock);
319+ local_irq_restore(flags);
263320 return work;
264321 }
265322
@@ -822,13 +879,19 @@ void debug_check_no_obj_freed(const void *address, unsigned long size) · 556 unmodified lines
822879
823880 static int debug_stats_show(struct seq_file *m, void *v)
824881 {
882+ int cpu, obj_percpu_free = 0;
883+
884+ for_each_possible_cpu(cpu)
885+ obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
886+
825887 seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
826888 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
827889 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
828890 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
829− seq_printf(m, "pool_free :%d\n", obj_pool_free);
891+ seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free);
892+ seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
830893 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831− seq_printf(m, "pool_used :%d\n", obj_pool_used);
894+ seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
832895 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833896 seq_printf(m, "on_free_list :%d\n", obj_nr_tofree);
834897 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
@@ -1165,9 +1228,20 @@ free: · 330 unmodified lines
11651228 */
11661229 void __init debug_objects_mem_init(void)
11671230 {
1231+ int cpu;
1232+
11681233 if (!debug_objects_enabled)
11691234 return;
11701235
1236+ /*
1237+ * Initialize the percpu object pools
1238+ *
1239+ * Initialization is not strictly necessary, but was done for
1240+ * completeness.
1241+ */
1242+ for_each_possible_cpu(cpu)
1243+ INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1244+
11711245 obj_cache = kmem_cache_create("debug_objects_cache",
11721246 sizeof (struct debug_obj), 0,
11731247 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
@@ -1179,11 +1253,4 @@ void __init debug_objects_mem_init(void) · 5 unmodified lines
11791253 pr_warn("out of memory.\n");
11801254 } else
11811255 debug_objects_selftest();
1182−
1183− /*
1184− * Increase the thresholds for allocating and freeing objects
1185− * according to the number of possible CPUs available in the system.
1186− */
1187− debug_objects_pool_size += num_possible_cpus() * 32;
1188− debug_objects_pool_min_level += num_possible_cpus() * 4;
11891256 }