debugobjects: Reduce number of pool_lock acquisitions in fill_pool()

In fill_pool(), the pool_lock is acquired and then released once per debug object. If many objects are to be filled, the constant lock and unlock operations are extra overhead. To reduce the overhead, batch them up and do an allocation of 4 objects per lock/unlock sequence.

Waiman Long committed 7 years ago

Commit d26bf501 parent 634d61f1 file · +16 −8Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+16−8

@@ -120,7 +120,7 @@ static const char *obj_states[ODEBUG_STATE_MAX] = { · 119 unmodified lines
120	120	static void fill_pool(void)
121	121	{
122	122	gfp_t gfp = GFP_ATOMIC \| __GFP_NORETRY \| __GFP_NOWARN;
123		− struct debug_obj new, obj;
	123	+ struct debug_obj *obj;
124	124	unsigned long flags;
125	125
126	126	if (likely(obj_pool_free >= debug_objects_pool_min_level))
@@ -136,7 +136,7 @@ static void fill_pool(void) · 9 unmodified lines
136	136	* Recheck with the lock held as the worker thread might have
137	137	* won the race and freed the global free list already.
138	138	*/
139		− if (obj_nr_tofree) {
	139	+ while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
140	140	obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
141	141	hlist_del(&obj->node);
142	142	obj_nr_tofree--;
@@ -150,15 +150,23 @@ static void fill_pool(void) · 7 unmodified lines
150	150	return;
151	151
152	152	while (obj_pool_free < debug_objects_pool_min_level) {
	153	+ struct debug_obj *new[ODEBUG_BATCH_SIZE];
	154	+ int cnt;
153	155
154		− new = kmem_cache_zalloc(obj_cache, gfp);
155		− if (!new)
	156	+ for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
	157	+ new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
	158	+ if (!new[cnt])
	159	+ break;
	160	+ }
	161	+ if (!cnt)
156	162	return;
157	163
158	164	raw_spin_lock_irqsave(&pool_lock, flags);
159		− hlist_add_head(&new->node, &obj_pool);
160		− debug_objects_allocated++;
161		− obj_pool_free++;
	165	+ while (cnt) {
	166	+ hlist_add_head(&new[--cnt]->node, &obj_pool);
	167	+ debug_objects_allocated++;
	168	+ obj_pool_free++;
	169	+ }
162	170	raw_spin_unlock_irqrestore(&pool_lock, flags);
163	171	}
164	172	}
@@ -280,7 +288,7 @@ static void free_obj_work(struct work_struct *work) · 115 unmodified lines
280	288	/*
281	289	* The objs on the pool list might be allocated before the work is
282	290	* run, so recheck if pool list it full or not, if not fill pool
283		− * list from the global free list
	291	+ * list from the global free list.
284	292	*/
285	293	while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
286	294	obj = hlist_entry(obj_to_free.first, typeof(*obj), node);