The newly added global free list allows to avoid lengthy pool_list iterations in free_obj_work() by putting objects either into the pool list when the fill level of the pool is below the maximum or by putting them on the global free list immediately. As the pool is now guaranteed to never exceed the maximum fill level this allows to remove the batch removal from pool list in free_obj_work(). Split free_object() into two parts, so the actual queueing function can be reused without invoking schedule_work() on every invocation. [ tglx: Remove the batch removal from pool list and massage changelog ]
Showing the changes to lib/debugobjects.c (1 of 1 file). Show all
Unified · Split
| @@ -201,18 +201,13 @@ alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr) · 200 unmodified lines | ||
| 201 | 201 | * workqueue function to free objects. |
| 202 | 202 | * |
| 203 | 203 | * To reduce contention on the global pool_lock, the actual freeing of |
| 204 | − * debug objects will be delayed if the pool_lock is busy. We also free | |
| 205 | − * the objects in a batch of 4 for each lock/unlock cycle. | |
| 204 | + * debug objects will be delayed if the pool_lock is busy. | |
| 206 | 205 | */ |
| 207 | −#define ODEBUG_FREE_BATCH 4 | |
| 208 | − | |
| 209 | 206 | static void free_obj_work(struct work_struct *work) |
| 210 | 207 | { |
| 211 | − struct debug_obj *objs[ODEBUG_FREE_BATCH]; | |
| 212 | 208 | struct hlist_node *tmp; |
| 213 | 209 | struct debug_obj *obj; |
| 214 | 210 | unsigned long flags; |
| 215 | − int i; | |
| 216 | 211 | HLIST_HEAD(tofree); |
| 217 | 212 | |
| 218 | 213 | if (!raw_spin_trylock_irqsave(&pool_lock, flags)) |
| @@ -240,26 +235,6 @@ static void free_obj_work(struct work_struct *work) · 21 unmodified lines | ||
| 240 | 235 | hlist_move_list(&obj_to_free, &tofree); |
| 241 | 236 | obj_nr_tofree = 0; |
| 242 | 237 | } |
| 243 | − | |
| 244 | − while (obj_pool_free >= debug_objects_pool_size + ODEBUG_FREE_BATCH) { | |
| 245 | − for (i = 0; i < ODEBUG_FREE_BATCH; i++) { | |
| 246 | − objs[i] = hlist_entry(obj_pool.first, | |
| 247 | − typeof(*objs[0]), node); | |
| 248 | − hlist_del(&objs[i]->node); | |
| 249 | − } | |
| 250 | − | |
| 251 | − obj_pool_free -= ODEBUG_FREE_BATCH; | |
| 252 | − debug_objects_freed += ODEBUG_FREE_BATCH; | |
| 253 | − /* | |
| 254 | − * We release pool_lock across kmem_cache_free() to | |
| 255 | − * avoid contention on pool_lock. | |
| 256 | − */ | |
| 257 | − raw_spin_unlock_irqrestore(&pool_lock, flags); | |
| 258 | − for (i = 0; i < ODEBUG_FREE_BATCH; i++) | |
| 259 | − kmem_cache_free(obj_cache, objs[i]); | |
| 260 | − if (!raw_spin_trylock_irqsave(&pool_lock, flags)) | |
| 261 | − return; | |
| 262 | − } | |
| 263 | 238 | raw_spin_unlock_irqrestore(&pool_lock, flags); |
| 264 | 239 | |
| 265 | 240 | hlist_for_each_entry_safe(obj, tmp, &tofree, node) { |
| @@ -268,27 +243,33 @@ static void free_obj_work(struct work_struct *work) · 2 unmodified lines | ||
| 268 | 243 | } |
| 269 | 244 | } |
| 270 | 245 | |
| 271 | −/* | |
| 272 | − * Put the object back into the pool and schedule work to free objects | |
| 273 | − * if necessary. | |
| 274 | − */ | |
| 275 | −static void free_object(struct debug_obj *obj) | |
| 246 | +static bool __free_object(struct debug_obj *obj) | |
| 276 | 247 | { |
| 277 | 248 | unsigned long flags; |
| 278 | − int sched = 0; | |
| 249 | + bool work; | |
| 279 | 250 | |
| 280 | 251 | raw_spin_lock_irqsave(&pool_lock, flags); |
| 281 | − /* | |
| 282 | − * schedule work when the pool is filled and the cache is | |
| 283 | − * initialized: | |
| 284 | − */ | |
| 285 | − if (obj_pool_free > debug_objects_pool_size && obj_cache) | |
| 286 | − sched = 1; | |
| 287 | − hlist_add_head(&obj->node, &obj_pool); | |
| 288 | − obj_pool_free++; | |
| 252 | + work = (obj_pool_free > debug_objects_pool_size) && obj_cache; | |
| 289 | 253 | obj_pool_used--; |
| 254 | + | |
| 255 | + if (work) { | |
| 256 | + obj_nr_tofree++; | |
| 257 | + hlist_add_head(&obj->node, &obj_to_free); | |
| 258 | + } else { | |
| 259 | + obj_pool_free++; | |
| 260 | + hlist_add_head(&obj->node, &obj_pool); | |
| 261 | + } | |
| 290 | 262 | raw_spin_unlock_irqrestore(&pool_lock, flags); |
| 291 | − if (sched) | |
| 263 | + return work; | |
| 264 | +} | |
| 265 | + | |
| 266 | +/* | |
| 267 | + * Put the object back into the pool and schedule work to free objects | |
| 268 | + * if necessary. | |
| 269 | + */ | |
| 270 | +static void free_object(struct debug_obj *obj) | |
| 271 | +{ | |
| 272 | + if (__free_object(obj)) | |
| 292 | 273 | schedule_work(&debug_obj_work); |
| 293 | 274 | } |
| 294 | 275 | |