History

debugobjects: Rework object allocation

The current allocation scheme tries to allocate from the per CPU pool first. If that fails it allocates one object from the global pool and then refills the per CPU pool from the global pool. That is in the way of switching the pool management to batch mode as the global pool needs to be a strict stack of batches, which does not allow to allocate single objects. Rework the code to refill the per CPU pool first and then allocate the object from the refilled batch. Also try to allocate from the to free pool first to avoid freeing and reallocating objects.

Thomas Gleixner committed 2 years ago

Commit fb60c001 parent 96a9a041 file · +69 −75Browse files

Showing the changes to lib/debugobjects.c (1 of 1 file). Show all

Unified · Split

lib/debugobjects.c+69−75
@@ -141,6 +141,64 @@ static __always_inline bool pool_must_refill(struct obj_pool *pool) · 140 unmodified lines
141 return pool_count(pool) < pool->min_cnt / 2;141 return pool_count(pool) < pool->min_cnt / 2;
142}142}
143 143
144static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src)
145{
146 if (dst->cnt + ODEBUG_BATCH_SIZE > dst->max_cnt || !src->cnt)
147 return false;
148
149 for (int i = 0; i < ODEBUG_BATCH_SIZE && src->cnt; i++) {
150 struct hlist_node *node = src->objects.first;
151
152 WRITE_ONCE(src->cnt, src->cnt - 1);
153 WRITE_ONCE(dst->cnt, dst->cnt + 1);
154
155 hlist_del(node);
156 hlist_add_head(node, &dst->objects);
157 }
158 return true;
159}
160
161static struct debug_obj *__alloc_object(struct hlist_head *list)
162{
163 struct debug_obj *obj;
164
165 if (unlikely(!list->first))
166 return NULL;
167
168 obj = hlist_entry(list->first, typeof(*obj), node);
169 hlist_del(&obj->node);
170 return obj;
171}
172
173static struct debug_obj *pcpu_alloc(void)
174{
175 struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu);
176
177 lockdep_assert_irqs_disabled();
178
179 for (;;) {
180 struct debug_obj *obj = __alloc_object(&pcp->objects);
181
182 if (likely(obj)) {
183 pcp->cnt--;
184 return obj;
185 }
186
187 guard(raw_spinlock)(&pool_lock);
188 if (!pool_move_batch(pcp, &pool_to_free)) {
189 if (!pool_move_batch(pcp, &pool_global))
190 return NULL;
191 }
192 obj_pool_used += pcp->cnt;
193
194 if (obj_pool_used > obj_pool_max_used)
195 obj_pool_max_used = obj_pool_used;
196
197 if (pool_global.cnt < obj_pool_min_free)
198 obj_pool_min_free = pool_global.cnt;
199 }
200}
201
144static void free_object_list(struct hlist_head *head)202static void free_object_list(struct hlist_head *head)
145{203{
146 struct hlist_node *tmp;204 struct hlist_node *tmp;
@@ -158,7 +216,6 @@ static void free_object_list(struct hlist_head *head) · 11 unmodified lines
158static void fill_pool_from_freelist(void)216static void fill_pool_from_freelist(void)
159{217{
160 static unsigned long state;218 static unsigned long state;
161 struct debug_obj *obj;
162 219
163 /*220 /*
164 * Reuse objs from the global obj_to_free list; they will be221 * Reuse objs from the global obj_to_free list; they will be
@@ -180,17 +237,11 @@ static void fill_pool_from_freelist(void) · 15 unmodified lines
180 if (test_bit(0, &state) || test_and_set_bit(0, &state))237 if (test_bit(0, &state) || test_and_set_bit(0, &state))
181 return;238 return;
182 239
183 guard(raw_spinlock)(&pool_lock);240 /* Avoid taking the lock when there is no work to do */
184 /*241 while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) {
185 * Recheck with the lock held as the worker thread might have242 guard(raw_spinlock)(&pool_lock);
186 * won the race and freed the global free list already.243 /* Move a batch if possible */
187 */244 pool_move_batch(&pool_global, &pool_to_free);
188 while (pool_to_free.cnt && (pool_global.cnt < pool_global.min_cnt)) {
189 obj = hlist_entry(pool_to_free.objects.first, typeof(*obj), node);
190 hlist_del(&obj->node);
191 WRITE_ONCE(pool_to_free.cnt, pool_to_free.cnt - 1);
192 hlist_add_head(&obj->node, &pool_global.objects);
193 WRITE_ONCE(pool_global.cnt, pool_global.cnt + 1);
194 }245 }
195 clear_bit(0, &state);246 clear_bit(0, &state);
196}247}
@@ -251,74 +302,17 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b) · 54 unmodified lines
251 return NULL;302 return NULL;
252}303}
253 304
254/*305static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,
255 * Allocate a new object from the hlist306 const struct debug_obj_descr *descr)
256 */
257static struct debug_obj *__alloc_object(struct hlist_head *list)
258{
259 struct debug_obj *obj = NULL;
260
261 if (list->first) {
262 obj = hlist_entry(list->first, typeof(*obj), node);
263 hlist_del(&obj->node);
264 }
265
266 return obj;
267}
268
269static struct debug_obj *
270alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
271{307{
272 struct obj_pool *percpu_pool = this_cpu_ptr(&pool_pcpu);
273 struct debug_obj *obj;308 struct debug_obj *obj;
274 309
275 if (likely(obj_cache)) {310 if (likely(obj_cache))
276 obj = __alloc_object(&percpu_pool->objects);311 obj = pcpu_alloc();
277 if (obj) {312 else
278 percpu_pool->cnt--;
279 goto init_obj;
280 }
281 } else {
282 obj = __alloc_object(&pool_boot);313 obj = __alloc_object(&pool_boot);
283 goto init_obj;
284 }
285
286 raw_spin_lock(&pool_lock);
287 obj = __alloc_object(&pool_global.objects);
288 if (obj) {
289 obj_pool_used++;
290 WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
291 314
292 /*315 if (likely(obj)) {
293 * Looking ahead, allocate one batch of debug objects and
294 * put them into the percpu free pool.
295 */
296 if (likely(obj_cache)) {
297 int i;
298
299 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
300 struct debug_obj *obj2;
301
302 obj2 = __alloc_object(&pool_global.objects);
303 if (!obj2)
304 break;
305 hlist_add_head(&obj2->node, &percpu_pool->objects);
306 percpu_pool->cnt++;
307 obj_pool_used++;
308 WRITE_ONCE(pool_global.cnt, pool_global.cnt - 1);
309 }
310 }
311
312 if (obj_pool_used > obj_pool_max_used)
313 obj_pool_max_used = obj_pool_used;
314
315 if (pool_global.cnt < obj_pool_min_free)
316 obj_pool_min_free = pool_global.cnt;
317 }
318 raw_spin_unlock(&pool_lock);
319
320init_obj:
321 if (obj) {
322 obj->object = addr;316 obj->object = addr;
323 obj->descr = descr;317 obj->descr = descr;
324 obj->state = ODEBUG_STATE_NONE;318 obj->state = ODEBUG_STATE_NONE;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t