History

tracing: Create a sparse bitmask for pid filtering

When the trace_pid_list was created, the default pid max was 32768. Creating a bitmask that can hold one bit for all 32768 took up 4096 (one page). Having a one page bitmask was not much of a problem, and that was used for mapping pids. But today, systems are bigger and can run more tasks, and now the default pid_max is usually set to 4194304. Which means to handle that many pids requires 524288 bytes. Worse yet, the pid_max can be set to 2^30 (1073741824 or 1G) which would take 134217728 (128M) of memory to store this array. Since the pid_list array is very sparsely populated, it is a huge waste of memory to store all possible bits for each pid when most will not be set. Instead, use a page table scheme to store the array, and allow this to handle up to 30 bit pids. The pid_mask will start out with 256 entries for the first 8 MSB bits. This will cost 1K for 32 bit architectures and 2K for 64 bit. Each of these will have a 256 array to store the next 8 bits of the pid (another 1 or 2K). These will hold an 2K byte bitmask (which will cover the LSB 14 bits or 16384 pids). When the trace_pid_list is allocated, it will have the 1/2K upper bits allocated, and then it will allocate a cache for the next upper chunks and the lower chunks (default 6 of each). Then when a bit is "set", these chunks will be pulled from the free list and added to the array. If the free list gets down to a lever (default 2), it will trigger an irqwork that will refill the cache back up. On clearing a bit, if the clear causes the bitmask to be zero, that chunk will then be placed back into the free cache for later use, keeping the need to allocate more down to a minimum.

Steven Rostedt (VMware) committed 5 years ago

Commit 8d6e9091 parent 6954e412 files · +445 −35Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/trace/pid_list.c +368 −33
  2. kernel/trace/pid_list.h +77 −2
kernel/trace/pid_list.c+368−33
@@ -2,10 +2,119 @@ · 1 unmodified line
2/*2/*
3 * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>3 * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
4 */4 */
5#include <linux/vmalloc.h>5#include <linux/spinlock.h>
6#include <linux/irq_work.h>
6#include <linux/slab.h>7#include <linux/slab.h>
7#include "trace.h"8#include "trace.h"
8 9
10/* See pid_list.h for details */
11
12static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
13{
14 union lower_chunk *chunk;
15
16 lockdep_assert_held(&pid_list->lock);
17
18 if (!pid_list->lower_list)
19 return NULL;
20
21 chunk = pid_list->lower_list;
22 pid_list->lower_list = chunk->next;
23 pid_list->free_lower_chunks--;
24 WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
25 chunk->next = NULL;
26 /*
27 * If a refill needs to happen, it can not happen here
28 * as the scheduler run queue locks are held.
29 */
30 if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
31 irq_work_queue(&pid_list->refill_irqwork);
32
33 return chunk;
34}
35
36static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
37{
38 union upper_chunk *chunk;
39
40 lockdep_assert_held(&pid_list->lock);
41
42 if (!pid_list->upper_list)
43 return NULL;
44
45 chunk = pid_list->upper_list;
46 pid_list->upper_list = chunk->next;
47 pid_list->free_upper_chunks--;
48 WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
49 chunk->next = NULL;
50 /*
51 * If a refill needs to happen, it can not happen here
52 * as the scheduler run queue locks are held.
53 */
54 if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
55 irq_work_queue(&pid_list->refill_irqwork);
56
57 return chunk;
58}
59
60static inline void put_lower_chunk(struct trace_pid_list *pid_list,
61 union lower_chunk *chunk)
62{
63 lockdep_assert_held(&pid_list->lock);
64
65 chunk->next = pid_list->lower_list;
66 pid_list->lower_list = chunk;
67 pid_list->free_lower_chunks++;
68}
69
70static inline void put_upper_chunk(struct trace_pid_list *pid_list,
71 union upper_chunk *chunk)
72{
73 lockdep_assert_held(&pid_list->lock);
74
75 chunk->next = pid_list->upper_list;
76 pid_list->upper_list = chunk;
77 pid_list->free_upper_chunks++;
78}
79
80static inline bool upper_empty(union upper_chunk *chunk)
81{
82 /*
83 * If chunk->data has no lower chunks, it will be the same
84 * as a zeroed bitmask. Use find_first_bit() to test it
85 * and if it doesn't find any bits set, then the array
86 * is empty.
87 */
88 int bit = find_first_bit((unsigned long *)chunk->data,
89 sizeof(chunk->data) * 8);
90 return bit >= sizeof(chunk->data) * 8;
91}
92
93static inline int pid_split(unsigned int pid, unsigned int *upper1,
94 unsigned int *upper2, unsigned int *lower)
95{
96 /* MAX_PID should cover all pids */
97 BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);
98
99 /* In case a bad pid is passed in, then fail */
100 if (unlikely(pid >= MAX_PID))
101 return -1;
102
103 *upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
104 *upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
105 *lower = pid & LOWER_MASK;
106
107 return 0;
108}
109
110static inline unsigned int pid_join(unsigned int upper1,
111 unsigned int upper2, unsigned int lower)
112{
113 return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
114 ((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
115 (lower & LOWER_MASK);
116}
117
9/**118/**
10 * trace_pid_list_is_set - test if the pid is set in the list119 * trace_pid_list_is_set - test if the pid is set in the list
11 * @pid_list: The pid list to test120 * @pid_list: The pid list to test
@@ -19,14 +128,30 @@ · 7 unmodified lines
19 */128 */
20bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)129bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
21{130{
22 /*131 union upper_chunk *upper_chunk;
23 * If pid_max changed after filtered_pids was created, we132 union lower_chunk *lower_chunk;
24 * by default ignore all pids greater than the previous pid_max.133 unsigned long flags;
25 */134 unsigned int upper1;
26 if (pid >= pid_list->pid_max)135 unsigned int upper2;
136 unsigned int lower;
137 bool ret = false;
138
139 if (!pid_list)
27 return false;140 return false;
28 141
29 return test_bit(pid, pid_list->pids);142 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
143 return false;
144
145 raw_spin_lock_irqsave(&pid_list->lock, flags);
146 upper_chunk = pid_list->upper[upper1];
147 if (upper_chunk) {
148 lower_chunk = upper_chunk->data[upper2];
149 if (lower_chunk)
150 ret = test_bit(lower, lower_chunk->data);
151 }
152 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
153
154 return ret;
30}155}
31 156
32/**157/**
@@ -42,13 +167,44 @@ bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid) · 9 unmodified lines
42 */167 */
43int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)168int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
44{169{
45 /* Sorry, but we don't support pid_max changing after setting */170 union upper_chunk *upper_chunk;
46 if (pid >= pid_list->pid_max)171 union lower_chunk *lower_chunk;
47 return -EINVAL;172 unsigned long flags;
173 unsigned int upper1;
174 unsigned int upper2;
175 unsigned int lower;
176 int ret;
48 177
49 set_bit(pid, pid_list->pids);178 if (!pid_list)
179 return -ENODEV;
50 180
51 return 0;181 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
182 return -EINVAL;
183
184 raw_spin_lock_irqsave(&pid_list->lock, flags);
185 upper_chunk = pid_list->upper[upper1];
186 if (!upper_chunk) {
187 upper_chunk = get_upper_chunk(pid_list);
188 if (!upper_chunk) {
189 ret = -ENOMEM;
190 goto out;
191 }
192 pid_list->upper[upper1] = upper_chunk;
193 }
194 lower_chunk = upper_chunk->data[upper2];
195 if (!lower_chunk) {
196 lower_chunk = get_lower_chunk(pid_list);
197 if (!lower_chunk) {
198 ret = -ENOMEM;
199 goto out;
200 }
201 upper_chunk->data[upper2] = lower_chunk;
202 }
203 set_bit(lower, lower_chunk->data);
204 ret = 0;
205 out:
206 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
207 return ret;
52}208}
53 209
54/**210/**
@@ -64,12 +220,41 @@ int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid) · 9 unmodified lines
64 */220 */
65int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)221int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
66{222{
67 /* Sorry, but we don't support pid_max changing after setting */223 union upper_chunk *upper_chunk;
68 if (pid >= pid_list->pid_max)224 union lower_chunk *lower_chunk;
225 unsigned long flags;
226 unsigned int upper1;
227 unsigned int upper2;
228 unsigned int lower;
229
230 if (!pid_list)
231 return -ENODEV;
232
233 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
69 return -EINVAL;234 return -EINVAL;
70 235
71 clear_bit(pid, pid_list->pids);236 raw_spin_lock_irqsave(&pid_list->lock, flags);
237 upper_chunk = pid_list->upper[upper1];
238 if (!upper_chunk)
239 goto out;
72 240
241 lower_chunk = upper_chunk->data[upper2];
242 if (!lower_chunk)
243 goto out;
244
245 clear_bit(lower, lower_chunk->data);
246
247 /* if there's no more bits set, add it to the free list */
248 if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
249 put_lower_chunk(pid_list, lower_chunk);
250 upper_chunk->data[upper2] = NULL;
251 if (upper_empty(upper_chunk)) {
252 put_upper_chunk(pid_list, upper_chunk);
253 pid_list->upper[upper1] = NULL;
254 }
255 }
256 out:
257 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
73 return 0;258 return 0;
74}259}
75 260
@@ -88,13 +273,45 @@ int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid) · 12 unmodified lines
88int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,273int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
89 unsigned int *next)274 unsigned int *next)
90{275{
91 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);276 union upper_chunk *upper_chunk;
277 union lower_chunk *lower_chunk;
278 unsigned long flags;
279 unsigned int upper1;
280 unsigned int upper2;
281 unsigned int lower;
282
283 if (!pid_list)
284 return -ENODEV;
285
286 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
287 return -EINVAL;
92 288
93 if (pid < pid_list->pid_max) {289 raw_spin_lock_irqsave(&pid_list->lock, flags);
94 *next = pid;290 for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
95 return 0;291 upper_chunk = pid_list->upper[upper1];
292
293 if (!upper_chunk)
294 continue;
295
296 for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
297 lower_chunk = upper_chunk->data[upper2];
298 if (!lower_chunk)
299 continue;
300
301 lower = find_next_bit(lower_chunk->data, LOWER_MAX,
302 lower);
303 if (lower < LOWER_MAX)
304 goto found;
305 }
96 }306 }
97 return -1;307
308 found:
309 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
310 if (upper1 > UPPER_MASK)
311 return -1;
312
313 *next = pid_join(upper1, upper2, lower);
314 return 0;
98}315}
99 316
100/**317/**
@@ -109,15 +326,79 @@ int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid, · 8 unmodified lines
109 */326 */
110int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)327int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
111{328{
112 unsigned int first;329 return trace_pid_list_next(pid_list, 0, pid);
330}
331
332static void pid_list_refill_irq(struct irq_work *iwork)
333{
334 struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
335 refill_irqwork);
336 union upper_chunk *upper;
337 union lower_chunk *lower;
338 union upper_chunk **upper_next = &upper;
339 union lower_chunk **lower_next = &lower;
340 int upper_count;
341 int lower_count;
342 int ucnt = 0;
343 int lcnt = 0;
113 344
114 first = find_first_bit(pid_list->pids, pid_list->pid_max);345 again:
346 raw_spin_lock(&pid_list->lock);
347 upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
348 lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
349 raw_spin_unlock(&pid_list->lock);
350
351 if (upper_count <= 0 && lower_count <= 0)
352 return;
115 353
116 if (first < pid_list->pid_max) {354 while (upper_count-- > 0) {
117 *pid = first;355 union upper_chunk *chunk;
118 return 0;356
357 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
358 if (!chunk)
359 break;
360 *upper_next = chunk;
361 upper_next = &chunk->next;
362 ucnt++;
119 }363 }
120 return -1;364
365 while (lower_count-- > 0) {
366 union lower_chunk *chunk;
367
368 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
369 if (!chunk)
370 break;
371 *lower_next = chunk;
372 lower_next = &chunk->next;
373 lcnt++;
374 }
375
376 raw_spin_lock(&pid_list->lock);
377 if (upper) {
378 *upper_next = pid_list->upper_list;
379 pid_list->upper_list = upper;
380 pid_list->free_upper_chunks += ucnt;
381 }
382 if (lower) {
383 *lower_next = pid_list->lower_list;
384 pid_list->lower_list = lower;
385 pid_list->free_lower_chunks += lcnt;
386 }
387 raw_spin_unlock(&pid_list->lock);
388
389 /*
390 * On success of allocating all the chunks, both counters
391 * will be less than zero. If they are not, then an allocation
392 * failed, and we should not try again.
393 */
394 if (upper_count >= 0 || lower_count >= 0)
395 return;
396 /*
397 * When the locks were released, free chunks could have
398 * been used and allocation needs to be done again. Might as
399 * well allocate it now.
400 */
401 goto again;
121}402}
122 403
123/**404/**
@@ -130,18 +411,41 @@ int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid) · 6 unmodified lines
130struct trace_pid_list *trace_pid_list_alloc(void)411struct trace_pid_list *trace_pid_list_alloc(void)
131{412{
132 struct trace_pid_list *pid_list;413 struct trace_pid_list *pid_list;
414 int i;
415
416 /* According to linux/thread.h, pids can be no bigger that 30 bits */
417 WARN_ON_ONCE(pid_max > (1 << 30));
133 418
134 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);419 pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
135 if (!pid_list)420 if (!pid_list)
136 return NULL;421 return NULL;
137 422
138 pid_list->pid_max = READ_ONCE(pid_max);423 init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);
139 424
140 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);425 raw_spin_lock_init(&pid_list->lock);
141 if (!pid_list->pids) {426
142 kfree(pid_list);427 for (i = 0; i < CHUNK_ALLOC; i++) {
143 return NULL;428 union upper_chunk *chunk;
429
430 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
431 if (!chunk)
432 break;
433 chunk->next = pid_list->upper_list;
434 pid_list->upper_list = chunk;
435 pid_list->free_upper_chunks++;
144 }436 }
437
438 for (i = 0; i < CHUNK_ALLOC; i++) {
439 union lower_chunk *chunk;
440
441 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
442 if (!chunk)
443 break;
444 chunk->next = pid_list->lower_list;
445 pid_list->lower_list = chunk;
446 pid_list->free_lower_chunks++;
447 }
448
145 return pid_list;449 return pid_list;
146}450}
147 451
@@ -152,9 +456,40 @@ struct trace_pid_list *trace_pid_list_alloc(void) · 4 unmodified lines
152 */456 */
153void trace_pid_list_free(struct trace_pid_list *pid_list)457void trace_pid_list_free(struct trace_pid_list *pid_list)
154{458{
459 union upper_chunk *upper;
460 union lower_chunk *lower;
461 int i, j;
462
155 if (!pid_list)463 if (!pid_list)
156 return;464 return;
157 465
158 vfree(pid_list->pids);466 irq_work_sync(&pid_list->refill_irqwork);
467
468 while (pid_list->lower_list) {
469 union lower_chunk *chunk;
470
471 chunk = pid_list->lower_list;
472 pid_list->lower_list = pid_list->lower_list->next;
473 kfree(chunk);
474 }
475
476 while (pid_list->upper_list) {
477 union upper_chunk *chunk;
478
479 chunk = pid_list->upper_list;
480 pid_list->upper_list = pid_list->upper_list->next;
481 kfree(chunk);
482 }
483
484 for (i = 0; i < UPPER1_SIZE; i++) {
485 upper = pid_list->upper[i];
486 if (upper) {
487 for (j = 0; j < UPPER2_SIZE; j++) {
488 lower = upper->data[j];
489 kfree(lower);
490 }
491 kfree(upper);
492 }
493 }
159 kfree(pid_list);494 kfree(pid_list);
160}495}
kernel/trace/pid_list.h+77−2
@@ -5,9 +5,84 @@ · 4 unmodified lines
5#ifndef _TRACE_INTERNAL_PID_LIST_H5#ifndef _TRACE_INTERNAL_PID_LIST_H
6#define _TRACE_INTERNAL_PID_LIST_H6#define _TRACE_INTERNAL_PID_LIST_H
7 7
8/*
9 * In order to keep track of what pids to trace, a tree is created much
10 * like page tables are used. This creates a sparse bit map, where
11 * the tree is filled in when needed. A PID is at most 30 bits (see
12 * linux/thread.h), and is broken up into 3 sections based on the bit map
13 * of the bits. The 8 MSB is the "upper1" section. The next 8 MSB is the
14 * "upper2" section and the 14 LSB is the "lower" section.
15 *
16 * A trace_pid_list structure holds the "upper1" section, in an
17 * array of 256 pointers (1 or 2K in size) to "upper_chunk" unions, where
18 * each has an array of 256 pointers (1 or 2K in size) to the "lower_chunk"
19 * structures, where each has an array of size 2K bytes representing a bitmask
20 * of the 14 LSB of the PID (256 * 8 = 2048)
21 *
22 * When a trace_pid_list is allocated, it includes the 256 pointer array
23 * of the upper1 unions. Then a "cache" of upper and lower is allocated
24 * where these will be assigned as needed.
25 *
26 * When a bit is set in the pid_list bitmask, the pid to use has
27 * the 8 MSB masked, and this is used to index the array in the
28 * pid_list to find the next upper union. If the element is NULL,
29 * then one is retrieved from the upper_list cache. If none is
30 * available, then -ENOMEM is returned.
31 *
32 * The next 8 MSB is used to index into the "upper2" section. If this
33 * element is NULL, then it is retrieved from the lower_list cache.
34 * Again, if one is not available -ENOMEM is returned.
35 *
36 * Finally the 14 LSB of the PID is used to set the bit in the 16384
37 * bitmask (made up of 2K bytes).
38 *
39 * When the second upper section or the lower section has their last
40 * bit cleared, they are added back to the free list to be reused
41 * when needed.
42 */
43
44#define UPPER_BITS 8
45#define UPPER_MAX (1 << UPPER_BITS)
46#define UPPER1_SIZE (1 << UPPER_BITS)
47#define UPPER2_SIZE (1 << UPPER_BITS)
48
49#define LOWER_BITS 14
50#define LOWER_MAX (1 << LOWER_BITS)
51#define LOWER_SIZE (LOWER_MAX / BITS_PER_LONG)
52
53#define UPPER1_SHIFT (LOWER_BITS + UPPER_BITS)
54#define UPPER2_SHIFT LOWER_BITS
55#define LOWER_MASK (LOWER_MAX - 1)
56
57#define UPPER_MASK (UPPER_MAX - 1)
58
59/* According to linux/thread.h pids can not be bigger than or equal to 1 << 30 */
60#define MAX_PID (1 << 30)
61
62/* Just keep 6 chunks of both upper and lower in the cache on alloc */
63#define CHUNK_ALLOC 6
64
65/* Have 2 chunks free, trigger a refill of the cache */
66#define CHUNK_REALLOC 2
67
68union lower_chunk {
69 union lower_chunk *next;
70 unsigned long data[LOWER_SIZE]; // 2K in size
71};
72
73union upper_chunk {
74 union upper_chunk *next;
75 union lower_chunk *data[UPPER2_SIZE]; // 1 or 2K in size
76};
77
8struct trace_pid_list {78struct trace_pid_list {
9 int pid_max;79 raw_spinlock_t lock;
10 unsigned long *pids;80 struct irq_work refill_irqwork;
81 union upper_chunk *upper[UPPER1_SIZE]; // 1 or 2K in size
82 union upper_chunk *upper_list;
83 union lower_chunk *lower_list;
84 int free_upper_chunks;
85 int free_lower_chunks;
11};86};
12 87
13#endif /* _TRACE_INTERNAL_PID_LIST_H */88#endif /* _TRACE_INTERNAL_PID_LIST_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t