History

tracing: Create a sparse bitmask for pid filtering

When the trace_pid_list was created, the default pid max was 32768. Creating a bitmask that can hold one bit for all 32768 took up 4096 (one page). Having a one page bitmask was not much of a problem, and that was used for mapping pids. But today, systems are bigger and can run more tasks, and now the default pid_max is usually set to 4194304. Which means to handle that many pids requires 524288 bytes. Worse yet, the pid_max can be set to 2^30 (1073741824 or 1G) which would take 134217728 (128M) of memory to store this array. Since the pid_list array is very sparsely populated, it is a huge waste of memory to store all possible bits for each pid when most will not be set. Instead, use a page table scheme to store the array, and allow this to handle up to 30 bit pids. The pid_mask will start out with 256 entries for the first 8 MSB bits. This will cost 1K for 32 bit architectures and 2K for 64 bit. Each of these will have a 256 array to store the next 8 bits of the pid (another 1 or 2K). These will hold an 2K byte bitmask (which will cover the LSB 14 bits or 16384 pids). When the trace_pid_list is allocated, it will have the 1/2K upper bits allocated, and then it will allocate a cache for the next upper chunks and the lower chunks (default 6 of each). Then when a bit is "set", these chunks will be pulled from the free list and added to the array. If the free list gets down to a lever (default 2), it will trigger an irqwork that will refill the cache back up. On clearing a bit, if the clear causes the bitmask to be zero, that chunk will then be placed back into the free cache for later use, keeping the need to allocate more down to a minimum.

Steven Rostedt (VMware) committed 5 years ago

Commit 8d6e9091 parent 6954e412 files · +445 −35Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/trace/pid_list.c +368 −33
  2. kernel/trace/pid_list.h +77 −2
kernel/trace/pid_list.c+368−33
@@ -2,10 +2,119 @@ · 1 unmodified line
22 /*
33 * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
44 */
5−#include <linux/vmalloc.h>
5+#include <linux/spinlock.h>
6+#include <linux/irq_work.h>
67 #include <linux/slab.h>
78 #include "trace.h"
89
10+/* See pid_list.h for details */
11+
12+static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
13+{
14+ union lower_chunk *chunk;
15+
16+ lockdep_assert_held(&pid_list->lock);
17+
18+ if (!pid_list->lower_list)
19+ return NULL;
20+
21+ chunk = pid_list->lower_list;
22+ pid_list->lower_list = chunk->next;
23+ pid_list->free_lower_chunks--;
24+ WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
25+ chunk->next = NULL;
26+ /*
27+ * If a refill needs to happen, it can not happen here
28+ * as the scheduler run queue locks are held.
29+ */
30+ if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
31+ irq_work_queue(&pid_list->refill_irqwork);
32+
33+ return chunk;
34+}
35+
36+static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
37+{
38+ union upper_chunk *chunk;
39+
40+ lockdep_assert_held(&pid_list->lock);
41+
42+ if (!pid_list->upper_list)
43+ return NULL;
44+
45+ chunk = pid_list->upper_list;
46+ pid_list->upper_list = chunk->next;
47+ pid_list->free_upper_chunks--;
48+ WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
49+ chunk->next = NULL;
50+ /*
51+ * If a refill needs to happen, it can not happen here
52+ * as the scheduler run queue locks are held.
53+ */
54+ if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
55+ irq_work_queue(&pid_list->refill_irqwork);
56+
57+ return chunk;
58+}
59+
60+static inline void put_lower_chunk(struct trace_pid_list *pid_list,
61+ union lower_chunk *chunk)
62+{
63+ lockdep_assert_held(&pid_list->lock);
64+
65+ chunk->next = pid_list->lower_list;
66+ pid_list->lower_list = chunk;
67+ pid_list->free_lower_chunks++;
68+}
69+
70+static inline void put_upper_chunk(struct trace_pid_list *pid_list,
71+ union upper_chunk *chunk)
72+{
73+ lockdep_assert_held(&pid_list->lock);
74+
75+ chunk->next = pid_list->upper_list;
76+ pid_list->upper_list = chunk;
77+ pid_list->free_upper_chunks++;
78+}
79+
80+static inline bool upper_empty(union upper_chunk *chunk)
81+{
82+ /*
83+ * If chunk->data has no lower chunks, it will be the same
84+ * as a zeroed bitmask. Use find_first_bit() to test it
85+ * and if it doesn't find any bits set, then the array
86+ * is empty.
87+ */
88+ int bit = find_first_bit((unsigned long *)chunk->data,
89+ sizeof(chunk->data) * 8);
90+ return bit >= sizeof(chunk->data) * 8;
91+}
92+
93+static inline int pid_split(unsigned int pid, unsigned int *upper1,
94+ unsigned int *upper2, unsigned int *lower)
95+{
96+ /* MAX_PID should cover all pids */
97+ BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);
98+
99+ /* In case a bad pid is passed in, then fail */
100+ if (unlikely(pid >= MAX_PID))
101+ return -1;
102+
103+ *upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
104+ *upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
105+ *lower = pid & LOWER_MASK;
106+
107+ return 0;
108+}
109+
110+static inline unsigned int pid_join(unsigned int upper1,
111+ unsigned int upper2, unsigned int lower)
112+{
113+ return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
114+ ((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
115+ (lower & LOWER_MASK);
116+}
117+
9118 /**
10119 * trace_pid_list_is_set - test if the pid is set in the list
11120 * @pid_list: The pid list to test
@@ -19,14 +128,30 @@ · 7 unmodified lines
19128 */
20129 bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
21130 {
22− /*
23− * If pid_max changed after filtered_pids was created, we
24− * by default ignore all pids greater than the previous pid_max.
25− */
26− if (pid >= pid_list->pid_max)
131+ union upper_chunk *upper_chunk;
132+ union lower_chunk *lower_chunk;
133+ unsigned long flags;
134+ unsigned int upper1;
135+ unsigned int upper2;
136+ unsigned int lower;
137+ bool ret = false;
138+
139+ if (!pid_list)
27140 return false;
28141
29− return test_bit(pid, pid_list->pids);
142+ if (pid_split(pid, &upper1, &upper2, &lower) < 0)
143+ return false;
144+
145+ raw_spin_lock_irqsave(&pid_list->lock, flags);
146+ upper_chunk = pid_list->upper[upper1];
147+ if (upper_chunk) {
148+ lower_chunk = upper_chunk->data[upper2];
149+ if (lower_chunk)
150+ ret = test_bit(lower, lower_chunk->data);
151+ }
152+ raw_spin_unlock_irqrestore(&pid_list->lock, flags);
153+
154+ return ret;
30155 }
31156
32157 /**
@@ -42,13 +167,44 @@ bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid) · 9 unmodified lines
42167 */
43168 int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
44169 {
45− /* Sorry, but we don't support pid_max changing after setting */
46− if (pid >= pid_list->pid_max)
47− return -EINVAL;
170+ union upper_chunk *upper_chunk;
171+ union lower_chunk *lower_chunk;
172+ unsigned long flags;
173+ unsigned int upper1;
174+ unsigned int upper2;
175+ unsigned int lower;
176+ int ret;
48177
49− set_bit(pid, pid_list->pids);
178+ if (!pid_list)
179+ return -ENODEV;
50180
51− return 0;
181+ if (pid_split(pid, &upper1, &upper2, &lower) < 0)
182+ return -EINVAL;
183+
184+ raw_spin_lock_irqsave(&pid_list->lock, flags);
185+ upper_chunk = pid_list->upper[upper1];
186+ if (!upper_chunk) {
187+ upper_chunk = get_upper_chunk(pid_list);
188+ if (!upper_chunk) {
189+ ret = -ENOMEM;
190+ goto out;
191+ }
192+ pid_list->upper[upper1] = upper_chunk;
193+ }
194+ lower_chunk = upper_chunk->data[upper2];
195+ if (!lower_chunk) {
196+ lower_chunk = get_lower_chunk(pid_list);
197+ if (!lower_chunk) {
198+ ret = -ENOMEM;
199+ goto out;
200+ }
201+ upper_chunk->data[upper2] = lower_chunk;
202+ }
203+ set_bit(lower, lower_chunk->data);
204+ ret = 0;
205+ out:
206+ raw_spin_unlock_irqrestore(&pid_list->lock, flags);
207+ return ret;
52208 }
53209
54210 /**
@@ -64,12 +220,41 @@ int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid) · 9 unmodified lines
64220 */
65221 int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
66222 {
67− /* Sorry, but we don't support pid_max changing after setting */
68− if (pid >= pid_list->pid_max)
223+ union upper_chunk *upper_chunk;
224+ union lower_chunk *lower_chunk;
225+ unsigned long flags;
226+ unsigned int upper1;
227+ unsigned int upper2;
228+ unsigned int lower;
229+
230+ if (!pid_list)
231+ return -ENODEV;
232+
233+ if (pid_split(pid, &upper1, &upper2, &lower) < 0)
69234 return -EINVAL;
70235
71− clear_bit(pid, pid_list->pids);
236+ raw_spin_lock_irqsave(&pid_list->lock, flags);
237+ upper_chunk = pid_list->upper[upper1];
238+ if (!upper_chunk)
239+ goto out;
72240
241+ lower_chunk = upper_chunk->data[upper2];
242+ if (!lower_chunk)
243+ goto out;
244+
245+ clear_bit(lower, lower_chunk->data);
246+
247+ /* if there's no more bits set, add it to the free list */
248+ if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
249+ put_lower_chunk(pid_list, lower_chunk);
250+ upper_chunk->data[upper2] = NULL;
251+ if (upper_empty(upper_chunk)) {
252+ put_upper_chunk(pid_list, upper_chunk);
253+ pid_list->upper[upper1] = NULL;
254+ }
255+ }
256+ out:
257+ raw_spin_unlock_irqrestore(&pid_list->lock, flags);
73258 return 0;
74259 }
75260
@@ -88,13 +273,45 @@ int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid) · 12 unmodified lines
88273 int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
89274 unsigned int *next)
90275 {
91− pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
276+ union upper_chunk *upper_chunk;
277+ union lower_chunk *lower_chunk;
278+ unsigned long flags;
279+ unsigned int upper1;
280+ unsigned int upper2;
281+ unsigned int lower;
282+
283+ if (!pid_list)
284+ return -ENODEV;
285+
286+ if (pid_split(pid, &upper1, &upper2, &lower) < 0)
287+ return -EINVAL;
92288
93− if (pid < pid_list->pid_max) {
94− *next = pid;
95− return 0;
289+ raw_spin_lock_irqsave(&pid_list->lock, flags);
290+ for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
291+ upper_chunk = pid_list->upper[upper1];
292+
293+ if (!upper_chunk)
294+ continue;
295+
296+ for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
297+ lower_chunk = upper_chunk->data[upper2];
298+ if (!lower_chunk)
299+ continue;
300+
301+ lower = find_next_bit(lower_chunk->data, LOWER_MAX,
302+ lower);
303+ if (lower < LOWER_MAX)
304+ goto found;
305+ }
96306 }
97− return -1;
307+
308+ found:
309+ raw_spin_unlock_irqrestore(&pid_list->lock, flags);
310+ if (upper1 > UPPER_MASK)
311+ return -1;
312+
313+ *next = pid_join(upper1, upper2, lower);
314+ return 0;
98315 }
99316
100317 /**
@@ -109,15 +326,79 @@ int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid, · 8 unmodified lines
109326 */
110327 int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
111328 {
112− unsigned int first;
329+ return trace_pid_list_next(pid_list, 0, pid);
330+}
331+
332+static void pid_list_refill_irq(struct irq_work *iwork)
333+{
334+ struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
335+ refill_irqwork);
336+ union upper_chunk *upper;
337+ union lower_chunk *lower;
338+ union upper_chunk **upper_next = &upper;
339+ union lower_chunk **lower_next = &lower;
340+ int upper_count;
341+ int lower_count;
342+ int ucnt = 0;
343+ int lcnt = 0;
113344
114− first = find_first_bit(pid_list->pids, pid_list->pid_max);
345+ again:
346+ raw_spin_lock(&pid_list->lock);
347+ upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
348+ lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
349+ raw_spin_unlock(&pid_list->lock);
350+
351+ if (upper_count <= 0 && lower_count <= 0)
352+ return;
115353
116− if (first < pid_list->pid_max) {
117− *pid = first;
118− return 0;
354+ while (upper_count-- > 0) {
355+ union upper_chunk *chunk;
356+
357+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
358+ if (!chunk)
359+ break;
360+ *upper_next = chunk;
361+ upper_next = &chunk->next;
362+ ucnt++;
119363 }
120− return -1;
364+
365+ while (lower_count-- > 0) {
366+ union lower_chunk *chunk;
367+
368+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
369+ if (!chunk)
370+ break;
371+ *lower_next = chunk;
372+ lower_next = &chunk->next;
373+ lcnt++;
374+ }
375+
376+ raw_spin_lock(&pid_list->lock);
377+ if (upper) {
378+ *upper_next = pid_list->upper_list;
379+ pid_list->upper_list = upper;
380+ pid_list->free_upper_chunks += ucnt;
381+ }
382+ if (lower) {
383+ *lower_next = pid_list->lower_list;
384+ pid_list->lower_list = lower;
385+ pid_list->free_lower_chunks += lcnt;
386+ }
387+ raw_spin_unlock(&pid_list->lock);
388+
389+ /*
390+ * On success of allocating all the chunks, both counters
391+ * will be less than zero. If they are not, then an allocation
392+ * failed, and we should not try again.
393+ */
394+ if (upper_count >= 0 || lower_count >= 0)
395+ return;
396+ /*
397+ * When the locks were released, free chunks could have
398+ * been used and allocation needs to be done again. Might as
399+ * well allocate it now.
400+ */
401+ goto again;
121402 }
122403
123404 /**
@@ -130,18 +411,41 @@ int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid) · 6 unmodified lines
130411 struct trace_pid_list *trace_pid_list_alloc(void)
131412 {
132413 struct trace_pid_list *pid_list;
414+ int i;
415+
416+ /* According to linux/thread.h, pids can be no bigger that 30 bits */
417+ WARN_ON_ONCE(pid_max > (1 << 30));
133418
134− pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
419+ pid_list = kzalloc(sizeof(*pid_list), GFP_KERNEL);
135420 if (!pid_list)
136421 return NULL;
137422
138− pid_list->pid_max = READ_ONCE(pid_max);
423+ init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);
139424
140− pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
141− if (!pid_list->pids) {
142− kfree(pid_list);
143− return NULL;
425+ raw_spin_lock_init(&pid_list->lock);
426+
427+ for (i = 0; i < CHUNK_ALLOC; i++) {
428+ union upper_chunk *chunk;
429+
430+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
431+ if (!chunk)
432+ break;
433+ chunk->next = pid_list->upper_list;
434+ pid_list->upper_list = chunk;
435+ pid_list->free_upper_chunks++;
144436 }
437+
438+ for (i = 0; i < CHUNK_ALLOC; i++) {
439+ union lower_chunk *chunk;
440+
441+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
442+ if (!chunk)
443+ break;
444+ chunk->next = pid_list->lower_list;
445+ pid_list->lower_list = chunk;
446+ pid_list->free_lower_chunks++;
447+ }
448+
145449 return pid_list;
146450 }
147451
@@ -152,9 +456,40 @@ struct trace_pid_list *trace_pid_list_alloc(void) · 4 unmodified lines
152456 */
153457 void trace_pid_list_free(struct trace_pid_list *pid_list)
154458 {
459+ union upper_chunk *upper;
460+ union lower_chunk *lower;
461+ int i, j;
462+
155463 if (!pid_list)
156464 return;
157465
158− vfree(pid_list->pids);
466+ irq_work_sync(&pid_list->refill_irqwork);
467+
468+ while (pid_list->lower_list) {
469+ union lower_chunk *chunk;
470+
471+ chunk = pid_list->lower_list;
472+ pid_list->lower_list = pid_list->lower_list->next;
473+ kfree(chunk);
474+ }
475+
476+ while (pid_list->upper_list) {
477+ union upper_chunk *chunk;
478+
479+ chunk = pid_list->upper_list;
480+ pid_list->upper_list = pid_list->upper_list->next;
481+ kfree(chunk);
482+ }
483+
484+ for (i = 0; i < UPPER1_SIZE; i++) {
485+ upper = pid_list->upper[i];
486+ if (upper) {
487+ for (j = 0; j < UPPER2_SIZE; j++) {
488+ lower = upper->data[j];
489+ kfree(lower);
490+ }
491+ kfree(upper);
492+ }
493+ }
159494 kfree(pid_list);
160495 }
kernel/trace/pid_list.h+77−2
@@ -5,9 +5,84 @@ · 4 unmodified lines
55 #ifndef _TRACE_INTERNAL_PID_LIST_H
66 #define _TRACE_INTERNAL_PID_LIST_H
77
8+/*
9+ * In order to keep track of what pids to trace, a tree is created much
10+ * like page tables are used. This creates a sparse bit map, where
11+ * the tree is filled in when needed. A PID is at most 30 bits (see
12+ * linux/thread.h), and is broken up into 3 sections based on the bit map
13+ * of the bits. The 8 MSB is the "upper1" section. The next 8 MSB is the
14+ * "upper2" section and the 14 LSB is the "lower" section.
15+ *
16+ * A trace_pid_list structure holds the "upper1" section, in an
17+ * array of 256 pointers (1 or 2K in size) to "upper_chunk" unions, where
18+ * each has an array of 256 pointers (1 or 2K in size) to the "lower_chunk"
19+ * structures, where each has an array of size 2K bytes representing a bitmask
20+ * of the 14 LSB of the PID (256 * 8 = 2048)
21+ *
22+ * When a trace_pid_list is allocated, it includes the 256 pointer array
23+ * of the upper1 unions. Then a "cache" of upper and lower is allocated
24+ * where these will be assigned as needed.
25+ *
26+ * When a bit is set in the pid_list bitmask, the pid to use has
27+ * the 8 MSB masked, and this is used to index the array in the
28+ * pid_list to find the next upper union. If the element is NULL,
29+ * then one is retrieved from the upper_list cache. If none is
30+ * available, then -ENOMEM is returned.
31+ *
32+ * The next 8 MSB is used to index into the "upper2" section. If this
33+ * element is NULL, then it is retrieved from the lower_list cache.
34+ * Again, if one is not available -ENOMEM is returned.
35+ *
36+ * Finally the 14 LSB of the PID is used to set the bit in the 16384
37+ * bitmask (made up of 2K bytes).
38+ *
39+ * When the second upper section or the lower section has their last
40+ * bit cleared, they are added back to the free list to be reused
41+ * when needed.
42+ */
43+
44+#define UPPER_BITS 8
45+#define UPPER_MAX (1 << UPPER_BITS)
46+#define UPPER1_SIZE (1 << UPPER_BITS)
47+#define UPPER2_SIZE (1 << UPPER_BITS)
48+
49+#define LOWER_BITS 14
50+#define LOWER_MAX (1 << LOWER_BITS)
51+#define LOWER_SIZE (LOWER_MAX / BITS_PER_LONG)
52+
53+#define UPPER1_SHIFT (LOWER_BITS + UPPER_BITS)
54+#define UPPER2_SHIFT LOWER_BITS
55+#define LOWER_MASK (LOWER_MAX - 1)
56+
57+#define UPPER_MASK (UPPER_MAX - 1)
58+
59+/* According to linux/thread.h pids can not be bigger than or equal to 1 << 30 */
60+#define MAX_PID (1 << 30)
61+
62+/* Just keep 6 chunks of both upper and lower in the cache on alloc */
63+#define CHUNK_ALLOC 6
64+
65+/* Have 2 chunks free, trigger a refill of the cache */
66+#define CHUNK_REALLOC 2
67+
68+union lower_chunk {
69+ union lower_chunk *next;
70+ unsigned long data[LOWER_SIZE]; // 2K in size
71+};
72+
73+union upper_chunk {
74+ union upper_chunk *next;
75+ union lower_chunk *data[UPPER2_SIZE]; // 1 or 2K in size
76+};
77+
878 struct trace_pid_list {
9− int pid_max;
10− unsigned long *pids;
79+ raw_spinlock_t lock;
80+ struct irq_work refill_irqwork;
81+ union upper_chunk *upper[UPPER1_SIZE]; // 1 or 2K in size
82+ union upper_chunk *upper_list;
83+ union lower_chunk *lower_list;
84+ int free_upper_chunks;
85+ int free_lower_chunks;
1186 };
1287
1388 #endif /* _TRACE_INTERNAL_PID_LIST_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t