History

bpf: Enable task local storage for tracing programs

To access per-task data, BPF programs usually creates a hash table with pid as the key. This is not ideal because: 1. The user need to estimate the proper size of the hash table, which may be inaccurate; 2. Big hash tables are slow; 3. To clean up the data properly during task terminations, the user need to write extra logic. Task local storage overcomes these issues and offers a better option for these per-task data. Task local storage is only available to BPF_LSM. Now enable it for tracing programs. Unlike LSM programs, tracing programs can be called in IRQ contexts. Helpers that access task local storage are updated to use raw_spin_lock_irqsave() instead of raw_spin_lock_bh(). Tracing programs can attach to functions on the task free path, e.g. exit_creds(). To avoid allocating task local storage after bpf_task_storage_free(). bpf_task_storage_get() is updated to not allocate new storage when the task is not refcounted (task->usage == 0).

Song Liu committed 5 years ago

Commit a10787e1 parent 9c8f21e10 files · +51 −72Browse files

Showing the changes to kernel (6 of 10 files). Show all

Unified · Split

6 files
  1. kernel/bpf/Makefile +1 −2
  2. kernel/bpf/bpf_local_storage.c +17 −11
  3. kernel/bpf/bpf_lsm.c +0 −4
  4. kernel/bpf/bpf_task_storage.c +11 −32
  5. kernel/fork.c +5 −0
  6. kernel/trace/bpf_trace.c +4 −0
kernel/bpf/Makefile+1−2
@@ -9,8 +9,8 @@ CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) · 8 unmodified lines
99 obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
1010 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
1111 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
12+obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
1213 obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
13−obj-${CONFIG_BPF_LSM} += bpf_task_storage.o
1414 obj-$(CONFIG_BPF_SYSCALL) += disasm.o
1515 obj-$(CONFIG_BPF_JIT) += trampoline.o
1616 obj-$(CONFIG_BPF_SYSCALL) += btf.o
@@ -18,7 +18,6 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o · 1 unmodified line
1818 ifeq ($(CONFIG_NET),y)
1919 obj-$(CONFIG_BPF_SYSCALL) += devmap.o
2020 obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
21−obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o
2221 obj-$(CONFIG_BPF_SYSCALL) += offload.o
2322 obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
2423 endif
kernel/bpf/bpf_local_storage.c+17−11
@@ -140,17 +140,18 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 139 unmodified lines
140140 {
141141 struct bpf_local_storage *local_storage;
142142 bool free_local_storage = false;
143+ unsigned long flags;
143144
144145 if (unlikely(!selem_linked_to_storage(selem)))
145146 /* selem has already been unlinked from sk */
146147 return;
147148
148149 local_storage = rcu_dereference(selem->local_storage);
149− raw_spin_lock_bh(&local_storage->lock);
150+ raw_spin_lock_irqsave(&local_storage->lock, flags);
150151 if (likely(selem_linked_to_storage(selem)))
151152 free_local_storage = bpf_selem_unlink_storage_nolock(
152153 local_storage, selem, true);
153− raw_spin_unlock_bh(&local_storage->lock);
154+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
154155
155156 if (free_local_storage)
156157 kfree_rcu(local_storage, rcu);
@@ -167,6 +168,7 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 10 unmodified lines
167168 {
168169 struct bpf_local_storage_map *smap;
169170 struct bpf_local_storage_map_bucket *b;
171+ unsigned long flags;
170172
171173 if (unlikely(!selem_linked_to_map(selem)))
172174 /* selem has already be unlinked from smap */
@@ -174,21 +176,22 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 1 unmodified line
174176
175177 smap = rcu_dereference(SDATA(selem)->smap);
176178 b = select_bucket(smap, selem);
177− raw_spin_lock_bh(&b->lock);
179+ raw_spin_lock_irqsave(&b->lock, flags);
178180 if (likely(selem_linked_to_map(selem)))
179181 hlist_del_init_rcu(&selem->map_node);
180− raw_spin_unlock_bh(&b->lock);
182+ raw_spin_unlock_irqrestore(&b->lock, flags);
181183 }
182184
183185 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
184186 struct bpf_local_storage_elem *selem)
185187 {
186188 struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
189+ unsigned long flags;
187190
188− raw_spin_lock_bh(&b->lock);
191+ raw_spin_lock_irqsave(&b->lock, flags);
189192 RCU_INIT_POINTER(SDATA(selem)->smap, smap);
190193 hlist_add_head_rcu(&selem->map_node, &b->list);
191− raw_spin_unlock_bh(&b->lock);
194+ raw_spin_unlock_irqrestore(&b->lock, flags);
192195 }
193196
194197 void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
@@ -224,16 +227,18 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, · 29 unmodified lines
224227
225228 sdata = SDATA(selem);
226229 if (cacheit_lockit) {
230+ unsigned long flags;
231+
227232 /* spinlock is needed to avoid racing with the
228233 * parallel delete. Otherwise, publishing an already
229234 * deleted sdata to the cache will become a use-after-free
230235 * problem in the next bpf_local_storage_lookup().
231236 */
232− raw_spin_lock_bh(&local_storage->lock);
237+ raw_spin_lock_irqsave(&local_storage->lock, flags);
233238 if (selem_linked_to_storage(selem))
234239 rcu_assign_pointer(local_storage->cache[smap->cache_idx],
235240 sdata);
236− raw_spin_unlock_bh(&local_storage->lock);
241+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
237242 }
238243
239244 return sdata;
@@ -327,6 +332,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 87 unmodified lines
327332 struct bpf_local_storage_data *old_sdata = NULL;
328333 struct bpf_local_storage_elem *selem;
329334 struct bpf_local_storage *local_storage;
335+ unsigned long flags;
330336 int err;
331337
332338 /* BPF_EXIST and BPF_NOEXIST cannot be both set */
@@ -374,7 +380,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 41 unmodified lines
374380 }
375381 }
376382
377− raw_spin_lock_bh(&local_storage->lock);
383+ raw_spin_lock_irqsave(&local_storage->lock, flags);
378384
379385 /* Recheck local_storage->list under local_storage->lock */
380386 if (unlikely(hlist_empty(&local_storage->list))) {
@@ -428,11 +434,11 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 47 unmodified lines
428434 }
429435
430436 unlock:
431− raw_spin_unlock_bh(&local_storage->lock);
437+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
432438 return SDATA(selem);
433439
434440 unlock_err:
435− raw_spin_unlock_bh(&local_storage->lock);
441+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
436442 return ERR_PTR(err);
437443 }
438444
kernel/bpf/bpf_lsm.c+0−4
@@ -115,10 +115,6 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 114 unmodified lines
115115 return &bpf_spin_lock_proto;
116116 case BPF_FUNC_spin_unlock:
117117 return &bpf_spin_unlock_proto;
118− case BPF_FUNC_task_storage_get:
119− return &bpf_task_storage_get_proto;
120− case BPF_FUNC_task_storage_delete:
121− return &bpf_task_storage_delete_proto;
122118 case BPF_FUNC_bprm_opts_set:
123119 return &bpf_bprm_opts_set_proto;
124120 case BPF_FUNC_ima_inode_hash:
kernel/bpf/bpf_task_storage.c+11−32
@@ -15,7 +15,6 @@ · 14 unmodified lines
1515 #include <linux/bpf_local_storage.h>
1616 #include <linux/filter.h>
1717 #include <uapi/linux/btf.h>
18−#include <linux/bpf_lsm.h>
1918 #include <linux/btf_ids.h>
2019 #include <linux/fdtable.h>
2120
@@ -24,12 +23,8 @@ DEFINE_BPF_STORAGE_CACHE(task_cache); · 2 unmodified lines
2423 static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
2524 {
2625 struct task_struct *task = owner;
27− struct bpf_storage_blob *bsb;
2826
29− bsb = bpf_task(task);
30− if (!bsb)
31− return NULL;
32− return &bsb->storage;
27+ return &task->bpf_storage;
3328 }
3429
3530 static struct bpf_local_storage_data *
@@ -38,13 +33,8 @@ task_storage_lookup(struct task_struct *task, struct bpf_map *map, · 2 unmodified lines
3833 {
3934 struct bpf_local_storage *task_storage;
4035 struct bpf_local_storage_map *smap;
41− struct bpf_storage_blob *bsb;
42−
43− bsb = bpf_task(task);
44− if (!bsb)
45− return NULL;
4636
47− task_storage = rcu_dereference(bsb->storage);
37+ task_storage = rcu_dereference(task->bpf_storage);
4838 if (!task_storage)
4939 return NULL;
5040
@@ -57,16 +47,12 @@ void bpf_task_storage_free(struct task_struct *task) · 6 unmodified lines
5747 struct bpf_local_storage_elem *selem;
5848 struct bpf_local_storage *local_storage;
5949 bool free_task_storage = false;
60− struct bpf_storage_blob *bsb;
6150 struct hlist_node *n;
62−
63− bsb = bpf_task(task);
64− if (!bsb)
65− return;
51+ unsigned long flags;
6652
6753 rcu_read_lock();
6854
69− local_storage = rcu_dereference(bsb->storage);
55+ local_storage = rcu_dereference(task->bpf_storage);
7056 if (!local_storage) {
7157 rcu_read_unlock();
7258 return;
@@ -81,7 +67,7 @@ void bpf_task_storage_free(struct task_struct *task) · 8 unmodified lines
8167 * when unlinking elem from the local_storage->list and
8268 * the map's bucket->list.
8369 */
84− raw_spin_lock_bh(&local_storage->lock);
70+ raw_spin_lock_irqsave(&local_storage->lock, flags);
8571 hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
8672 /* Always unlink from map before unlinking from
8773 * local_storage.
@@ -90,7 +76,7 @@ void bpf_task_storage_free(struct task_struct *task) · 2 unmodified lines
9076 free_task_storage = bpf_selem_unlink_storage_nolock(
9177 local_storage, selem, false);
9278 }
93− raw_spin_unlock_bh(&local_storage->lock);
79+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
9480 rcu_read_unlock();
9581
9682 /* free_task_storage should always be true as long as
@@ -150,7 +136,7 @@ static int bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, · 53 unmodified lines
150136 */
151137 WARN_ON_ONCE(!rcu_read_lock_held());
152138 task = pid_task(pid, PIDTYPE_PID);
153− if (!task || !task_storage_ptr(task)) {
139+ if (!task) {
154140 err = -ENOENT;
155141 goto out;
156142 }
@@ -213,23 +199,16 @@ BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, · 56 unmodified lines
213199 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
214200 return (unsigned long)NULL;
215201
216− /* explicitly check that the task_storage_ptr is not
217− * NULL as task_storage_lookup returns NULL in this case and
218− * bpf_local_storage_update expects the owner to have a
219− * valid storage pointer.
220− */
221− if (!task || !task_storage_ptr(task))
202+ if (!task)
222203 return (unsigned long)NULL;
223204
224205 sdata = task_storage_lookup(task, map, true);
225206 if (sdata)
226207 return (unsigned long)sdata->data;
227208
228− /* This helper must only be called from places where the lifetime of the task
229− * is guaranteed. Either by being refcounted or by being protected
230− * by an RCU read-side critical section.
231− */
232− if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
209+ /* only allocate new storage, when the task is refcounted */
210+ if (refcount_read(&task->usage) &&
211+ (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {
233212 sdata = bpf_local_storage_update(
234213 task, (struct bpf_local_storage_map *)map, value,
235214 BPF_NOEXIST);
kernel/fork.c+5−0
@@ -96,6 +96,7 @@ · 95 unmodified lines
9696 #include <linux/kasan.h>
9797 #include <linux/scs.h>
9898 #include <linux/io_uring.h>
99+#include <linux/bpf.h>
99100
100101 #include <asm/pgalloc.h>
101102 #include <linux/uaccess.h>
@@ -734,6 +735,7 @@ void __put_task_struct(struct task_struct *tsk) · 632 unmodified lines
734735 cgroup_free(tsk);
735736 task_numa_free(tsk, true);
736737 security_task_free(tsk);
738+ bpf_task_storage_free(tsk);
737739 exit_creds(tsk);
738740 delayacct_tsk_free(tsk);
739741 put_signal_struct(tsk->signal);
@@ -2062,6 +2064,9 @@ static __latent_entropy struct task_struct *copy_process( · 1322 unmodified lines
20622064 p->sequential_io = 0;
20632065 p->sequential_io_avg = 0;
20642066 #endif
2067+#ifdef CONFIG_BPF_SYSCALL
2068+ RCU_INIT_POINTER(p->bpf_storage, NULL);
2069+#endif
20652070
20662071 /* Perform scheduler related setup. Assign this task to a CPU. */
20672072 retval = sched_fork(clone_flags, p);
kernel/trace/bpf_trace.c+4−0
@@ -1367,6 +1367,10 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1366 unmodified lines
13671367 return &bpf_per_cpu_ptr_proto;
13681368 case BPF_FUNC_this_cpu_ptr:
13691369 return &bpf_this_cpu_ptr_proto;
1370+ case BPF_FUNC_task_storage_get:
1371+ return &bpf_task_storage_get_proto;
1372+ case BPF_FUNC_task_storage_delete:
1373+ return &bpf_task_storage_delete_proto;
13701374 default:
13711375 return NULL;
13721376 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t