bpf: Enable task local storage for tracing programs

To access per-task data, BPF programs usually creates a hash table with pid as the key. This is not ideal because: 1. The user need to estimate the proper size of the hash table, which may be inaccurate; 2. Big hash tables are slow; 3. To clean up the data properly during task terminations, the user need to write extra logic. Task local storage overcomes these issues and offers a better option for these per-task data. Task local storage is only available to BPF_LSM. Now enable it for tracing programs. Unlike LSM programs, tracing programs can be called in IRQ contexts. Helpers that access task local storage are updated to use raw_spin_lock_irqsave() instead of raw_spin_lock_bh(). Tracing programs can attach to functions on the task free path, e.g. exit_creds(). To avoid allocating task local storage after bpf_task_storage_free(). bpf_task_storage_get() is updated to not allocate new storage when the task is not refcounted (task->usage == 0).

Song Liu committed 5 years ago

Commit a10787e1 parent 9c8f21e10 files · +51 −72Browse files

Unified · Split

10 files

include/linux/bpf.h +7 −0
include/linux/bpf_lsm.h +0 −22
include/linux/bpf_types.h +1 −1
include/linux/sched.h +5 −0
kernel/bpf/Makefile +1 −2
kernel/bpf/bpf_local_storage.c +17 −11
kernel/bpf/bpf_lsm.c +0 −4
kernel/bpf/bpf_task_storage.c +11 −32
kernel/fork.c +5 −0
kernel/trace/bpf_trace.c +4 −0

include/linux/bpf.h+7−0

@@ -1499,6 +1499,7 @@ struct bpf_prog *bpf_prog_by_id(u32 id); · 1498 unmodified lines
1499	1499	struct bpf_link *bpf_link_by_id(u32 id);
1500	1500
1501	1501	const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
	1502	+void bpf_task_storage_free(struct task_struct *task);
1502	1503	#else /* !CONFIG_BPF_SYSCALL */
1503	1504	static inline struct bpf_prog *bpf_prog_get(u32 ufd)
1504	1505	{
@@ -1684,6 +1685,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 179 unmodified lines
1684	1685	{
1685	1686	return NULL;
1686	1687	}
	1688	+
	1689	+static inline void bpf_task_storage_free(struct task_struct *task)
	1690	+{
	1691	+}
1687	1692	#endif /* CONFIG_BPF_SYSCALL */
1688	1693
1689	1694	void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
@@ -1886,6 +1891,8 @@ extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; · 196 unmodified lines
1886	1891	extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
1887	1892	extern const struct bpf_func_proto bpf_sock_from_file_proto;
1888	1893	extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
	1894	+extern const struct bpf_func_proto bpf_task_storage_get_proto;
	1895	+extern const struct bpf_func_proto bpf_task_storage_delete_proto;
1889	1896
1890	1897	const struct bpf_func_proto *bpf_tracing_func_proto(
1891	1898	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_lsm.h+0−22

@@ -38,21 +38,9 @@ static inline struct bpf_storage_blob *bpf_inode( · 37 unmodified lines
38	38	return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;
39	39	}
40	40
41		−static inline struct bpf_storage_blob *bpf_task(
42		− const struct task_struct *task)
43		−{
44		− if (unlikely(!task->security))
45		− return NULL;
46		−
47		− return task->security + bpf_lsm_blob_sizes.lbs_task;
48		−}
49		−
50	41	extern const struct bpf_func_proto bpf_inode_storage_get_proto;
51	42	extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
52		−extern const struct bpf_func_proto bpf_task_storage_get_proto;
53		−extern const struct bpf_func_proto bpf_task_storage_delete_proto;
54	43	void bpf_inode_storage_free(struct inode *inode);
55		−void bpf_task_storage_free(struct task_struct *task);
56	44
57	45	#else /* !CONFIG_BPF_LSM */
58	46
@@ -73,20 +61,10 @@ static inline struct bpf_storage_blob *bpf_inode( · 14 unmodified lines
73	61	return NULL;
74	62	}
75	63
76		−static inline struct bpf_storage_blob *bpf_task(
77		− const struct task_struct *task)
78		−{
79		− return NULL;
80		−}
81		−
82	64	static inline void bpf_inode_storage_free(struct inode *inode)
83	65	{
84	66	}
85	67
86		−static inline void bpf_task_storage_free(struct task_struct *task)
87		−{
88		−}
89		−
90	68	#endif /* CONFIG_BPF_LSM */
91	69
92	70	#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_types.h+1−1

@@ -109,8 +109,8 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) · 108 unmodified lines
109	109	#endif
110	110	#ifdef CONFIG_BPF_LSM
111	111	BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)
112		−BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)
113	112	#endif
	113	+BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)
114	114	BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
115	115	#if defined(CONFIG_XDP_SOCKETS)
116	116	BPF_MAP_TYPE(BPF_MAP_TYPE_XSKMAP, xsk_map_ops)

include/linux/sched.h+5−0

@@ -42,6 +42,7 @@ struct audit_context; · 41 unmodified lines
42	42	struct backing_dev_info;
43	43	struct bio_list;
44	44	struct blk_plug;
	45	+struct bpf_local_storage;
45	46	struct capture_control;
46	47	struct cfs_rq;
47	48	struct fs_struct;
@@ -1348,6 +1349,10 @@ struct task_struct { · 1300 unmodified lines
1348	1349	/* Used by LSM modules for access restriction: */
1349	1350	void *security;
1350	1351	#endif
	1352	+#ifdef CONFIG_BPF_SYSCALL
	1353	+ /* Used by BPF task local storage */
	1354	+ struct bpf_local_storage __rcu *bpf_storage;
	1355	+#endif
1351	1356
1352	1357	#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
1353	1358	unsigned long lowest_stack;

kernel/bpf/Makefile+1−2

@@ -9,8 +9,8 @@ CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) · 8 unmodified lines
9	9	obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
10	10	obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
11	11	obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
	12	+obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
12	13	obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
13		−obj-${CONFIG_BPF_LSM} += bpf_task_storage.o
14	14	obj-$(CONFIG_BPF_SYSCALL) += disasm.o
15	15	obj-$(CONFIG_BPF_JIT) += trampoline.o
16	16	obj-$(CONFIG_BPF_SYSCALL) += btf.o
@@ -18,7 +18,6 @@ obj-$(CONFIG_BPF_JIT) += dispatcher.o · 1 unmodified line
18	18	ifeq ($(CONFIG_NET),y)
19	19	obj-$(CONFIG_BPF_SYSCALL) += devmap.o
20	20	obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
21		−obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o
22	21	obj-$(CONFIG_BPF_SYSCALL) += offload.o
23	22	obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
24	23	endif

kernel/bpf/bpf_local_storage.c+17−11

@@ -140,17 +140,18 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem) · 139 unmodified lines
140	140	{
141	141	struct bpf_local_storage *local_storage;
142	142	bool free_local_storage = false;
	143	+ unsigned long flags;
143	144
144	145	if (unlikely(!selem_linked_to_storage(selem)))
145	146	/* selem has already been unlinked from sk */
146	147	return;
147	148
148	149	local_storage = rcu_dereference(selem->local_storage);
149		− raw_spin_lock_bh(&local_storage->lock);
	150	+ raw_spin_lock_irqsave(&local_storage->lock, flags);
150	151	if (likely(selem_linked_to_storage(selem)))
151	152	free_local_storage = bpf_selem_unlink_storage_nolock(
152	153	local_storage, selem, true);
153		− raw_spin_unlock_bh(&local_storage->lock);
	154	+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
154	155
155	156	if (free_local_storage)
156	157	kfree_rcu(local_storage, rcu);
@@ -167,6 +168,7 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 10 unmodified lines
167	168	{
168	169	struct bpf_local_storage_map *smap;
169	170	struct bpf_local_storage_map_bucket *b;
	171	+ unsigned long flags;
170	172
171	173	if (unlikely(!selem_linked_to_map(selem)))
172	174	/* selem has already be unlinked from smap */
@@ -174,21 +176,22 @@ void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem) · 1 unmodified line
174	176
175	177	smap = rcu_dereference(SDATA(selem)->smap);
176	178	b = select_bucket(smap, selem);
177		− raw_spin_lock_bh(&b->lock);
	179	+ raw_spin_lock_irqsave(&b->lock, flags);
178	180	if (likely(selem_linked_to_map(selem)))
179	181	hlist_del_init_rcu(&selem->map_node);
180		− raw_spin_unlock_bh(&b->lock);
	182	+ raw_spin_unlock_irqrestore(&b->lock, flags);
181	183	}
182	184
183	185	void bpf_selem_link_map(struct bpf_local_storage_map *smap,
184	186	struct bpf_local_storage_elem *selem)
185	187	{
186	188	struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
	189	+ unsigned long flags;
187	190
188		− raw_spin_lock_bh(&b->lock);
	191	+ raw_spin_lock_irqsave(&b->lock, flags);
189	192	RCU_INIT_POINTER(SDATA(selem)->smap, smap);
190	193	hlist_add_head_rcu(&selem->map_node, &b->list);
191		− raw_spin_unlock_bh(&b->lock);
	194	+ raw_spin_unlock_irqrestore(&b->lock, flags);
192	195	}
193	196
194	197	void bpf_selem_unlink(struct bpf_local_storage_elem *selem)
@@ -224,16 +227,18 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage, · 29 unmodified lines
224	227
225	228	sdata = SDATA(selem);
226	229	if (cacheit_lockit) {
	230	+ unsigned long flags;
	231	+
227	232	/* spinlock is needed to avoid racing with the
228	233	* parallel delete. Otherwise, publishing an already
229	234	* deleted sdata to the cache will become a use-after-free
230	235	* problem in the next bpf_local_storage_lookup().
231	236	*/
232		− raw_spin_lock_bh(&local_storage->lock);
	237	+ raw_spin_lock_irqsave(&local_storage->lock, flags);
233	238	if (selem_linked_to_storage(selem))
234	239	rcu_assign_pointer(local_storage->cache[smap->cache_idx],
235	240	sdata);
236		− raw_spin_unlock_bh(&local_storage->lock);
	241	+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
237	242	}
238	243
239	244	return sdata;
@@ -327,6 +332,7 @@ bpf_local_storage_update(void owner, struct bpf_local_storage_map smap, · 87 unmodified lines
327	332	struct bpf_local_storage_data *old_sdata = NULL;
328	333	struct bpf_local_storage_elem *selem;
329	334	struct bpf_local_storage *local_storage;
	335	+ unsigned long flags;
330	336	int err;
331	337
332	338	/* BPF_EXIST and BPF_NOEXIST cannot be both set */
@@ -374,7 +380,7 @@ bpf_local_storage_update(void owner, struct bpf_local_storage_map smap, · 41 unmodified lines
374	380	}
375	381	}
376	382
377		− raw_spin_lock_bh(&local_storage->lock);
	383	+ raw_spin_lock_irqsave(&local_storage->lock, flags);
378	384
379	385	/* Recheck local_storage->list under local_storage->lock */
380	386	if (unlikely(hlist_empty(&local_storage->list))) {
@@ -428,11 +434,11 @@ bpf_local_storage_update(void owner, struct bpf_local_storage_map smap, · 47 unmodified lines
428	434	}
429	435
430	436	unlock:
431		− raw_spin_unlock_bh(&local_storage->lock);
	437	+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
432	438	return SDATA(selem);
433	439
434	440	unlock_err:
435		− raw_spin_unlock_bh(&local_storage->lock);
	441	+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
436	442	return ERR_PTR(err);
437	443	}
438	444

kernel/bpf/bpf_lsm.c+0−4

@@ -115,10 +115,6 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 114 unmodified lines
115	115	return &bpf_spin_lock_proto;
116	116	case BPF_FUNC_spin_unlock:
117	117	return &bpf_spin_unlock_proto;
118		− case BPF_FUNC_task_storage_get:
119		− return &bpf_task_storage_get_proto;
120		− case BPF_FUNC_task_storage_delete:
121		− return &bpf_task_storage_delete_proto;
122	118	case BPF_FUNC_bprm_opts_set:
123	119	return &bpf_bprm_opts_set_proto;
124	120	case BPF_FUNC_ima_inode_hash:

kernel/bpf/bpf_task_storage.c+11−32

@@ -15,7 +15,6 @@ · 14 unmodified lines
15	15	#include <linux/bpf_local_storage.h>
16	16	#include <linux/filter.h>
17	17	#include <uapi/linux/btf.h>
18		−#include <linux/bpf_lsm.h>
19	18	#include <linux/btf_ids.h>
20	19	#include <linux/fdtable.h>
21	20
@@ -24,12 +23,8 @@ DEFINE_BPF_STORAGE_CACHE(task_cache); · 2 unmodified lines
24	23	static struct bpf_local_storage __rcu *task_storage_ptr(void owner)
25	24	{
26	25	struct task_struct *task = owner;
27		− struct bpf_storage_blob *bsb;
28	26
29		− bsb = bpf_task(task);
30		− if (!bsb)
31		− return NULL;
32		− return &bsb->storage;
	27	+ return &task->bpf_storage;
33	28	}
34	29
35	30	static struct bpf_local_storage_data *
@@ -38,13 +33,8 @@ task_storage_lookup(struct task_struct task, struct bpf_map map, · 2 unmodified lines
38	33	{
39	34	struct bpf_local_storage *task_storage;
40	35	struct bpf_local_storage_map *smap;
41		− struct bpf_storage_blob *bsb;
42		−
43		− bsb = bpf_task(task);
44		− if (!bsb)
45		− return NULL;
46	36
47		− task_storage = rcu_dereference(bsb->storage);
	37	+ task_storage = rcu_dereference(task->bpf_storage);
48	38	if (!task_storage)
49	39	return NULL;
50	40
@@ -57,16 +47,12 @@ void bpf_task_storage_free(struct task_struct *task) · 6 unmodified lines
57	47	struct bpf_local_storage_elem *selem;
58	48	struct bpf_local_storage *local_storage;
59	49	bool free_task_storage = false;
60		− struct bpf_storage_blob *bsb;
61	50	struct hlist_node *n;
62		−
63		− bsb = bpf_task(task);
64		− if (!bsb)
65		− return;
	51	+ unsigned long flags;
66	52
67	53	rcu_read_lock();
68	54
69		− local_storage = rcu_dereference(bsb->storage);
	55	+ local_storage = rcu_dereference(task->bpf_storage);
70	56	if (!local_storage) {
71	57	rcu_read_unlock();
72	58	return;
@@ -81,7 +67,7 @@ void bpf_task_storage_free(struct task_struct *task) · 8 unmodified lines
81	67	* when unlinking elem from the local_storage->list and
82	68	* the map's bucket->list.
83	69	*/
84		− raw_spin_lock_bh(&local_storage->lock);
	70	+ raw_spin_lock_irqsave(&local_storage->lock, flags);
85	71	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
86	72	/* Always unlink from map before unlinking from
87	73	* local_storage.
@@ -90,7 +76,7 @@ void bpf_task_storage_free(struct task_struct *task) · 2 unmodified lines
90	76	free_task_storage = bpf_selem_unlink_storage_nolock(
91	77	local_storage, selem, false);
92	78	}
93		− raw_spin_unlock_bh(&local_storage->lock);
	79	+ raw_spin_unlock_irqrestore(&local_storage->lock, flags);
94	80	rcu_read_unlock();
95	81
96	82	/* free_task_storage should always be true as long as
@@ -150,7 +136,7 @@ static int bpf_pid_task_storage_update_elem(struct bpf_map map, void key, · 53 unmodified lines
150	136	*/
151	137	WARN_ON_ONCE(!rcu_read_lock_held());
152	138	task = pid_task(pid, PIDTYPE_PID);
153		− if (!task \|\| !task_storage_ptr(task)) {
	139	+ if (!task) {
154	140	err = -ENOENT;
155	141	goto out;
156	142	}
@@ -213,23 +199,16 @@ BPF_CALL_4(bpf_task_storage_get, struct bpf_map , map, struct task_struct , · 56 unmodified lines
213	199	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
214	200	return (unsigned long)NULL;
215	201
216		− /* explicitly check that the task_storage_ptr is not
217		− * NULL as task_storage_lookup returns NULL in this case and
218		− * bpf_local_storage_update expects the owner to have a
219		− * valid storage pointer.
220		− */
221		− if (!task \|\| !task_storage_ptr(task))
	202	+ if (!task)
222	203	return (unsigned long)NULL;
223	204
224	205	sdata = task_storage_lookup(task, map, true);
225	206	if (sdata)
226	207	return (unsigned long)sdata->data;
227	208
228		− /* This helper must only be called from places where the lifetime of the task
229		− * is guaranteed. Either by being refcounted or by being protected
230		− * by an RCU read-side critical section.
231		− */
232		− if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
	209	+ /* only allocate new storage, when the task is refcounted */
	210	+ if (refcount_read(&task->usage) &&
	211	+ (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {
233	212	sdata = bpf_local_storage_update(
234	213	task, (struct bpf_local_storage_map *)map, value,
235	214	BPF_NOEXIST);

kernel/fork.c+5−0

@@ -96,6 +96,7 @@ · 95 unmodified lines
96	96	#include <linux/kasan.h>
97	97	#include <linux/scs.h>
98	98	#include <linux/io_uring.h>
	99	+#include <linux/bpf.h>
99	100
100	101	#include <asm/pgalloc.h>
101	102	#include <linux/uaccess.h>
@@ -734,6 +735,7 @@ void __put_task_struct(struct task_struct *tsk) · 632 unmodified lines
734	735	cgroup_free(tsk);
735	736	task_numa_free(tsk, true);
736	737	security_task_free(tsk);
	738	+ bpf_task_storage_free(tsk);
737	739	exit_creds(tsk);
738	740	delayacct_tsk_free(tsk);
739	741	put_signal_struct(tsk->signal);
@@ -2062,6 +2064,9 @@ static __latent_entropy struct task_struct *copy_process( · 1322 unmodified lines
2062	2064	p->sequential_io = 0;
2063	2065	p->sequential_io_avg = 0;
2064	2066	#endif
	2067	+#ifdef CONFIG_BPF_SYSCALL
	2068	+ RCU_INIT_POINTER(p->bpf_storage, NULL);
	2069	+#endif
2065	2070
2066	2071	/* Perform scheduler related setup. Assign this task to a CPU. */
2067	2072	retval = sched_fork(clone_flags, p);

kernel/trace/bpf_trace.c+4−0

@@ -1367,6 +1367,10 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1366 unmodified lines
1367	1367	return &bpf_per_cpu_ptr_proto;
1368	1368	case BPF_FUNC_this_cpu_ptr:
1369	1369	return &bpf_this_cpu_ptr_proto;
	1370	+ case BPF_FUNC_task_storage_get:
	1371	+ return &bpf_task_storage_get_proto;
	1372	+ case BPF_FUNC_task_storage_delete:
	1373	+ return &bpf_task_storage_delete_proto;
1370	1374	default:
1371	1375	return NULL;
1372	1376	}

bpf: Enable task local storage for tracing programs

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code