bpf: decouple the lifetime of cgroup_bpf from cgroup itself

Currently the lifetime of bpf programs attached to a cgroup is bound to the lifetime of the cgroup itself. It means that if a user forgets (or intentionally avoids) to detach a bpf program before removing the cgroup, it will stay attached up to the release of the cgroup. Since the cgroup can stay in the dying state (the state between being rmdir()'ed and being released) for a very long time, it leads to a waste of memory. Also, it blocks a possibility to implement the memcg-based memory accounting for bpf objects, because a circular reference dependency will occur. Charged memory pages are pinning the corresponding memory cgroup, and if the memory cgroup is pinning the attached bpf program, nothing will be ever released. A dying cgroup can not contain any processes, so the only chance for an attached bpf program to be executed is a live socket associated with the cgroup. So in order to release all bpf data early, let's count associated sockets using a new percpu refcounter. On cgroup removal the counter is transitioned to the atomic mode, and as soon as it reaches 0, all bpf programs are detached. Because cgroup_bpf_release() can block, it can't be called from the percpu ref counter callback directly, so instead an asynchronous work is scheduled. The reference counter is not socket specific, and can be used for any other types of programs, which can be executed from a cgroup-bpf hook outside of the process context, had such a need arise in the future.

Roman Gushchin committed 7 years ago

Commit 4bfc0bb1 parent 37b54ae4 files · +72 −9Browse files

Unified · Split

4 files

include/linux/bpf-cgroup.h +9 −2
include/linux/cgroup.h +18 −0
kernel/bpf/cgroup.c +37 −4
kernel/cgroup/cgroup.c +8 −3

include/linux/bpf-cgroup.h+9−2

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/errno.h>
7	7	#include <linux/jump_label.h>
8	8	#include <linux/percpu.h>
	9	+#include <linux/percpu-refcount.h>
9	10	#include <linux/rbtree.h>
10	11	#include <uapi/linux/bpf.h>
11	12
@@ -72,10 +73,16 @@ struct cgroup_bpf { · 60 unmodified lines
72	73
73	74	/* temp storage for effective prog array used by prog_attach/detach */
74	75	struct bpf_prog_array __rcu *inactive;
	76	+
	77	+ /* reference counter used to detach bpf programs after cgroup removal */
	78	+ struct percpu_ref refcnt;
	79	+
	80	+ /* cgroup_bpf is released using a work queue */
	81	+ struct work_struct release_work;
75	82	};
76	83
77		−void cgroup_bpf_put(struct cgroup *cgrp);
78	84	int cgroup_bpf_inherit(struct cgroup *cgrp);
	85	+void cgroup_bpf_offline(struct cgroup *cgrp);
79	86
80	87	int __cgroup_bpf_attach(struct cgroup cgrp, struct bpf_prog prog,
81	88	enum bpf_attach_type type, u32 flags);
@@ -283,8 +290,8 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, · 201 unmodified lines
283	290
284	291	struct bpf_prog;
285	292	struct cgroup_bpf {};
286		−static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
287	293	static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
	294	+static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
288	295
289	296	static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
290	297	enum bpf_prog_type ptype,

include/linux/cgroup.h+18−0

@@ -924,4 +924,22 @@ static inline bool cgroup_task_frozen(struct task_struct *task) · 923 unmodified lines
924	924
925	925	#endif /* !CONFIG_CGROUPS */
926	926
	927	+#ifdef CONFIG_CGROUP_BPF
	928	+static inline void cgroup_bpf_get(struct cgroup *cgrp)
	929	+{
	930	+ percpu_ref_get(&cgrp->bpf.refcnt);
	931	+}
	932	+
	933	+static inline void cgroup_bpf_put(struct cgroup *cgrp)
	934	+{
	935	+ percpu_ref_put(&cgrp->bpf.refcnt);
	936	+}
	937	+
	938	+#else /* CONFIG_CGROUP_BPF */
	939	+
	940	+static inline void cgroup_bpf_get(struct cgroup *cgrp) {}
	941	+static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
	942	+
	943	+#endif /* CONFIG_CGROUP_BPF */
	944	+
927	945	#endif /* _LINUX_CGROUP_H */

kernel/bpf/cgroup.c+37−4

@@ -22,12 +22,21 @@ · 21 unmodified lines
22	22	DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
23	23	EXPORT_SYMBOL(cgroup_bpf_enabled_key);
24	24
	25	+void cgroup_bpf_offline(struct cgroup *cgrp)
	26	+{
	27	+ cgroup_get(cgrp);
	28	+ percpu_ref_kill(&cgrp->bpf.refcnt);
	29	+}
	30	+
25	31	/**
26		− * cgroup_bpf_put() - put references of all bpf programs
27		− * @cgrp: the cgroup to modify
	32	+ * cgroup_bpf_release() - put references of all bpf programs and
	33	+ * release all cgroup bpf data
	34	+ * @work: work structure embedded into the cgroup to modify
28	35	*/
29		−void cgroup_bpf_put(struct cgroup *cgrp)
	36	+static void cgroup_bpf_release(struct work_struct *work)
30	37	{
	38	+ struct cgroup *cgrp = container_of(work, struct cgroup,
	39	+ bpf.release_work);
31	40	enum bpf_cgroup_storage_type stype;
32	41	unsigned int type;
33	42
@@ -47,6 +56,22 @@ void cgroup_bpf_put(struct cgroup *cgrp) · 13 unmodified lines
47	56	}
48	57	bpf_prog_array_free(cgrp->bpf.effective[type]);
49	58	}
	59	+
	60	+ percpu_ref_exit(&cgrp->bpf.refcnt);
	61	+ cgroup_put(cgrp);
	62	+}
	63	+
	64	+/**
	65	+ * cgroup_bpf_release_fn() - callback used to schedule releasing
	66	+ * of bpf cgroup data
	67	+ * @ref: percpu ref counter structure
	68	+ */
	69	+static void cgroup_bpf_release_fn(struct percpu_ref *ref)
	70	+{
	71	+ struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt);
	72	+
	73	+ INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);
	74	+ queue_work(system_wq, &cgrp->bpf.release_work);
50	75	}
51	76
52	77	/* count number of elements in the list.
@@ -167,7 +192,12 @@ int cgroup_bpf_inherit(struct cgroup *cgrp) · 114 unmodified lines
167	192	*/
168	193	#define NR ARRAY_SIZE(cgrp->bpf.effective)
169	194	struct bpf_prog_array __rcu *arrays[NR] = {};
170		− int i;
	195	+ int ret, i;
	196	+
	197	+ ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,
	198	+ GFP_KERNEL);
	199	+ if (ret)
	200	+ return ret;
171	201
172	202	for (i = 0; i < NR; i++)
173	203	INIT_LIST_HEAD(&cgrp->bpf.progs[i]);
@@ -183,6 +213,9 @@ int cgroup_bpf_inherit(struct cgroup *cgrp) · 9 unmodified lines
183	213	cleanup:
184	214	for (i = 0; i < NR; i++)
185	215	bpf_prog_array_free(arrays[i]);
	216	+
	217	+ percpu_ref_exit(&cgrp->bpf.refcnt);
	218	+
186	219	return -ENOMEM;
187	220	}
188	221

kernel/cgroup/cgroup.c+8−3

@@ -4955,8 +4955,6 @@ static void css_release_work_fn(struct work_struct *work) · 4954 unmodified lines
4955	4955	if (cgrp->kn)
4956	4956	RCU_INIT_POINTER((void __rcu __force *)&cgrp->kn->priv,
4957	4957	NULL);
4958		−
4959		− cgroup_bpf_put(cgrp);
4960	4958	}
4961	4959
4962	4960	mutex_unlock(&cgroup_mutex);
@@ -5482,6 +5480,8 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) · 519 unmodified lines
5482	5480
5483	5481	cgroup1_check_for_release(parent);
5484	5482
	5483	+ cgroup_bpf_offline(cgrp);
	5484	+
5485	5485	/* put the base reference */
5486	5486	percpu_ref_kill(&cgrp->self.refcnt);
5487	5487
@@ -6221,6 +6221,7 @@ void cgroup_sk_alloc(struct sock_cgroup_data *skcd) · 733 unmodified lines
6221	6221	* Don't use cgroup_get_live().
6222	6222	*/
6223	6223	cgroup_get(sock_cgroup_ptr(skcd));
	6224	+ cgroup_bpf_get(sock_cgroup_ptr(skcd));
6224	6225	return;
6225	6226	}
6226	6227
@@ -6232,6 +6233,7 @@ void cgroup_sk_alloc(struct sock_cgroup_data *skcd) · 5 unmodified lines
6232	6233	cset = task_css_set(current);
6233	6234	if (likely(cgroup_tryget(cset->dfl_cgrp))) {
6234	6235	skcd->val = (unsigned long)cset->dfl_cgrp;
	6236	+ cgroup_bpf_get(cset->dfl_cgrp);
6235	6237	break;
6236	6238	}
6237	6239	cpu_relax();
@@ -6242,7 +6244,10 @@ void cgroup_sk_alloc(struct sock_cgroup_data *skcd) · 4 unmodified lines
6242	6244
6243	6245	void cgroup_sk_free(struct sock_cgroup_data *skcd)
6244	6246	{
6245		− cgroup_put(sock_cgroup_ptr(skcd));
	6247	+ struct cgroup *cgrp = sock_cgroup_ptr(skcd);
	6248	+
	6249	+ cgroup_bpf_put(cgrp);
	6250	+ cgroup_put(cgrp);
6246	6251	}
6247	6252
6248	6253	#endif /* CONFIG_SOCK_CGROUP_DATA */

bpf: decouple the lifetime of cgroup_bpf from cgroup itself

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code