bpf: per-cgroup lsm flavor

Allow attaching to lsm hooks in the cgroup context. Attaching to per-cgroup LSM works exactly like attaching to other per-cgroup hooks. New BPF_LSM_CGROUP is added to trigger new mode; the actual lsm hook we attach to is signaled via existing attach_btf_id. For the hooks that have 'struct socket' or 'struct sock' as its first argument, we use the cgroup associated with that socket. For the rest, we use 'current' cgroup (this is all on default hierarchy == v2 only). Note that for some hooks that work on 'struct sock' we still take the cgroup from 'current' because some of them work on the socket that hasn't been properly initialized yet. Behind the scenes, we allocate a shim program that is attached to the trampoline and runs cgroup effective BPF programs array. This shim has some rudimentary ref counting and can be shared between several programs attaching to the same lsm hook from different cgroups. Note that this patch bloats cgroup size because we add 211 cgroup_bpf_attach_type(s) for simplicity sake. This will be addressed in the subsequent patch. Also note that we only add non-sleepable flavor for now. To enable sleepable use-cases, bpf_prog_run_array_cg has to grab trace rcu, shim programs have to be freed via trace rcu, cgroup_bpf.effective should be also trace-rcu-managed + maybe some other changes that I'm not aware of.

Stanislav Fomichev committed 4 years ago

Commit 69fd3371 parent 004421415 files · +498 −20Browse files

Unified · Split

15 files

arch/x86/net/bpf_jit_comp.c +16 −8
include/linux/bpf-cgroup-defs.h +8 −0
include/linux/bpf-cgroup.h +7 −0
include/linux/bpf.h +24 −0
include/linux/bpf_lsm.h +13 −0
include/linux/btf_ids.h +2 −1
include/uapi/linux/bpf.h +1 −0
kernel/bpf/bpf_lsm.c +48 −0
kernel/bpf/btf.c +11 −0
kernel/bpf/cgroup.c +125 −11
kernel/bpf/core.c +2 −0
kernel/bpf/syscall.c +10 −0
kernel/bpf/trampoline.c +198 −0
kernel/bpf/verifier.c +32 −0
tools/include/uapi/linux/bpf.h +1 −0

arch/x86/net/bpf_jit_comp.c+16−8

@@ -1770,6 +1770,10 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 1769 unmodified lines
1770	1770	struct bpf_tramp_link *l, int stack_size,
1771	1771	int run_ctx_off, bool save_ret)
1772	1772	{
	1773	+ void (exit)(struct bpf_prog prog, u64 start,
	1774	+ struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit;
	1775	+ u64 (enter)(struct bpf_prog prog,
	1776	+ struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter;
1773	1777	u8 prog = pprog;
1774	1778	u8 *jmp_insn;
1775	1779	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
@@ -1788,15 +1792,21 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 12 unmodified lines
1788	1792	*/
1789	1793	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);
1790	1794
	1795	+ if (p->aux->sleepable) {
	1796	+ enter = __bpf_prog_enter_sleepable;
	1797	+ exit = __bpf_prog_exit_sleepable;
	1798	+ } else if (p->expected_attach_type == BPF_LSM_CGROUP) {
	1799	+ enter = __bpf_prog_enter_lsm_cgroup;
	1800	+ exit = __bpf_prog_exit_lsm_cgroup;
	1801	+ }
	1802	+
1791	1803	/* arg1: mov rdi, progs[i] */
1792	1804	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1793	1805	/* arg2: lea rsi, [rbp - ctx_cookie_off] */
1794	1806	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
1795	1807
1796		− if (emit_call(&prog,
1797		− p->aux->sleepable ? __bpf_prog_enter_sleepable :
1798		− __bpf_prog_enter, prog))
1799		− return -EINVAL;
	1808	+ if (emit_call(&prog, enter, prog))
	1809	+ return -EINVAL;
1800	1810	/* remember prog start time returned by __bpf_prog_enter */
1801	1811	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
1802	1812
@@ -1840,10 +1850,8 @@ static int invoke_bpf_prog(const struct btf_func_model m, u8 *pprog, · 37 unmodified lines
1840	1850	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1841	1851	/* arg3: lea rdx, [rbp - run_ctx_off] */
1842	1852	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
1843		− if (emit_call(&prog,
1844		− p->aux->sleepable ? __bpf_prog_exit_sleepable :
1845		− __bpf_prog_exit, prog))
1846		− return -EINVAL;
	1853	+ if (emit_call(&prog, exit, prog))
	1854	+ return -EINVAL;
1847	1855
1848	1856	*pprog = prog;
1849	1857	return 0;

include/linux/bpf-cgroup-defs.h+8−0

@@ -10,6 +10,12 @@ · 9 unmodified lines
10	10
11	11	struct bpf_prog_array;
12	12
	13	+#ifdef CONFIG_BPF_LSM
	14	+#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */
	15	+#else
	16	+#define CGROUP_LSM_NUM 0
	17	+#endif
	18	+
13	19	enum cgroup_bpf_attach_type {
14	20	CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
15	21	CGROUP_INET_INGRESS = 0,
@@ -35,6 +41,8 @@ enum cgroup_bpf_attach_type { · 19 unmodified lines
35	41	CGROUP_INET4_GETSOCKNAME,
36	42	CGROUP_INET6_GETSOCKNAME,
37	43	CGROUP_INET_SOCK_RELEASE,
	44	+ CGROUP_LSM_START,
	45	+ CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1,
38	46	MAX_CGROUP_BPF_ATTACH_TYPE
39	47	};
40	48

include/linux/bpf-cgroup.h+7−0

@@ -23,6 +23,13 @@ struct ctl_table; · 22 unmodified lines
23	23	struct ctl_table_header;
24	24	struct task_struct;
25	25
	26	+unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
	27	+ const struct bpf_insn *insn);
	28	+unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
	29	+ const struct bpf_insn *insn);
	30	+unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
	31	+ const struct bpf_insn *insn);
	32	+
26	33	#ifdef CONFIG_CGROUP_BPF
27	34
28	35	#define CGROUP_ATYPE(type) \

include/linux/bpf.h+24−0

@@ -794,6 +794,10 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ · 793 unmodified lines
794	794	u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog prog, struct bpf_tramp_run_ctx run_ctx);
795	795	void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
796	796	struct bpf_tramp_run_ctx *run_ctx);
	797	+u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
	798	+ struct bpf_tramp_run_ctx *run_ctx);
	799	+void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
	800	+ struct bpf_tramp_run_ctx *run_ctx);
797	801	void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
798	802	void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
799	803
@@ -1060,6 +1064,7 @@ struct bpf_prog_aux { · 260 unmodified lines
1060	1064	struct user_struct *user;
1061	1065	u64 load_time; /* ns since boottime */
1062	1066	u32 verified_insns;
	1067	+ int cgroup_atype; /* enum cgroup_bpf_attach_type */
1063	1068	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1064	1069	char name[BPF_OBJ_NAME_LEN];
1065	1070	#ifdef CONFIG_SECURITY
@@ -1167,6 +1172,11 @@ struct bpf_tramp_link { · 101 unmodified lines
1167	1172	u64 cookie;
1168	1173	};
1169	1174
	1175	+struct bpf_shim_tramp_link {
	1176	+ struct bpf_tramp_link link;
	1177	+ struct bpf_trampoline *trampoline;
	1178	+};
	1179	+
1170	1180	struct bpf_tracing_link {
1171	1181	struct bpf_tramp_link link;
1172	1182	enum bpf_attach_type attach_type;
@@ -1245,6 +1255,9 @@ struct bpf_dummy_ops { · 72 unmodified lines
1245	1255	int bpf_struct_ops_test_run(struct bpf_prog prog, const union bpf_attr kattr,
1246	1256	union bpf_attr __user *uattr);
1247	1257	#endif
	1258	+int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
	1259	+ int cgroup_atype);
	1260	+void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1248	1261	#else
1249	1262	static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
1250	1263	{
@@ -1268,6 +1281,14 @@ static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, · 17 unmodified lines
1268	1281	{
1269	1282	return -EINVAL;
1270	1283	}
	1284	+static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
	1285	+ int cgroup_atype)
	1286	+{
	1287	+ return -EOPNOTSUPP;
	1288	+}
	1289	+static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
	1290	+{
	1291	+}
1271	1292	#endif
1272	1293
1273	1294	struct bpf_array {
@@ -2368,6 +2389,8 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; · 1094 unmodified lines
2368	2389	extern const struct bpf_func_proto bpf_find_vma_proto;
2369	2390	extern const struct bpf_func_proto bpf_loop_proto;
2370	2391	extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
	2392	+extern const struct bpf_func_proto bpf_set_retval_proto;
	2393	+extern const struct bpf_func_proto bpf_get_retval_proto;
2371	2394
2372	2395	const struct bpf_func_proto *tracing_prog_func_proto(
2373	2396	enum bpf_func_id func_id, const struct bpf_prog *prog);
@@ -2485,6 +2508,7 @@ int bpf_arch_text_invalidate(void *dst, size_t len); · 111 unmodified lines
2485	2508
2486	2509	struct btf_id_set;
2487	2510	bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
	2511	+int btf_id_set_index(const struct btf_id_set *set, u32 id);
2488	2512
2489	2513	#define MAX_BPRINTF_VARARGS 12
2490	2514

include/linux/bpf_lsm.h+13−0

@@ -42,6 +42,9 @@ extern const struct bpf_func_proto bpf_inode_storage_get_proto; · 41 unmodified lines
42	42	extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
43	43	void bpf_inode_storage_free(struct inode *inode);
44	44
	45	+int bpf_lsm_hook_idx(u32 btf_id);
	46	+void bpf_lsm_find_cgroup_shim(const struct bpf_prog prog, bpf_func_t bpf_func);
	47	+
45	48	#else /* !CONFIG_BPF_LSM */
46	49
47	50	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@@ -65,6 +68,16 @@ static inline void bpf_inode_storage_free(struct inode *inode) · 17 unmodified lines
65	68	{
66	69	}
67	70
	71	+static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
	72	+ bpf_func_t *bpf_func)
	73	+{
	74	+}
	75	+
	76	+static inline int bpf_lsm_hook_idx(u32 btf_id)
	77	+{
	78	+ return -EINVAL;
	79	+}
	80	+
68	81	#endif /* CONFIG_BPF_LSM */
69	82
70	83	#endif /* _LINUX_BPF_LSM_H */

include/linux/btf_ids.h+2−1

@@ -179,7 +179,8 @@ extern struct btf_id_set name; · 178 unmodified lines
179	179	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP, udp_sock) \
180	180	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) \
181	181	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UNIX, unix_sock) \
182		− BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock)
	182	+ BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock) \
	183	+ BTF_SOCK_TYPE(BTF_SOCK_TYPE_SOCKET, socket)
183	184
184	185	enum {
185	186	#define BTF_SOCK_TYPE(name, str) name,

include/uapi/linux/bpf.h+1−0

@@ -998,6 +998,7 @@ enum bpf_attach_type { · 997 unmodified lines
998	998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
999	999	BPF_PERF_EVENT,
1000	1000	BPF_TRACE_KPROBE_MULTI,
	1001	+ BPF_LSM_CGROUP,
1001	1002	__MAX_BPF_ATTACH_TYPE
1002	1003	};
1003	1004

kernel/bpf/bpf_lsm.c+48−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <linux/bpf_local_storage.h>
17	17	#include <linux/btf_ids.h>
18	18	#include <linux/ima.h>
	19	+#include <linux/bpf-cgroup.h>
19	20
20	21	/* For every LSM hook that allows attachment of BPF programs, declare a nop
21	22	* function where a BPF program can be attached.
@@ -35,6 +36,44 @@ BTF_SET_START(bpf_lsm_hooks) · 13 unmodified lines
35	36	#undef LSM_HOOK
36	37	BTF_SET_END(bpf_lsm_hooks)
37	38
	39	+/* List of LSM hooks that should operate on 'current' cgroup regardless
	40	+ * of function signature.
	41	+ */
	42	+BTF_SET_START(bpf_lsm_current_hooks)
	43	+/* operate on freshly allocated sk without any cgroup association */
	44	+BTF_ID(func, bpf_lsm_sk_alloc_security)
	45	+BTF_ID(func, bpf_lsm_sk_free_security)
	46	+BTF_SET_END(bpf_lsm_current_hooks)
	47	+
	48	+void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
	49	+ bpf_func_t *bpf_func)
	50	+{
	51	+ const struct btf_param *args;
	52	+
	53	+ if (btf_type_vlen(prog->aux->attach_func_proto) < 1 \|\|
	54	+ btf_id_set_contains(&bpf_lsm_current_hooks,
	55	+ prog->aux->attach_btf_id)) {
	56	+ *bpf_func = __cgroup_bpf_run_lsm_current;
	57	+ return;
	58	+ }
	59	+
	60	+ args = btf_params(prog->aux->attach_func_proto);
	61	+
	62	+#ifdef CONFIG_NET
	63	+ if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCKET])
	64	+ *bpf_func = __cgroup_bpf_run_lsm_socket;
	65	+ else if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCK])
	66	+ *bpf_func = __cgroup_bpf_run_lsm_sock;
	67	+ else
	68	+#endif
	69	+ *bpf_func = __cgroup_bpf_run_lsm_current;
	70	+}
	71	+
	72	+int bpf_lsm_hook_idx(u32 btf_id)
	73	+{
	74	+ return btf_id_set_index(&bpf_lsm_hooks, btf_id);
	75	+}
	76	+
38	77	int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
39	78	const struct bpf_prog *prog)
40	79	{
@@ -158,6 +197,15 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 117 unmodified lines
158	197	return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;
159	198	case BPF_FUNC_get_attach_cookie:
160	199	return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;
	200	+ case BPF_FUNC_get_local_storage:
	201	+ return prog->expected_attach_type == BPF_LSM_CGROUP ?
	202	+ &bpf_get_local_storage_proto : NULL;
	203	+ case BPF_FUNC_set_retval:
	204	+ return prog->expected_attach_type == BPF_LSM_CGROUP ?
	205	+ &bpf_set_retval_proto : NULL;
	206	+ case BPF_FUNC_get_retval:
	207	+ return prog->expected_attach_type == BPF_LSM_CGROUP ?
	208	+ &bpf_get_retval_proto : NULL;
161	209	default:
162	210	return tracing_prog_func_proto(func_id, prog);
163	211	}

kernel/bpf/btf.c+11−0

@@ -5363,6 +5363,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 5362 unmodified lines
5363	5363
5364	5364	if (arg == nr_args) {
5365	5365	switch (prog->expected_attach_type) {
	5366	+ case BPF_LSM_CGROUP:
5366	5367	case BPF_LSM_MAC:
5367	5368	case BPF_TRACE_FEXIT:
5368	5369	/* When LSM programs are attached to void LSM hooks
@@ -6842,6 +6843,16 @@ static int btf_id_cmp_func(const void a, const void b) · 1473 unmodified lines
6842	6843	return pa - pb;
6843	6844	}
6844	6845
	6846	+int btf_id_set_index(const struct btf_id_set *set, u32 id)
	6847	+{
	6848	+ const u32 *p;
	6849	+
	6850	+ p = bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func);
	6851	+ if (!p)
	6852	+ return -1;
	6853	+ return p - set->ids;
	6854	+}
	6855	+
6845	6856	bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
6846	6857	{
6847	6858	return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;

kernel/bpf/cgroup.c+125−11

@@ -14,6 +14,8 @@ · 13 unmodified lines
14	14	#include <linux/string.h>
15	15	#include <linux/bpf.h>
16	16	#include <linux/bpf-cgroup.h>
	17	+#include <linux/bpf_lsm.h>
	18	+#include <linux/bpf_verifier.h>
17	19	#include <net/sock.h>
18	20	#include <net/bpf_sk_storage.h>
19	21
@@ -61,6 +63,87 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp, · 41 unmodified lines
61	63	return run_ctx.retval;
62	64	}
63	65
	66	+unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
	67	+ const struct bpf_insn *insn)
	68	+{
	69	+ const struct bpf_prog *shim_prog;
	70	+ struct sock *sk;
	71	+ struct cgroup *cgrp;
	72	+ int ret = 0;
	73	+ u64 *args;
	74	+
	75	+ args = (u64 *)ctx;
	76	+ sk = (void *)(unsigned long)args[0];
	77	+ /shim_prog = container_of(insn, struct bpf_prog, insnsi);/
	78	+ shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
	79	+
	80	+ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
	81	+ if (likely(cgrp))
	82	+ ret = bpf_prog_run_array_cg(&cgrp->bpf,
	83	+ shim_prog->aux->cgroup_atype,
	84	+ ctx, bpf_prog_run, 0, NULL);
	85	+ return ret;
	86	+}
	87	+
	88	+unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
	89	+ const struct bpf_insn *insn)
	90	+{
	91	+ const struct bpf_prog *shim_prog;
	92	+ struct socket *sock;
	93	+ struct cgroup *cgrp;
	94	+ int ret = 0;
	95	+ u64 *args;
	96	+
	97	+ args = (u64 *)ctx;
	98	+ sock = (void *)(unsigned long)args[0];
	99	+ /shim_prog = container_of(insn, struct bpf_prog, insnsi);/
	100	+ shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
	101	+
	102	+ cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);
	103	+ if (likely(cgrp))
	104	+ ret = bpf_prog_run_array_cg(&cgrp->bpf,
	105	+ shim_prog->aux->cgroup_atype,
	106	+ ctx, bpf_prog_run, 0, NULL);
	107	+ return ret;
	108	+}
	109	+
	110	+unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
	111	+ const struct bpf_insn *insn)
	112	+{
	113	+ const struct bpf_prog *shim_prog;
	114	+ struct cgroup *cgrp;
	115	+ int ret = 0;
	116	+
	117	+ /shim_prog = container_of(insn, struct bpf_prog, insnsi);/
	118	+ shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
	119	+
	120	+ /* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */
	121	+ cgrp = task_dfl_cgroup(current);
	122	+ if (likely(cgrp))
	123	+ ret = bpf_prog_run_array_cg(&cgrp->bpf,
	124	+ shim_prog->aux->cgroup_atype,
	125	+ ctx, bpf_prog_run, 0, NULL);
	126	+ return ret;
	127	+}
	128	+
	129	+#ifdef CONFIG_BPF_LSM
	130	+static enum cgroup_bpf_attach_type
	131	+bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
	132	+{
	133	+ if (attach_type != BPF_LSM_CGROUP)
	134	+ return to_cgroup_bpf_attach_type(attach_type);
	135	+ return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id);
	136	+}
	137	+#else
	138	+static enum cgroup_bpf_attach_type
	139	+bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
	140	+{
	141	+ if (attach_type != BPF_LSM_CGROUP)
	142	+ return to_cgroup_bpf_attach_type(attach_type);
	143	+ return -EOPNOTSUPP;
	144	+}
	145	+#endif /* CONFIG_BPF_LSM */
	146	+
64	147	void cgroup_bpf_offline(struct cgroup *cgrp)
65	148	{
66	149	cgroup_get(cgrp);
@@ -163,10 +246,16 @@ static void cgroup_bpf_release(struct work_struct *work) · 96 unmodified lines
163	246
164	247	hlist_for_each_entry_safe(pl, pltmp, progs, node) {
165	248	hlist_del(&pl->node);
166		− if (pl->prog)
	249	+ if (pl->prog) {
	250	+ if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)
	251	+ bpf_trampoline_unlink_cgroup_shim(pl->prog);
167	252	bpf_prog_put(pl->prog);
168		− if (pl->link)
	253	+ }
	254	+ if (pl->link) {
	255	+ if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
	256	+ bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);
169	257	bpf_cgroup_link_auto_detach(pl->link);
	258	+ }
170	259	kfree(pl);
171	260	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
172	261	}
@@ -479,6 +568,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 306 unmodified lines
479	568	struct bpf_prog *old_prog = NULL;
480	569	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
481	570	struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
	571	+ struct bpf_prog *new_prog = prog ? : link->link.prog;
482	572	enum cgroup_bpf_attach_type atype;
483	573	struct bpf_prog_list *pl;
484	574	struct hlist_head *progs;
@@ -495,7 +585,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 10 unmodified lines
495	585	/* replace_prog implies BPF_F_REPLACE, and vice versa */
496	586	return -EINVAL;
497	587
498		− atype = to_cgroup_bpf_attach_type(type);
	588	+ atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
499	589	if (atype < 0)
500	590	return -EINVAL;
501	591
@@ -549,17 +639,30 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 47 unmodified lines
549	639	bpf_cgroup_storages_assign(pl->storage, storage);
550	640	cgrp->bpf.flags[atype] = saved_flags;
551	641
	642	+ if (type == BPF_LSM_CGROUP) {
	643	+ err = bpf_trampoline_link_cgroup_shim(new_prog, atype);
	644	+ if (err)
	645	+ goto cleanup;
	646	+ }
	647	+
552	648	err = update_effective_progs(cgrp, atype);
553	649	if (err)
554		− goto cleanup;
	650	+ goto cleanup_trampoline;
555	651
556		− if (old_prog)
	652	+ if (old_prog) {
	653	+ if (type == BPF_LSM_CGROUP)
	654	+ bpf_trampoline_unlink_cgroup_shim(old_prog);
557	655	bpf_prog_put(old_prog);
558		− else
	656	+ } else {
559	657	static_branch_inc(&cgroup_bpf_enabled_key[atype]);
	658	+ }
560	659	bpf_cgroup_storages_link(new_storage, cgrp, type);
561	660	return 0;
562	661
	662	+cleanup_trampoline:
	663	+ if (type == BPF_LSM_CGROUP)
	664	+ bpf_trampoline_unlink_cgroup_shim(new_prog);
	665	+
563	666	cleanup:
564	667	if (old_prog) {
565	668	pl->prog = old_prog;
@@ -651,7 +754,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, · 85 unmodified lines
651	754	struct hlist_head *progs;
652	755	bool found = false;
653	756
654		− atype = to_cgroup_bpf_attach_type(link->type);
	757	+ atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id);
655	758	if (atype < 0)
656	759	return -EINVAL;
657	760
@@ -803,9 +906,15 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 145 unmodified lines
803	906	struct bpf_prog *old_prog;
804	907	struct bpf_prog_list *pl;
805	908	struct hlist_head *progs;
	909	+ u32 attach_btf_id = 0;
806	910	u32 flags;
807	911
808		− atype = to_cgroup_bpf_attach_type(type);
	912	+ if (prog)
	913	+ attach_btf_id = prog->aux->attach_btf_id;
	914	+ if (link)
	915	+ attach_btf_id = link->link.prog->aux->attach_btf_id;
	916	+
	917	+ atype = bpf_cgroup_atype_find(type, attach_btf_id);
809	918	if (atype < 0)
810	919	return -EINVAL;
811	920
@@ -839,8 +948,11 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 27 unmodified lines
839	948	if (hlist_empty(progs))
840	949	/* last program was detached, reset flags to zero */
841	950	cgrp->bpf.flags[atype] = 0;
842		− if (old_prog)
	951	+ if (old_prog) {
	952	+ if (type == BPF_LSM_CGROUP)
	953	+ bpf_trampoline_unlink_cgroup_shim(old_prog);
843	954	bpf_prog_put(old_prog);
	955	+ }
844	956	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
845	957	return 0;
846	958	}
@@ -999,6 +1111,8 @@ static void bpf_cgroup_link_release(struct bpf_link *link) · 152 unmodified lines
999	1111
1000	1112	WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,
1001	1113	cg_link->type));
	1114	+ if (cg_link->type == BPF_LSM_CGROUP)
	1115	+ bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);
1002	1116
1003	1117	cg = cg_link->cgroup;
1004	1118	cg_link->cgroup = NULL;
@@ -1343,7 +1457,7 @@ BPF_CALL_0(bpf_get_retval) · 338 unmodified lines
1343	1457	return ctx->retval;
1344	1458	}
1345	1459
1346		−static const struct bpf_func_proto bpf_get_retval_proto = {
	1460	+const struct bpf_func_proto bpf_get_retval_proto = {
1347	1461	.func = bpf_get_retval,
1348	1462	.gpl_only = false,
1349	1463	.ret_type = RET_INTEGER,
@@ -1358,7 +1472,7 @@ BPF_CALL_1(bpf_set_retval, int, retval) · 8 unmodified lines
1358	1472	return 0;
1359	1473	}
1360	1474
1361		−static const struct bpf_func_proto bpf_set_retval_proto = {
	1475	+const struct bpf_func_proto bpf_set_retval_proto = {
1362	1476	.func = bpf_set_retval,
1363	1477	.gpl_only = false,
1364	1478	.ret_type = RET_INTEGER,

kernel/bpf/core.c+2−0

@@ -2666,6 +2666,8 @@ const struct bpf_func_proto bpf_get_local_storage_proto __weak; · 2665 unmodified lines
2666	2666	const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;
2667	2667	const struct bpf_func_proto bpf_snprintf_btf_proto __weak;
2668	2668	const struct bpf_func_proto bpf_seq_printf_btf_proto __weak;
	2669	+const struct bpf_func_proto bpf_set_retval_proto __weak;
	2670	+const struct bpf_func_proto bpf_get_retval_proto __weak;
2669	2671
2670	2672	const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
2671	2673	{

kernel/bpf/syscall.c+10−0

@@ -3416,6 +3416,8 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type) · 3415 unmodified lines
3416	3416	return BPF_PROG_TYPE_SK_LOOKUP;
3417	3417	case BPF_XDP:
3418	3418	return BPF_PROG_TYPE_XDP;
	3419	+ case BPF_LSM_CGROUP:
	3420	+ return BPF_PROG_TYPE_LSM;
3419	3421	default:
3420	3422	return BPF_PROG_TYPE_UNSPEC;
3421	3423	}
@@ -3469,6 +3471,11 @@ static int bpf_prog_attach(const union bpf_attr *attr) · 47 unmodified lines
3469	3471	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3470	3472	case BPF_PROG_TYPE_CGROUP_SYSCTL:
3471	3473	case BPF_PROG_TYPE_SOCK_OPS:
	3474	+ case BPF_PROG_TYPE_LSM:
	3475	+ if (ptype == BPF_PROG_TYPE_LSM &&
	3476	+ prog->expected_attach_type != BPF_LSM_CGROUP)
	3477	+ return -EINVAL;
	3478	+
3472	3479	ret = cgroup_bpf_prog_attach(attr, ptype, prog);
3473	3480	break;
3474	3481	default:
@@ -3506,6 +3513,7 @@ static int bpf_prog_detach(const union bpf_attr *attr) · 31 unmodified lines
3506	3513	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3507	3514	case BPF_PROG_TYPE_CGROUP_SYSCTL:
3508	3515	case BPF_PROG_TYPE_SOCK_OPS:
	3516	+ case BPF_PROG_TYPE_LSM:
3509	3517	return cgroup_bpf_prog_detach(attr, ptype);
3510	3518	default:
3511	3519	return -EINVAL;
@@ -4540,6 +4548,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) · 1028 unmodified lines
4540	4548	ret = bpf_raw_tp_link_attach(prog, NULL);
4541	4549	else if (prog->expected_attach_type == BPF_TRACE_ITER)
4542	4550	ret = bpf_iter_link_attach(attr, uattr, prog);
	4551	+ else if (prog->expected_attach_type == BPF_LSM_CGROUP)
	4552	+ ret = cgroup_bpf_link_attach(attr, prog);
4543	4553	else
4544	4554	ret = bpf_tracing_prog_attach(prog,
4545	4555	attr->link_create.target_fd,

kernel/bpf/trampoline.c+198−0

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	11	#include <linux/rcupdate_wait.h>
12	12	#include <linux/module.h>
13	13	#include <linux/static_call.h>
	14	+#include <linux/bpf_verifier.h>
	15	+#include <linux/bpf_lsm.h>
14	16
15	17	/* dummy _ops. The verifier will operate on target program's ops. */
16	18	const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -496,6 +498,177 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin · 479 unmodified lines
496	498	return err;
497	499	}
498	500
	501	+#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
	502	+static void bpf_shim_tramp_link_release(struct bpf_link *link)
	503	+{
	504	+ struct bpf_shim_tramp_link *shim_link =
	505	+ container_of(link, struct bpf_shim_tramp_link, link.link);
	506	+
	507	+ /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
	508	+ if (!shim_link->trampoline)
	509	+ return;
	510	+
	511	+ WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));
	512	+ bpf_trampoline_put(shim_link->trampoline);
	513	+}
	514	+
	515	+static void bpf_shim_tramp_link_dealloc(struct bpf_link *link)
	516	+{
	517	+ struct bpf_shim_tramp_link *shim_link =
	518	+ container_of(link, struct bpf_shim_tramp_link, link.link);
	519	+
	520	+ kfree(shim_link);
	521	+}
	522	+
	523	+static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
	524	+ .release = bpf_shim_tramp_link_release,
	525	+ .dealloc = bpf_shim_tramp_link_dealloc,
	526	+};
	527	+
	528	+static struct bpf_shim_tramp_link cgroup_shim_alloc(const struct bpf_prog prog,
	529	+ bpf_func_t bpf_func,
	530	+ int cgroup_atype)
	531	+{
	532	+ struct bpf_shim_tramp_link *shim_link = NULL;
	533	+ struct bpf_prog *p;
	534	+
	535	+ shim_link = kzalloc(sizeof(*shim_link), GFP_USER);
	536	+ if (!shim_link)
	537	+ return NULL;
	538	+
	539	+ p = bpf_prog_alloc(1, 0);
	540	+ if (!p) {
	541	+ kfree(shim_link);
	542	+ return NULL;
	543	+ }
	544	+
	545	+ p->jited = false;
	546	+ p->bpf_func = bpf_func;
	547	+
	548	+ p->aux->cgroup_atype = cgroup_atype;
	549	+ p->aux->attach_func_proto = prog->aux->attach_func_proto;
	550	+ p->aux->attach_btf_id = prog->aux->attach_btf_id;
	551	+ p->aux->attach_btf = prog->aux->attach_btf;
	552	+ btf_get(p->aux->attach_btf);
	553	+ p->type = BPF_PROG_TYPE_LSM;
	554	+ p->expected_attach_type = BPF_LSM_MAC;
	555	+ bpf_prog_inc(p);
	556	+ bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
	557	+ &bpf_shim_tramp_link_lops, p);
	558	+
	559	+ return shim_link;
	560	+}
	561	+
	562	+static struct bpf_shim_tramp_link cgroup_shim_find(struct bpf_trampoline tr,
	563	+ bpf_func_t bpf_func)
	564	+{
	565	+ struct bpf_tramp_link *link;
	566	+ int kind;
	567	+
	568	+ for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
	569	+ hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
	570	+ struct bpf_prog *p = link->link.prog;
	571	+
	572	+ if (p->bpf_func == bpf_func)
	573	+ return container_of(link, struct bpf_shim_tramp_link, link);
	574	+ }
	575	+ }
	576	+
	577	+ return NULL;
	578	+}
	579	+
	580	+int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
	581	+ int cgroup_atype)
	582	+{
	583	+ struct bpf_shim_tramp_link *shim_link = NULL;
	584	+ struct bpf_attach_target_info tgt_info = {};
	585	+ struct bpf_trampoline *tr;
	586	+ bpf_func_t bpf_func;
	587	+ u64 key;
	588	+ int err;
	589	+
	590	+ err = bpf_check_attach_target(NULL, prog, NULL,
	591	+ prog->aux->attach_btf_id,
	592	+ &tgt_info);
	593	+ if (err)
	594	+ return err;
	595	+
	596	+ key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
	597	+ prog->aux->attach_btf_id);
	598	+
	599	+ bpf_lsm_find_cgroup_shim(prog, &bpf_func);
	600	+ tr = bpf_trampoline_get(key, &tgt_info);
	601	+ if (!tr)
	602	+ return -ENOMEM;
	603	+
	604	+ mutex_lock(&tr->mutex);
	605	+
	606	+ shim_link = cgroup_shim_find(tr, bpf_func);
	607	+ if (shim_link) {
	608	+ /* Reusing existing shim attached by the other program. */
	609	+ bpf_link_inc(&shim_link->link.link);
	610	+
	611	+ mutex_unlock(&tr->mutex);
	612	+ bpf_trampoline_put(tr); /* bpf_trampoline_get above */
	613	+ return 0;
	614	+ }
	615	+
	616	+ /* Allocate and install new shim. */
	617	+
	618	+ shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);
	619	+ if (!shim_link) {
	620	+ err = -ENOMEM;
	621	+ goto err;
	622	+ }
	623	+
	624	+ err = __bpf_trampoline_link_prog(&shim_link->link, tr);
	625	+ if (err)
	626	+ goto err;
	627	+
	628	+ shim_link->trampoline = tr;
	629	+ /* note, we're still holding tr refcnt from above */
	630	+
	631	+ mutex_unlock(&tr->mutex);
	632	+
	633	+ return 0;
	634	+err:
	635	+ mutex_unlock(&tr->mutex);
	636	+
	637	+ if (shim_link)
	638	+ bpf_link_put(&shim_link->link.link);
	639	+
	640	+ /* have to release tr while _not_ holding its mutex */
	641	+ bpf_trampoline_put(tr); /* bpf_trampoline_get above */
	642	+
	643	+ return err;
	644	+}
	645	+
	646	+void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
	647	+{
	648	+ struct bpf_shim_tramp_link *shim_link = NULL;
	649	+ struct bpf_trampoline *tr;
	650	+ bpf_func_t bpf_func;
	651	+ u64 key;
	652	+
	653	+ key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
	654	+ prog->aux->attach_btf_id);
	655	+
	656	+ bpf_lsm_find_cgroup_shim(prog, &bpf_func);
	657	+ tr = bpf_trampoline_lookup(key);
	658	+ if (WARN_ON_ONCE(!tr))
	659	+ return;
	660	+
	661	+ mutex_lock(&tr->mutex);
	662	+ shim_link = cgroup_shim_find(tr, bpf_func);
	663	+ mutex_unlock(&tr->mutex);
	664	+
	665	+ if (shim_link)
	666	+ bpf_link_put(&shim_link->link.link);
	667	+
	668	+ bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
	669	+}
	670	+#endif
	671	+
499	672	struct bpf_trampoline *bpf_trampoline_get(u64 key,
500	673	struct bpf_attach_target_info *tgt_info)
501	674	{
@@ -628,6 +801,31 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ · 126 unmodified lines
628	801	rcu_read_unlock();
629	802	}
630	803
	804	+u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
	805	+ struct bpf_tramp_run_ctx *run_ctx)
	806	+ __acquires(RCU)
	807	+{
	808	+ /* Runtime stats are exported via actual BPF_LSM_CGROUP
	809	+ * programs, not the shims.
	810	+ */
	811	+ rcu_read_lock();
	812	+ migrate_disable();
	813	+
	814	+ run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
	815	+
	816	+ return NO_START_TIME;
	817	+}
	818	+
	819	+void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
	820	+ struct bpf_tramp_run_ctx *run_ctx)
	821	+ __releases(RCU)
	822	+{
	823	+ bpf_reset_run_ctx(run_ctx->saved_run_ctx);
	824	+
	825	+ migrate_enable();
	826	+ rcu_read_unlock();
	827	+}
	828	+
631	829	u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog prog, struct bpf_tramp_run_ctx run_ctx)
632	830	{
633	831	rcu_read_lock_trace();

kernel/bpf/verifier.c+32−0

@@ -7322,6 +7322,18 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 7321 unmodified lines
7322	7322	reg_type_str(env, regs[BPF_REG_1].type));
7323	7323	return -EACCES;
7324	7324	}
	7325	+ break;
	7326	+ case BPF_FUNC_set_retval:
	7327	+ if (env->prog->expected_attach_type == BPF_LSM_CGROUP) {
	7328	+ if (!env->prog->aux->attach_func_proto->type) {
	7329	+ /* Make sure programs that attach to void
	7330	+ * hooks don't try to modify return value.
	7331	+ */
	7332	+ verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
	7333	+ return -EINVAL;
	7334	+ }
	7335	+ }
	7336	+ break;
7325	7337	}
7326	7338
7327	7339	if (err)
@@ -10527,6 +10539,22 @@ static int check_return_code(struct bpf_verifier_env *env) · 3199 unmodified lines
10527	10539	case BPF_PROG_TYPE_SK_LOOKUP:
10528	10540	range = tnum_range(SK_DROP, SK_PASS);
10529	10541	break;
	10542	+
	10543	+ case BPF_PROG_TYPE_LSM:
	10544	+ if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
	10545	+ /* Regular BPF_PROG_TYPE_LSM programs can return
	10546	+ * any value.
	10547	+ */
	10548	+ return 0;
	10549	+ }
	10550	+ if (!env->prog->aux->attach_func_proto->type) {
	10551	+ /* Make sure programs that attach to void
	10552	+ * hooks don't try to modify return value.
	10553	+ */
	10554	+ range = tnum_range(1, 1);
	10555	+ }
	10556	+ break;
	10557	+
10530	10558	case BPF_PROG_TYPE_EXT:
10531	10559	/* freplace program can return anything as its return value
10532	10560	* depends on the to-be-replaced kernel func or bpf program.
@@ -10543,6 +10571,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 10 unmodified lines
10543	10571
10544	10572	if (!tnum_in(range, reg->var_off)) {
10545	10573	verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
	10574	+ if (prog->expected_attach_type == BPF_LSM_CGROUP &&
	10575	+ !prog->aux->attach_func_proto->type)
	10576	+ verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
10546	10577	return -EINVAL;
10547	10578	}
10548	10579
@@ -14902,6 +14933,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 4353 unmodified lines
14902	14933	fallthrough;
14903	14934	case BPF_MODIFY_RETURN:
14904	14935	case BPF_LSM_MAC:
	14936	+ case BPF_LSM_CGROUP:
14905	14937	case BPF_TRACE_FENTRY:
14906	14938	case BPF_TRACE_FEXIT:
14907	14939	if (!btf_type_is_func(t)) {

tools/include/uapi/linux/bpf.h+1−0

@@ -998,6 +998,7 @@ enum bpf_attach_type { · 997 unmodified lines
998	998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
999	999	BPF_PERF_EVENT,
1000	1000	BPF_TRACE_KPROBE_MULTI,
	1001	+ BPF_LSM_CGROUP,
1001	1002	__MAX_BPF_ATTACH_TYPE
1002	1003	};
1003	1004

bpf: per-cgroup lsm flavor

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code