bpf: per-cgroup lsm flavor

Allow attaching to lsm hooks in the cgroup context. Attaching to per-cgroup LSM works exactly like attaching to other per-cgroup hooks. New BPF_LSM_CGROUP is added to trigger new mode; the actual lsm hook we attach to is signaled via existing attach_btf_id. For the hooks that have 'struct socket' or 'struct sock' as its first argument, we use the cgroup associated with that socket. For the rest, we use 'current' cgroup (this is all on default hierarchy == v2 only). Note that for some hooks that work on 'struct sock' we still take the cgroup from 'current' because some of them work on the socket that hasn't been properly initialized yet. Behind the scenes, we allocate a shim program that is attached to the trampoline and runs cgroup effective BPF programs array. This shim has some rudimentary ref counting and can be shared between several programs attaching to the same lsm hook from different cgroups. Note that this patch bloats cgroup size because we add 211 cgroup_bpf_attach_type(s) for simplicity sake. This will be addressed in the subsequent patch. Also note that we only add non-sleepable flavor for now. To enable sleepable use-cases, bpf_prog_run_array_cg has to grab trace rcu, shim programs have to be freed via trace rcu, cgroup_bpf.effective should be also trace-rcu-managed + maybe some other changes that I'm not aware of.

Stanislav Fomichev committed 4 years ago

Commit 69fd3371 parent 004421415 files · +498 −20Browse files

Unified · Split

15 files

arch/x86/net/bpf_jit_comp.c +16 −8
include/linux/bpf-cgroup-defs.h +8 −0
include/linux/bpf-cgroup.h +7 −0
include/linux/bpf.h +24 −0
include/linux/bpf_lsm.h +13 −0
include/linux/btf_ids.h +2 −1
include/uapi/linux/bpf.h +1 −0
kernel/bpf/bpf_lsm.c +48 −0
kernel/bpf/btf.c +11 −0
kernel/bpf/cgroup.c +125 −11
kernel/bpf/core.c +2 −0
kernel/bpf/syscall.c +10 −0
kernel/bpf/trampoline.c +198 −0
kernel/bpf/verifier.c +32 −0
tools/include/uapi/linux/bpf.h +1 −0

arch/x86/net/bpf_jit_comp.c+16−8

 			   struct bpf_tramp_link *l, int stack_size,
 			   int run_ctx_off, bool save_ret)
 {
+	void (*exit)(struct bpf_prog *prog, u64 start,
+		     struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_exit;
+	u64 (*enter)(struct bpf_prog *prog,
+		     struct bpf_tramp_run_ctx *run_ctx) = __bpf_prog_enter;
 	u8 *prog = *pprog;
 	u8 *jmp_insn;
 	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	 */
 	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);
+	if (p->aux->sleepable) {
+		enter = __bpf_prog_enter_sleepable;
+		exit = __bpf_prog_exit_sleepable;
+	} else if (p->expected_attach_type == BPF_LSM_CGROUP) {
+		enter = __bpf_prog_enter_lsm_cgroup;
+		exit = __bpf_prog_exit_lsm_cgroup;
+	}
 	/* arg1: mov rdi, progs[i] */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
 	/* arg2: lea rsi, [rbp - ctx_cookie_off] */
 	EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
-	if (emit_call(&prog,
+	if (emit_call(&prog, enter, prog))
-		      p->aux->sleepable ? __bpf_prog_enter_sleepable :
+		return -EINVAL;
-		      __bpf_prog_enter, prog))
-			return -EINVAL;
 	/* remember prog start time returned by __bpf_prog_enter */
 	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
 	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
 	/* arg3: lea rdx, [rbp - run_ctx_off] */
 	EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
-	if (emit_call(&prog,
+	if (emit_call(&prog, exit, prog))
-		      p->aux->sleepable ? __bpf_prog_exit_sleepable :
+		return -EINVAL;
-		      __bpf_prog_exit, prog))
-			return -EINVAL;
 	*pprog = prog;
 	return 0;

include/linux/bpf-cgroup-defs.h+8−0

 struct bpf_prog_array;
+#ifdef CONFIG_BPF_LSM
+#define CGROUP_LSM_NUM 211 /* will be addressed in the next patch */
+#else
+#define CGROUP_LSM_NUM 0
+#endif
 enum cgroup_bpf_attach_type {
 	CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
 	CGROUP_INET_INGRESS = 0,
 	CGROUP_INET4_GETSOCKNAME,
 	CGROUP_INET6_GETSOCKNAME,
 	CGROUP_INET_SOCK_RELEASE,
+	CGROUP_LSM_START,
+	CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1,
 	MAX_CGROUP_BPF_ATTACH_TYPE
 };

include/linux/bpf-cgroup.h+7−0

@@ -23,6 +23,13 @@ struct ctl_table; · 22 unmodified lines
23	struct ctl_table_header;	23	struct ctl_table_header;
24	struct task_struct;	24	struct task_struct;
25		25
		26	unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
		27	const struct bpf_insn *insn);
		28	unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
		29	const struct bpf_insn *insn);
		30	unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
		31	const struct bpf_insn *insn);
		32
26	#ifdef CONFIG_CGROUP_BPF	33	#ifdef CONFIG_CGROUP_BPF
27		34
28	#define CGROUP_ATYPE(type) \	35	#define CGROUP_ATYPE(type) \

include/linux/bpf.h+24−0

 u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
 				       struct bpf_tramp_run_ctx *run_ctx);
+u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
+					struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
+					struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
 	struct user_struct *user;
 	u64 load_time; /* ns since boottime */
 	u32 verified_insns;
+	int cgroup_atype; /* enum cgroup_bpf_attach_type */
 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
 	char name[BPF_OBJ_NAME_LEN];
 #ifdef CONFIG_SECURITY
 	u64 cookie;
 };
+struct bpf_shim_tramp_link {
+	struct bpf_tramp_link link;
+	struct bpf_trampoline *trampoline;
+};
 struct bpf_tracing_link {
 	struct bpf_tramp_link link;
 	enum bpf_attach_type attach_type;
 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
 			    union bpf_attr __user *uattr);
 #endif
+int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
+				    int cgroup_atype);
+void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
 #else
 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
 {
 {
 	return -EINVAL;
 }
+static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
+						  int cgroup_atype)
+{
+	return -EOPNOTSUPP;
+}
+static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
+{
+}
 #endif
 struct bpf_array {
 extern const struct bpf_func_proto bpf_find_vma_proto;
 extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
+extern const struct bpf_func_proto bpf_set_retval_proto;
+extern const struct bpf_func_proto bpf_get_retval_proto;
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
 struct btf_id_set;
 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
+int btf_id_set_index(const struct btf_id_set *set, u32 id);
 #define MAX_BPRINTF_VARARGS		12

include/linux/bpf_lsm.h+13−0

@@ -42,6 +42,9 @@ extern const struct bpf_func_proto bpf_inode_storage_get_proto; · 41 unmodified lines
42	extern const struct bpf_func_proto bpf_inode_storage_delete_proto;	42	extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
43	void bpf_inode_storage_free(struct inode *inode);	43	void bpf_inode_storage_free(struct inode *inode);
44		44
		45	int bpf_lsm_hook_idx(u32 btf_id);
		46	void bpf_lsm_find_cgroup_shim(const struct bpf_prog prog, bpf_func_t bpf_func);
		47
45	#else /* !CONFIG_BPF_LSM */	48	#else /* !CONFIG_BPF_LSM */
46		49
47	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)	50	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@@ -65,6 +68,16 @@ static inline void bpf_inode_storage_free(struct inode *inode) · 17 unmodified lines
65	{	68	{
66	}	69	}
67		70
		71	static inline void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
		72	bpf_func_t *bpf_func)
		73	{
		74	}
		75
		76	static inline int bpf_lsm_hook_idx(u32 btf_id)
		77	{
		78	return -EINVAL;
		79	}
		80
68	#endif /* CONFIG_BPF_LSM */	81	#endif /* CONFIG_BPF_LSM */
69		82
70	#endif /* _LINUX_BPF_LSM_H */	83	#endif /* _LINUX_BPF_LSM_H */

include/linux/btf_ids.h+2−1

@@ -179,7 +179,8 @@ extern struct btf_id_set name; · 178 unmodified lines
179	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP, udp_sock) \	179	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP, udp_sock) \
180	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) \	180	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) \
181	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UNIX, unix_sock) \	181	BTF_SOCK_TYPE(BTF_SOCK_TYPE_UNIX, unix_sock) \
182	BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock)	182	BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock) \
		183	BTF_SOCK_TYPE(BTF_SOCK_TYPE_SOCKET, socket)
183		184
184	enum {	185	enum {
185	#define BTF_SOCK_TYPE(name, str) name,	186	#define BTF_SOCK_TYPE(name, str) name,

include/uapi/linux/bpf.h+1−0

@@ -998,6 +998,7 @@ enum bpf_attach_type { · 997 unmodified lines
998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,	998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
999	BPF_PERF_EVENT,	999	BPF_PERF_EVENT,
1000	BPF_TRACE_KPROBE_MULTI,	1000	BPF_TRACE_KPROBE_MULTI,
		1001	BPF_LSM_CGROUP,
1001	__MAX_BPF_ATTACH_TYPE	1002	__MAX_BPF_ATTACH_TYPE
1002	};	1003	};
1003		1004

kernel/bpf/bpf_lsm.c+48−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	#include <linux/bpf_local_storage.h>	16	#include <linux/bpf_local_storage.h>
17	#include <linux/btf_ids.h>	17	#include <linux/btf_ids.h>
18	#include <linux/ima.h>	18	#include <linux/ima.h>
		19	#include <linux/bpf-cgroup.h>
19		20
20	/* For every LSM hook that allows attachment of BPF programs, declare a nop	21	/* For every LSM hook that allows attachment of BPF programs, declare a nop
21	* function where a BPF program can be attached.	22	* function where a BPF program can be attached.
@@ -35,6 +36,44 @@ BTF_SET_START(bpf_lsm_hooks) · 13 unmodified lines
35	#undef LSM_HOOK	36	#undef LSM_HOOK
36	BTF_SET_END(bpf_lsm_hooks)	37	BTF_SET_END(bpf_lsm_hooks)
37		38
		39	/* List of LSM hooks that should operate on 'current' cgroup regardless
		40	* of function signature.
		41	*/
		42	BTF_SET_START(bpf_lsm_current_hooks)
		43	/* operate on freshly allocated sk without any cgroup association */
		44	BTF_ID(func, bpf_lsm_sk_alloc_security)
		45	BTF_ID(func, bpf_lsm_sk_free_security)
		46	BTF_SET_END(bpf_lsm_current_hooks)
		47
		48	void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
		49	bpf_func_t *bpf_func)
		50	{
		51	const struct btf_param *args;
		52
		53	if (btf_type_vlen(prog->aux->attach_func_proto) < 1 \|\|
		54	btf_id_set_contains(&bpf_lsm_current_hooks,
		55	prog->aux->attach_btf_id)) {
		56	*bpf_func = __cgroup_bpf_run_lsm_current;
		57	return;
		58	}
		59
		60	args = btf_params(prog->aux->attach_func_proto);
		61
		62	#ifdef CONFIG_NET
		63	if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCKET])
		64	*bpf_func = __cgroup_bpf_run_lsm_socket;
		65	else if (args[0].type == btf_sock_ids[BTF_SOCK_TYPE_SOCK])
		66	*bpf_func = __cgroup_bpf_run_lsm_sock;
		67	else
		68	#endif
		69	*bpf_func = __cgroup_bpf_run_lsm_current;
		70	}
		71
		72	int bpf_lsm_hook_idx(u32 btf_id)
		73	{
		74	return btf_id_set_index(&bpf_lsm_hooks, btf_id);
		75	}
		76
38	int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,	77	int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
39	const struct bpf_prog *prog)	78	const struct bpf_prog *prog)
40	{	79	{
@@ -158,6 +197,15 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 117 unmodified lines
158	return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;	197	return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;
159	case BPF_FUNC_get_attach_cookie:	198	case BPF_FUNC_get_attach_cookie:
160	return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;	199	return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;
		200	case BPF_FUNC_get_local_storage:
		201	return prog->expected_attach_type == BPF_LSM_CGROUP ?
		202	&bpf_get_local_storage_proto : NULL;
		203	case BPF_FUNC_set_retval:
		204	return prog->expected_attach_type == BPF_LSM_CGROUP ?
		205	&bpf_set_retval_proto : NULL;
		206	case BPF_FUNC_get_retval:
		207	return prog->expected_attach_type == BPF_LSM_CGROUP ?
		208	&bpf_get_retval_proto : NULL;
161	default:	209	default:
162	return tracing_prog_func_proto(func_id, prog);	210	return tracing_prog_func_proto(func_id, prog);
163	}	211	}

kernel/bpf/btf.c+11−0

 	if (arg == nr_args) {
 		switch (prog->expected_attach_type) {
+		case BPF_LSM_CGROUP:
 		case BPF_LSM_MAC:
 		case BPF_TRACE_FEXIT:
 			/* When LSM programs are attached to void LSM hooks
 	return *pa - *pb;
 }
+int btf_id_set_index(const struct btf_id_set *set, u32 id)
+{
+	const u32 *p;
+	p = bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func);
+	if (!p)
+		return -1;
+	return p - set->ids;
+}
 bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
 {
 	return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;

kernel/bpf/cgroup.c+125−11

@@ -14,6 +14,8 @@ · 13 unmodified lines
14	#include <linux/string.h>	14	#include <linux/string.h>
15	#include <linux/bpf.h>	15	#include <linux/bpf.h>
16	#include <linux/bpf-cgroup.h>	16	#include <linux/bpf-cgroup.h>
		17	#include <linux/bpf_lsm.h>
		18	#include <linux/bpf_verifier.h>
17	#include <net/sock.h>	19	#include <net/sock.h>
18	#include <net/bpf_sk_storage.h>	20	#include <net/bpf_sk_storage.h>
19		21
@@ -61,6 +63,87 @@ bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp, · 41 unmodified lines
61	return run_ctx.retval;	63	return run_ctx.retval;
62	}	64	}
63		65
		66	unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,
		67	const struct bpf_insn *insn)
		68	{
		69	const struct bpf_prog *shim_prog;
		70	struct sock *sk;
		71	struct cgroup *cgrp;
		72	int ret = 0;
		73	u64 *args;
		74
		75	args = (u64 *)ctx;
		76	sk = (void *)(unsigned long)args[0];
		77	/shim_prog = container_of(insn, struct bpf_prog, insnsi);/
		78	shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
		79
		80	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
		81	if (likely(cgrp))
		82	ret = bpf_prog_run_array_cg(&cgrp->bpf,
		83	shim_prog->aux->cgroup_atype,
		84	ctx, bpf_prog_run, 0, NULL);
		85	return ret;
		86	}
		87
		88	unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,
		89	const struct bpf_insn *insn)
		90	{
		91	const struct bpf_prog *shim_prog;
		92	struct socket *sock;
		93	struct cgroup *cgrp;
		94	int ret = 0;
		95	u64 *args;
		96
		97	args = (u64 *)ctx;
		98	sock = (void *)(unsigned long)args[0];
		99	/shim_prog = container_of(insn, struct bpf_prog, insnsi);/
		100	shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
		101
		102	cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);
		103	if (likely(cgrp))
		104	ret = bpf_prog_run_array_cg(&cgrp->bpf,
		105	shim_prog->aux->cgroup_atype,
		106	ctx, bpf_prog_run, 0, NULL);
		107	return ret;
		108	}
		109
		110	unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,
		111	const struct bpf_insn *insn)
		112	{
		113	const struct bpf_prog *shim_prog;
		114	struct cgroup *cgrp;
		115	int ret = 0;
		116
		117	/shim_prog = container_of(insn, struct bpf_prog, insnsi);/
		118	shim_prog = (const struct bpf_prog )((void )insn - offsetof(struct bpf_prog, insnsi));
		119
		120	/* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */
		121	cgrp = task_dfl_cgroup(current);
		122	if (likely(cgrp))
		123	ret = bpf_prog_run_array_cg(&cgrp->bpf,
		124	shim_prog->aux->cgroup_atype,
		125	ctx, bpf_prog_run, 0, NULL);
		126	return ret;
		127	}
		128
		129	#ifdef CONFIG_BPF_LSM
		130	static enum cgroup_bpf_attach_type
		131	bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
		132	{
		133	if (attach_type != BPF_LSM_CGROUP)
		134	return to_cgroup_bpf_attach_type(attach_type);
		135	return CGROUP_LSM_START + bpf_lsm_hook_idx(attach_btf_id);
		136	}
		137	#else
		138	static enum cgroup_bpf_attach_type
		139	bpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id)
		140	{
		141	if (attach_type != BPF_LSM_CGROUP)
		142	return to_cgroup_bpf_attach_type(attach_type);
		143	return -EOPNOTSUPP;
		144	}
		145	#endif /* CONFIG_BPF_LSM */
		146
64	void cgroup_bpf_offline(struct cgroup *cgrp)	147	void cgroup_bpf_offline(struct cgroup *cgrp)
65	{	148	{
66	cgroup_get(cgrp);	149	cgroup_get(cgrp);
@@ -163,10 +246,16 @@ static void cgroup_bpf_release(struct work_struct *work) · 96 unmodified lines
163		246
164	hlist_for_each_entry_safe(pl, pltmp, progs, node) {	247	hlist_for_each_entry_safe(pl, pltmp, progs, node) {
165	hlist_del(&pl->node);	248	hlist_del(&pl->node);
166	if (pl->prog)	249	if (pl->prog) {
		250	if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)
		251	bpf_trampoline_unlink_cgroup_shim(pl->prog);
167	bpf_prog_put(pl->prog);	252	bpf_prog_put(pl->prog);
168	if (pl->link)	253	}
		254	if (pl->link) {
		255	if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
		256	bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);
169	bpf_cgroup_link_auto_detach(pl->link);	257	bpf_cgroup_link_auto_detach(pl->link);
		258	}
170	kfree(pl);	259	kfree(pl);
171	static_branch_dec(&cgroup_bpf_enabled_key[atype]);	260	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
172	}	261	}
@@ -479,6 +568,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 306 unmodified lines
479	struct bpf_prog *old_prog = NULL;	568	struct bpf_prog *old_prog = NULL;
480	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};	569	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
481	struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};	570	struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {};
		571	struct bpf_prog *new_prog = prog ? : link->link.prog;
482	enum cgroup_bpf_attach_type atype;	572	enum cgroup_bpf_attach_type atype;
483	struct bpf_prog_list *pl;	573	struct bpf_prog_list *pl;
484	struct hlist_head *progs;	574	struct hlist_head *progs;
@@ -495,7 +585,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 10 unmodified lines
495	/* replace_prog implies BPF_F_REPLACE, and vice versa */	585	/* replace_prog implies BPF_F_REPLACE, and vice versa */
496	return -EINVAL;	586	return -EINVAL;
497		587
498	atype = to_cgroup_bpf_attach_type(type);	588	atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
499	if (atype < 0)	589	if (atype < 0)
500	return -EINVAL;	590	return -EINVAL;
501		591
@@ -549,17 +639,30 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 47 unmodified lines
549	bpf_cgroup_storages_assign(pl->storage, storage);	639	bpf_cgroup_storages_assign(pl->storage, storage);
550	cgrp->bpf.flags[atype] = saved_flags;	640	cgrp->bpf.flags[atype] = saved_flags;
551		641
		642	if (type == BPF_LSM_CGROUP) {
		643	err = bpf_trampoline_link_cgroup_shim(new_prog, atype);
		644	if (err)
		645	goto cleanup;
		646	}
		647
552	err = update_effective_progs(cgrp, atype);	648	err = update_effective_progs(cgrp, atype);
553	if (err)	649	if (err)
554	goto cleanup;	650	goto cleanup_trampoline;
555		651
556	if (old_prog)	652	if (old_prog) {
		653	if (type == BPF_LSM_CGROUP)
		654	bpf_trampoline_unlink_cgroup_shim(old_prog);
557	bpf_prog_put(old_prog);	655	bpf_prog_put(old_prog);
558	else	656	} else {
559	static_branch_inc(&cgroup_bpf_enabled_key[atype]);	657	static_branch_inc(&cgroup_bpf_enabled_key[atype]);
		658	}
560	bpf_cgroup_storages_link(new_storage, cgrp, type);	659	bpf_cgroup_storages_link(new_storage, cgrp, type);
561	return 0;	660	return 0;
562		661
		662	cleanup_trampoline:
		663	if (type == BPF_LSM_CGROUP)
		664	bpf_trampoline_unlink_cgroup_shim(new_prog);
		665
563	cleanup:	666	cleanup:
564	if (old_prog) {	667	if (old_prog) {
565	pl->prog = old_prog;	668	pl->prog = old_prog;
@@ -651,7 +754,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, · 85 unmodified lines
651	struct hlist_head *progs;	754	struct hlist_head *progs;
652	bool found = false;	755	bool found = false;
653		756
654	atype = to_cgroup_bpf_attach_type(link->type);	757	atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id);
655	if (atype < 0)	758	if (atype < 0)
656	return -EINVAL;	759	return -EINVAL;
657		760
@@ -803,9 +906,15 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 145 unmodified lines
803	struct bpf_prog *old_prog;	906	struct bpf_prog *old_prog;
804	struct bpf_prog_list *pl;	907	struct bpf_prog_list *pl;
805	struct hlist_head *progs;	908	struct hlist_head *progs;
		909	u32 attach_btf_id = 0;
806	u32 flags;	910	u32 flags;
807		911
808	atype = to_cgroup_bpf_attach_type(type);	912	if (prog)
		913	attach_btf_id = prog->aux->attach_btf_id;
		914	if (link)
		915	attach_btf_id = link->link.prog->aux->attach_btf_id;
		916
		917	atype = bpf_cgroup_atype_find(type, attach_btf_id);
809	if (atype < 0)	918	if (atype < 0)
810	return -EINVAL;	919	return -EINVAL;
811		920
@@ -839,8 +948,11 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 27 unmodified lines
839	if (hlist_empty(progs))	948	if (hlist_empty(progs))
840	/* last program was detached, reset flags to zero */	949	/* last program was detached, reset flags to zero */
841	cgrp->bpf.flags[atype] = 0;	950	cgrp->bpf.flags[atype] = 0;
842	if (old_prog)	951	if (old_prog) {
		952	if (type == BPF_LSM_CGROUP)
		953	bpf_trampoline_unlink_cgroup_shim(old_prog);
843	bpf_prog_put(old_prog);	954	bpf_prog_put(old_prog);
		955	}
844	static_branch_dec(&cgroup_bpf_enabled_key[atype]);	956	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
845	return 0;	957	return 0;
846	}	958	}
@@ -999,6 +1111,8 @@ static void bpf_cgroup_link_release(struct bpf_link *link) · 152 unmodified lines
999		1111
1000	WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,	1112	WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,
1001	cg_link->type));	1113	cg_link->type));
		1114	if (cg_link->type == BPF_LSM_CGROUP)
		1115	bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);
1002		1116
1003	cg = cg_link->cgroup;	1117	cg = cg_link->cgroup;
1004	cg_link->cgroup = NULL;	1118	cg_link->cgroup = NULL;
@@ -1343,7 +1457,7 @@ BPF_CALL_0(bpf_get_retval) · 338 unmodified lines
1343	return ctx->retval;	1457	return ctx->retval;
1344	}	1458	}
1345		1459
1346	static const struct bpf_func_proto bpf_get_retval_proto = {	1460	const struct bpf_func_proto bpf_get_retval_proto = {
1347	.func = bpf_get_retval,	1461	.func = bpf_get_retval,
1348	.gpl_only = false,	1462	.gpl_only = false,
1349	.ret_type = RET_INTEGER,	1463	.ret_type = RET_INTEGER,
@@ -1358,7 +1472,7 @@ BPF_CALL_1(bpf_set_retval, int, retval) · 8 unmodified lines
1358	return 0;	1472	return 0;
1359	}	1473	}
1360		1474
1361	static const struct bpf_func_proto bpf_set_retval_proto = {	1475	const struct bpf_func_proto bpf_set_retval_proto = {
1362	.func = bpf_set_retval,	1476	.func = bpf_set_retval,
1363	.gpl_only = false,	1477	.gpl_only = false,
1364	.ret_type = RET_INTEGER,	1478	.ret_type = RET_INTEGER,

kernel/bpf/core.c+2−0

@@ -2666,6 +2666,8 @@ const struct bpf_func_proto bpf_get_local_storage_proto __weak; · 2665 unmodified lines
2666	const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;	2666	const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;
2667	const struct bpf_func_proto bpf_snprintf_btf_proto __weak;	2667	const struct bpf_func_proto bpf_snprintf_btf_proto __weak;
2668	const struct bpf_func_proto bpf_seq_printf_btf_proto __weak;	2668	const struct bpf_func_proto bpf_seq_printf_btf_proto __weak;
		2669	const struct bpf_func_proto bpf_set_retval_proto __weak;
		2670	const struct bpf_func_proto bpf_get_retval_proto __weak;
2669		2671
2670	const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)	2672	const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
2671	{	2673	{

kernel/bpf/syscall.c+10−0

 		return BPF_PROG_TYPE_SK_LOOKUP;
 	case BPF_XDP:
 		return BPF_PROG_TYPE_XDP;
+	case BPF_LSM_CGROUP:
+		return BPF_PROG_TYPE_LSM;
 	default:
 		return BPF_PROG_TYPE_UNSPEC;
 	}
 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
 	case BPF_PROG_TYPE_SOCK_OPS:
+	case BPF_PROG_TYPE_LSM:
+		if (ptype == BPF_PROG_TYPE_LSM &&
+		    prog->expected_attach_type != BPF_LSM_CGROUP)
+			return -EINVAL;
 		ret = cgroup_bpf_prog_attach(attr, ptype, prog);
 		break;
 	default:
 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
 	case BPF_PROG_TYPE_SOCK_OPS:
+	case BPF_PROG_TYPE_LSM:
 		return cgroup_bpf_prog_detach(attr, ptype);
 	default:
 		return -EINVAL;
 			ret = bpf_raw_tp_link_attach(prog, NULL);
 		else if (prog->expected_attach_type == BPF_TRACE_ITER)
 			ret = bpf_iter_link_attach(attr, uattr, prog);
+		else if (prog->expected_attach_type == BPF_LSM_CGROUP)
+			ret = cgroup_bpf_link_attach(attr, prog);
 		else
 			ret = bpf_tracing_prog_attach(prog,
 						      attr->link_create.target_fd,

kernel/bpf/trampoline.c+198−0

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	#include <linux/rcupdate_wait.h>	11	#include <linux/rcupdate_wait.h>
12	#include <linux/module.h>	12	#include <linux/module.h>
13	#include <linux/static_call.h>	13	#include <linux/static_call.h>
		14	#include <linux/bpf_verifier.h>
		15	#include <linux/bpf_lsm.h>
14		16
15	/* dummy _ops. The verifier will operate on target program's ops. */	17	/* dummy _ops. The verifier will operate on target program's ops. */
16	const struct bpf_verifier_ops bpf_extension_verifier_ops = {	18	const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -496,6 +498,177 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin · 479 unmodified lines
496	return err;	498	return err;
497	}	499	}
498		500
		501	#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
		502	static void bpf_shim_tramp_link_release(struct bpf_link *link)
		503	{
		504	struct bpf_shim_tramp_link *shim_link =
		505	container_of(link, struct bpf_shim_tramp_link, link.link);
		506
		507	/* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */
		508	if (!shim_link->trampoline)
		509	return;
		510
		511	WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));
		512	bpf_trampoline_put(shim_link->trampoline);
		513	}
		514
		515	static void bpf_shim_tramp_link_dealloc(struct bpf_link *link)
		516	{
		517	struct bpf_shim_tramp_link *shim_link =
		518	container_of(link, struct bpf_shim_tramp_link, link.link);
		519
		520	kfree(shim_link);
		521	}
		522
		523	static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
		524	.release = bpf_shim_tramp_link_release,
		525	.dealloc = bpf_shim_tramp_link_dealloc,
		526	};
		527
		528	static struct bpf_shim_tramp_link cgroup_shim_alloc(const struct bpf_prog prog,
		529	bpf_func_t bpf_func,
		530	int cgroup_atype)
		531	{
		532	struct bpf_shim_tramp_link *shim_link = NULL;
		533	struct bpf_prog *p;
		534
		535	shim_link = kzalloc(sizeof(*shim_link), GFP_USER);
		536	if (!shim_link)
		537	return NULL;
		538
		539	p = bpf_prog_alloc(1, 0);
		540	if (!p) {
		541	kfree(shim_link);
		542	return NULL;
		543	}
		544
		545	p->jited = false;
		546	p->bpf_func = bpf_func;
		547
		548	p->aux->cgroup_atype = cgroup_atype;
		549	p->aux->attach_func_proto = prog->aux->attach_func_proto;
		550	p->aux->attach_btf_id = prog->aux->attach_btf_id;
		551	p->aux->attach_btf = prog->aux->attach_btf;
		552	btf_get(p->aux->attach_btf);
		553	p->type = BPF_PROG_TYPE_LSM;
		554	p->expected_attach_type = BPF_LSM_MAC;
		555	bpf_prog_inc(p);
		556	bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
		557	&bpf_shim_tramp_link_lops, p);
		558
		559	return shim_link;
		560	}
		561
		562	static struct bpf_shim_tramp_link cgroup_shim_find(struct bpf_trampoline tr,
		563	bpf_func_t bpf_func)
		564	{
		565	struct bpf_tramp_link *link;
		566	int kind;
		567
		568	for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
		569	hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) {
		570	struct bpf_prog *p = link->link.prog;
		571
		572	if (p->bpf_func == bpf_func)
		573	return container_of(link, struct bpf_shim_tramp_link, link);
		574	}
		575	}
		576
		577	return NULL;
		578	}
		579
		580	int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
		581	int cgroup_atype)
		582	{
		583	struct bpf_shim_tramp_link *shim_link = NULL;
		584	struct bpf_attach_target_info tgt_info = {};
		585	struct bpf_trampoline *tr;
		586	bpf_func_t bpf_func;
		587	u64 key;
		588	int err;
		589
		590	err = bpf_check_attach_target(NULL, prog, NULL,
		591	prog->aux->attach_btf_id,
		592	&tgt_info);
		593	if (err)
		594	return err;
		595
		596	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
		597	prog->aux->attach_btf_id);
		598
		599	bpf_lsm_find_cgroup_shim(prog, &bpf_func);
		600	tr = bpf_trampoline_get(key, &tgt_info);
		601	if (!tr)
		602	return -ENOMEM;
		603
		604	mutex_lock(&tr->mutex);
		605
		606	shim_link = cgroup_shim_find(tr, bpf_func);
		607	if (shim_link) {
		608	/* Reusing existing shim attached by the other program. */
		609	bpf_link_inc(&shim_link->link.link);
		610
		611	mutex_unlock(&tr->mutex);
		612	bpf_trampoline_put(tr); /* bpf_trampoline_get above */
		613	return 0;
		614	}
		615
		616	/* Allocate and install new shim. */
		617
		618	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);
		619	if (!shim_link) {
		620	err = -ENOMEM;
		621	goto err;
		622	}
		623
		624	err = __bpf_trampoline_link_prog(&shim_link->link, tr);
		625	if (err)
		626	goto err;
		627
		628	shim_link->trampoline = tr;
		629	/* note, we're still holding tr refcnt from above */
		630
		631	mutex_unlock(&tr->mutex);
		632
		633	return 0;
		634	err:
		635	mutex_unlock(&tr->mutex);
		636
		637	if (shim_link)
		638	bpf_link_put(&shim_link->link.link);
		639
		640	/* have to release tr while _not_ holding its mutex */
		641	bpf_trampoline_put(tr); /* bpf_trampoline_get above */
		642
		643	return err;
		644	}
		645
		646	void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
		647	{
		648	struct bpf_shim_tramp_link *shim_link = NULL;
		649	struct bpf_trampoline *tr;
		650	bpf_func_t bpf_func;
		651	u64 key;
		652
		653	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,
		654	prog->aux->attach_btf_id);
		655
		656	bpf_lsm_find_cgroup_shim(prog, &bpf_func);
		657	tr = bpf_trampoline_lookup(key);
		658	if (WARN_ON_ONCE(!tr))
		659	return;
		660
		661	mutex_lock(&tr->mutex);
		662	shim_link = cgroup_shim_find(tr, bpf_func);
		663	mutex_unlock(&tr->mutex);
		664
		665	if (shim_link)
		666	bpf_link_put(&shim_link->link.link);
		667
		668	bpf_trampoline_put(tr); /* bpf_trampoline_lookup above */
		669	}
		670	#endif
		671
499	struct bpf_trampoline *bpf_trampoline_get(u64 key,	672	struct bpf_trampoline *bpf_trampoline_get(u64 key,
500	struct bpf_attach_target_info *tgt_info)	673	struct bpf_attach_target_info *tgt_info)
501	{	674	{
@@ -628,6 +801,31 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ · 126 unmodified lines
628	rcu_read_unlock();	801	rcu_read_unlock();
629	}	802	}
630		803
		804	u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
		805	struct bpf_tramp_run_ctx *run_ctx)
		806	__acquires(RCU)
		807	{
		808	/* Runtime stats are exported via actual BPF_LSM_CGROUP
		809	* programs, not the shims.
		810	*/
		811	rcu_read_lock();
		812	migrate_disable();
		813
		814	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
		815
		816	return NO_START_TIME;
		817	}
		818
		819	void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
		820	struct bpf_tramp_run_ctx *run_ctx)
		821	__releases(RCU)
		822	{
		823	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
		824
		825	migrate_enable();
		826	rcu_read_unlock();
		827	}
		828
631	u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog prog, struct bpf_tramp_run_ctx run_ctx)	829	u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog prog, struct bpf_tramp_run_ctx run_ctx)
632	{	830	{
633	rcu_read_lock_trace();	831	rcu_read_lock_trace();

kernel/bpf/verifier.c+32−0

@@ -7322,6 +7322,18 @@ static int check_helper_call(struct bpf_verifier_env env, struct bpf_insn insn · 7321 unmodified lines
7322	reg_type_str(env, regs[BPF_REG_1].type));	7322	reg_type_str(env, regs[BPF_REG_1].type));
7323	return -EACCES;	7323	return -EACCES;
7324	}	7324	}
		7325	break;
		7326	case BPF_FUNC_set_retval:
		7327	if (env->prog->expected_attach_type == BPF_LSM_CGROUP) {
		7328	if (!env->prog->aux->attach_func_proto->type) {
		7329	/* Make sure programs that attach to void
		7330	* hooks don't try to modify return value.
		7331	*/
		7332	verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
		7333	return -EINVAL;
		7334	}
		7335	}
		7336	break;
7325	}	7337	}
7326		7338
7327	if (err)	7339	if (err)
@@ -10527,6 +10539,22 @@ static int check_return_code(struct bpf_verifier_env *env) · 3199 unmodified lines
10527	case BPF_PROG_TYPE_SK_LOOKUP:	10539	case BPF_PROG_TYPE_SK_LOOKUP:
10528	range = tnum_range(SK_DROP, SK_PASS);	10540	range = tnum_range(SK_DROP, SK_PASS);
10529	break;	10541	break;
		10542
		10543	case BPF_PROG_TYPE_LSM:
		10544	if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
		10545	/* Regular BPF_PROG_TYPE_LSM programs can return
		10546	* any value.
		10547	*/
		10548	return 0;
		10549	}
		10550	if (!env->prog->aux->attach_func_proto->type) {
		10551	/* Make sure programs that attach to void
		10552	* hooks don't try to modify return value.
		10553	*/
		10554	range = tnum_range(1, 1);
		10555	}
		10556	break;
		10557
10530	case BPF_PROG_TYPE_EXT:	10558	case BPF_PROG_TYPE_EXT:
10531	/* freplace program can return anything as its return value	10559	/* freplace program can return anything as its return value
10532	* depends on the to-be-replaced kernel func or bpf program.	10560	* depends on the to-be-replaced kernel func or bpf program.
@@ -10543,6 +10571,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 10 unmodified lines
10543		10571
10544	if (!tnum_in(range, reg->var_off)) {	10572	if (!tnum_in(range, reg->var_off)) {
10545	verbose_invalid_scalar(env, reg, &range, "program exit", "R0");	10573	verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
		10574	if (prog->expected_attach_type == BPF_LSM_CGROUP &&
		10575	!prog->aux->attach_func_proto->type)
		10576	verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
10546	return -EINVAL;	10577	return -EINVAL;
10547	}	10578	}
10548		10579
@@ -14902,6 +14933,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 4353 unmodified lines
14902	fallthrough;	14933	fallthrough;
14903	case BPF_MODIFY_RETURN:	14934	case BPF_MODIFY_RETURN:
14904	case BPF_LSM_MAC:	14935	case BPF_LSM_MAC:
		14936	case BPF_LSM_CGROUP:
14905	case BPF_TRACE_FENTRY:	14937	case BPF_TRACE_FENTRY:
14906	case BPF_TRACE_FEXIT:	14938	case BPF_TRACE_FEXIT:
14907	if (!btf_type_is_func(t)) {	14939	if (!btf_type_is_func(t)) {

tools/include/uapi/linux/bpf.h+1−0

@@ -998,6 +998,7 @@ enum bpf_attach_type { · 997 unmodified lines
998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,	998	BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
999	BPF_PERF_EVENT,	999	BPF_PERF_EVENT,
1000	BPF_TRACE_KPROBE_MULTI,	1000	BPF_TRACE_KPROBE_MULTI,
		1001	BPF_LSM_CGROUP,
1001	__MAX_BPF_ATTACH_TYPE	1002	__MAX_BPF_ATTACH_TYPE
1002	};	1003	};
1003		1004

bpf: per-cgroup lsm flavor

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code