// SPDX-License-Identifier: GPL-2.0-only/* * Functions to manage eBPF programs attached to cgroups * * Copyright (c) 2016 Daniel Mack */ #include <linux/kernel.h>#include <linux/atomic.h>#include <linux/cgroup.h>#include <linux/filter.h>#include <linux/slab.h>#include <linux/sysctl.h>#include <linux/string.h>#include <linux/bpf.h>#include <linux/bpf-cgroup.h>#include <linux/bpf_lsm.h>#include <linux/bpf_verifier.h>#include <net/sock.h>#include <net/bpf_sk_storage.h> #include "../cgroup/cgroup-internal.h" DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE);EXPORT_SYMBOL(cgroup_bpf_enabled_key); /* * cgroup bpf destruction makes heavy use of work items and there can be a lot * of concurrent destructions.  Use a separate workqueue so that cgroup bpf * destruction work items don't end up filling up max_active of system_percpu_wq * which may lead to deadlock. */static struct workqueue_struct *cgroup_bpf_destroy_wq; static int __init cgroup_bpf_wq_init(void){	cgroup_bpf_destroy_wq = alloc_workqueue("cgroup_bpf_destroy",						WQ_PERCPU, 1);	if (!cgroup_bpf_destroy_wq)		panic("Failed to alloc workqueue for cgroup bpf destroy.\n");	return 0;}core_initcall(cgroup_bpf_wq_init); static int cgroup_bpf_lifetime_notify(struct notifier_block *nb,				      unsigned long action, void *data); static struct notifier_block cgroup_bpf_lifetime_nb = {	.notifier_call = cgroup_bpf_lifetime_notify,}; void __init cgroup_bpf_lifetime_notifier_init(void){	BUG_ON(blocking_notifier_chain_register(&cgroup_lifetime_notifier,						&cgroup_bpf_lifetime_nb));} #ifdef CONFIG_BPF_LSMstruct cgroup_lsm_atype {	u32 attach_btf_id;	int refcnt;	bool returns_errno;}; static struct cgroup_lsm_atype cgroup_lsm_atype[CGROUP_LSM_NUM]; static bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype){	if (atype >= CGROUP_LSM_START && atype <= CGROUP_LSM_END)		return READ_ONCE(cgroup_lsm_atype[atype - CGROUP_LSM_START].returns_errno);	return true;}#elsestatic bool cgroup_bpf_hook_returns_errno(enum cgroup_bpf_attach_type atype){	return true;}#endif /* __always_inline is necessary to prevent indirect call through run_prog * function pointer. */static __always_inline intbpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,		      enum cgroup_bpf_attach_type atype,		      const void *ctx, bpf_prog_run_fn run_prog,		      int retval, u32 *ret_flags){	const struct bpf_prog_array_item *item;	const struct bpf_prog *prog;	const struct bpf_prog_array *array;	struct bpf_run_ctx *old_run_ctx;	struct bpf_cg_run_ctx run_ctx;	u32 func_ret; 	run_ctx.retval = retval;	rcu_read_lock_dont_migrate();	array = rcu_dereference(cgrp->effective[atype]);	item = &array->items[0];	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);	while ((prog = READ_ONCE(item->prog))) {		run_ctx.prog_item = item;		func_ret = run_prog(prog, ctx);		if (ret_flags) {			*(ret_flags) |= (func_ret >> 1);			func_ret &= 1;		}		if (!func_ret && cgroup_bpf_hook_returns_errno(atype) &&		    !IS_ERR_VALUE((long)run_ctx.retval))			run_ctx.retval = -EPERM;		item++;	}	bpf_reset_run_ctx(old_run_ctx);	rcu_read_unlock_migrate();	return run_ctx.retval;} unsigned int __cgroup_bpf_run_lsm_sock(const void *ctx,				       const struct bpf_insn *insn){	const struct bpf_prog *shim_prog;	struct sock *sk;	struct cgroup *cgrp;	int ret = 0;	u64 *args; 	args = (u64 *)ctx;	sk = (void *)(unsigned long)args[0];	/*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/	shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);	if (likely(cgrp))		ret = bpf_prog_run_array_cg(&cgrp->bpf,					    shim_prog->aux->cgroup_atype,					    ctx, bpf_prog_run, 0, NULL);	return ret;} unsigned int __cgroup_bpf_run_lsm_socket(const void *ctx,					 const struct bpf_insn *insn){	const struct bpf_prog *shim_prog;	struct socket *sock;	struct cgroup *cgrp;	int ret = 0;	u64 *args; 	args = (u64 *)ctx;	sock = (void *)(unsigned long)args[0];	/*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/	shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); 	cgrp = sock_cgroup_ptr(&sock->sk->sk_cgrp_data);	if (likely(cgrp))		ret = bpf_prog_run_array_cg(&cgrp->bpf,					    shim_prog->aux->cgroup_atype,					    ctx, bpf_prog_run, 0, NULL);	return ret;} unsigned int __cgroup_bpf_run_lsm_current(const void *ctx,					  const struct bpf_insn *insn){	const struct bpf_prog *shim_prog;	struct cgroup *cgrp;	int ret = 0; 	/*shim_prog = container_of(insn, struct bpf_prog, insnsi);*/	shim_prog = (const struct bpf_prog *)((void *)insn - offsetof(struct bpf_prog, insnsi)); 	/* We rely on trampoline's __bpf_prog_enter_lsm_cgroup to grab RCU read lock. */	cgrp = task_dfl_cgroup(current);	if (likely(cgrp))		ret = bpf_prog_run_array_cg(&cgrp->bpf,					    shim_prog->aux->cgroup_atype,					    ctx, bpf_prog_run, 0, NULL);	return ret;} #ifdef CONFIG_BPF_LSMstatic enum cgroup_bpf_attach_typebpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id){	int i; 	lockdep_assert_held(&cgroup_mutex); 	if (attach_type != BPF_LSM_CGROUP)		return to_cgroup_bpf_attach_type(attach_type); 	for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)		if (cgroup_lsm_atype[i].attach_btf_id == attach_btf_id)			return CGROUP_LSM_START + i; 	for (i = 0; i < ARRAY_SIZE(cgroup_lsm_atype); i++)		if (cgroup_lsm_atype[i].attach_btf_id == 0)			return CGROUP_LSM_START + i; 	return -E2BIG; } void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype){	int i = cgroup_atype - CGROUP_LSM_START; 	lockdep_assert_held(&cgroup_mutex); 	if (!cgroup_lsm_atype[i].attach_btf_id) {		cgroup_lsm_atype[i].attach_btf_id = attach_btf_id;		WRITE_ONCE(cgroup_lsm_atype[i].returns_errno,			   bpf_lsm_hook_returns_errno(attach_btf_id));	} else {		WARN_ON_ONCE(cgroup_lsm_atype[i].attach_btf_id != attach_btf_id);	}	cgroup_lsm_atype[i].refcnt++;} void bpf_cgroup_atype_put(int cgroup_atype){	int i = cgroup_atype - CGROUP_LSM_START; 	cgroup_lock();	if (--cgroup_lsm_atype[i].refcnt <= 0) {		WRITE_ONCE(cgroup_lsm_atype[i].returns_errno, true);		cgroup_lsm_atype[i].attach_btf_id = 0;	}	WARN_ON_ONCE(cgroup_lsm_atype[i].refcnt < 0);	cgroup_unlock();}#elsestatic enum cgroup_bpf_attach_typebpf_cgroup_atype_find(enum bpf_attach_type attach_type, u32 attach_btf_id){	if (attach_type != BPF_LSM_CGROUP)		return to_cgroup_bpf_attach_type(attach_type);	return -EOPNOTSUPP;}#endif /* CONFIG_BPF_LSM */ static void cgroup_bpf_offline(struct cgroup *cgrp){	cgroup_get(cgrp);	percpu_ref_kill(&cgrp->bpf.refcnt);} static void bpf_cgroup_storages_free(struct bpf_cgroup_storage *storages[]){	enum bpf_cgroup_storage_type stype; 	for_each_cgroup_storage_type(stype)		bpf_cgroup_storage_free(storages[stype]);} static int bpf_cgroup_storages_alloc(struct bpf_cgroup_storage *storages[],				     struct bpf_cgroup_storage *new_storages[],				     enum bpf_attach_type type,				     struct bpf_prog *prog,				     struct cgroup *cgrp){	enum bpf_cgroup_storage_type stype;	struct bpf_cgroup_storage_key key;	struct bpf_map *map; 	key.cgroup_inode_id = cgroup_id(cgrp);	key.attach_type = type; 	for_each_cgroup_storage_type(stype) {		map = prog->aux->cgroup_storage[stype];		if (!map)			continue; 		storages[stype] = cgroup_storage_lookup((void *)map, &key, false);		if (storages[stype])			continue; 		storages[stype] = bpf_cgroup_storage_alloc(prog, stype);		if (IS_ERR(storages[stype])) {			bpf_cgroup_storages_free(new_storages);			return -ENOMEM;		} 		new_storages[stype] = storages[stype];	} 	return 0;} static void bpf_cgroup_storages_assign(struct bpf_cgroup_storage *dst[],				       struct bpf_cgroup_storage *src[]){	enum bpf_cgroup_storage_type stype; 	for_each_cgroup_storage_type(stype)		dst[stype] = src[stype];} static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],				     struct cgroup *cgrp,				     enum bpf_attach_type attach_type){	enum bpf_cgroup_storage_type stype; 	for_each_cgroup_storage_type(stype)		bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);} /* Called when bpf_cgroup_link is auto-detached from dying cgroup. * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It * doesn't free link memory, which will eventually be done by bpf_link's * release() callback, when its last FD is closed. */static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link){	cgroup_put(link->cgroup);	link->cgroup = NULL;} /** * cgroup_bpf_release() - put references of all bpf programs and *                        release all cgroup bpf data * @work: work structure embedded into the cgroup to modify */static void cgroup_bpf_release(struct work_struct *work){	struct cgroup *p, *cgrp = container_of(work, struct cgroup,					       bpf.release_work);	struct bpf_prog_array *old_array;	struct list_head *storages = &cgrp->bpf.storages;	struct bpf_cgroup_storage *storage, *stmp; 	unsigned int atype; 	cgroup_lock(); 	for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {		struct hlist_head *progs = &cgrp->bpf.progs[atype];		struct bpf_prog_list *pl;		struct hlist_node *pltmp; 		hlist_for_each_entry_safe(pl, pltmp, progs, node) {			hlist_del(&pl->node);			if (pl->prog) {				if (pl->prog->expected_attach_type == BPF_LSM_CGROUP)					bpf_trampoline_unlink_cgroup_shim(pl->prog);				bpf_prog_put(pl->prog);			}			if (pl->link) {				if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)					bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);				bpf_cgroup_link_auto_detach(pl->link);			}			kfree(pl);			static_branch_dec(&cgroup_bpf_enabled_key[atype]);		}		old_array = rcu_dereference_protected(				cgrp->bpf.effective[atype],				lockdep_is_held(&cgroup_mutex));		bpf_prog_array_free(old_array);	} 	list_for_each_entry_safe(storage, stmp, storages, list_cg) {		bpf_cgroup_storage_unlink(storage);		bpf_cgroup_storage_free(storage);	} 	cgroup_unlock(); 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))		cgroup_bpf_put(p); 	percpu_ref_exit(&cgrp->bpf.refcnt);	cgroup_put(cgrp);} /** * cgroup_bpf_release_fn() - callback used to schedule releasing *                           of bpf cgroup data * @ref: percpu ref counter structure */static void cgroup_bpf_release_fn(struct percpu_ref *ref){	struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); 	INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);	queue_work(cgroup_bpf_destroy_wq, &cgrp->bpf.release_work);} /* Get underlying bpf_prog of bpf_prog_list entry, regardless if it's through * link or direct prog. */static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl){	if (pl->prog)		return pl->prog;	if (pl->link)		return pl->link->link.prog;	return NULL;} /* count number of elements in the list. * it's slow but the list cannot be long */static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt){	struct bpf_prog_list *pl;	u32 cnt = 0; 	hlist_for_each_entry(pl, head, node) {		if (!prog_list_prog(pl))			continue;		if (preorder_cnt && (pl->flags & BPF_F_PREORDER))			(*preorder_cnt)++;		cnt++;	}	return cnt;} /* if parent has non-overridable prog attached, * disallow attaching new programs to the descendent cgroup. * if parent has overridable or multi-prog, allow attaching */static bool hierarchy_allows_attach(struct cgroup *cgrp,				    enum cgroup_bpf_attach_type atype){	struct cgroup *p; 	p = cgroup_parent(cgrp);	if (!p)		return true;	do {		u32 flags = p->bpf.flags[atype];		u32 cnt; 		if (flags & BPF_F_ALLOW_MULTI)			return true;		cnt = prog_list_length(&p->bpf.progs[atype], NULL);		WARN_ON_ONCE(cnt > 1);		if (cnt == 1)			return !!(flags & BPF_F_ALLOW_OVERRIDE);		p = cgroup_parent(p);	} while (p);	return true;} /* compute a chain of effective programs for a given cgroup: * start from the list of programs in this cgroup and add * all parent programs. * Note that parent's F_ALLOW_OVERRIDE-type program is yielding * to programs in this cgroup */static int compute_effective_progs(struct cgroup *cgrp,				   enum cgroup_bpf_attach_type atype,				   struct bpf_prog_array **array){	struct bpf_prog_array_item *item;	struct bpf_prog_array *progs;	struct bpf_prog_list *pl;	struct cgroup *p = cgrp;	int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart; 	/* count number of effective programs by walking parents */	do {		if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))			cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);		p = cgroup_parent(p);	} while (p); 	progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);	if (!progs)		return -ENOMEM; 	/* populate the array with effective progs */	cnt = 0;	p = cgrp;	fstart = preorder_cnt;	bstart = preorder_cnt - 1;	do {		if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))			continue; 		init_bstart = bstart;		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {			if (!prog_list_prog(pl))				continue; 			if (pl->flags & BPF_F_PREORDER) {				item = &progs->items[bstart];				bstart--;			} else {				item = &progs->items[fstart];				fstart++;			}			item->prog = prog_list_prog(pl);			bpf_cgroup_storages_assign(item->cgroup_storage,						   pl->storage);			cnt++;		} 

Lines 1–500 of 2821. Load more