1552 lines · 39.6 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-only/* * Functions to manage eBPF programs attached to cgroups * * Copyright (c) 2016 Daniel Mack */ #include <linux/kernel.h>#include <linux/atomic.h>#include <linux/cgroup.h>#include <linux/filter.h>#include <linux/slab.h>#include <linux/sysctl.h>#include <linux/string.h>#include <linux/bpf.h>#include <linux/bpf-cgroup.h>#include <net/sock.h>#include <net/bpf_sk_storage.h> #include "../cgroup/cgroup-internal.h" DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);EXPORT_SYMBOL(cgroup_bpf_enabled_key); void cgroup_bpf_offline(struct cgroup *cgrp){	cgroup_get(cgrp);	percpu_ref_kill(&cgrp->bpf.refcnt);} /** * cgroup_bpf_release() - put references of all bpf programs and *                        release all cgroup bpf data * @work: work structure embedded into the cgroup to modify */static void cgroup_bpf_release(struct work_struct *work){	struct cgroup *cgrp = container_of(work, struct cgroup,					   bpf.release_work);	enum bpf_cgroup_storage_type stype;	struct bpf_prog_array *old_array;	unsigned int type; 	mutex_lock(&cgroup_mutex); 	for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {		struct list_head *progs = &cgrp->bpf.progs[type];		struct bpf_prog_list *pl, *tmp; 		list_for_each_entry_safe(pl, tmp, progs, node) {			list_del(&pl->node);			bpf_prog_put(pl->prog);			for_each_cgroup_storage_type(stype) {				bpf_cgroup_storage_unlink(pl->storage[stype]);				bpf_cgroup_storage_free(pl->storage[stype]);			}			kfree(pl);			static_branch_dec(&cgroup_bpf_enabled_key);		}		old_array = rcu_dereference_protected(				cgrp->bpf.effective[type],				lockdep_is_held(&cgroup_mutex));		bpf_prog_array_free(old_array);	} 	mutex_unlock(&cgroup_mutex); 	percpu_ref_exit(&cgrp->bpf.refcnt);	cgroup_put(cgrp);} /** * cgroup_bpf_release_fn() - callback used to schedule releasing *                           of bpf cgroup data * @ref: percpu ref counter structure */static void cgroup_bpf_release_fn(struct percpu_ref *ref){	struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); 	INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release);	queue_work(system_wq, &cgrp->bpf.release_work);} /* count number of elements in the list. * it's slow but the list cannot be long */static u32 prog_list_length(struct list_head *head){	struct bpf_prog_list *pl;	u32 cnt = 0; 	list_for_each_entry(pl, head, node) {		if (!pl->prog)			continue;		cnt++;	}	return cnt;} /* if parent has non-overridable prog attached, * disallow attaching new programs to the descendent cgroup. * if parent has overridable or multi-prog, allow attaching */static bool hierarchy_allows_attach(struct cgroup *cgrp,				    enum bpf_attach_type type,				    u32 new_flags){	struct cgroup *p; 	p = cgroup_parent(cgrp);	if (!p)		return true;	do {		u32 flags = p->bpf.flags[type];		u32 cnt; 		if (flags & BPF_F_ALLOW_MULTI)			return true;		cnt = prog_list_length(&p->bpf.progs[type]);		WARN_ON_ONCE(cnt > 1);		if (cnt == 1)			return !!(flags & BPF_F_ALLOW_OVERRIDE);		p = cgroup_parent(p);	} while (p);	return true;} /* compute a chain of effective programs for a given cgroup: * start from the list of programs in this cgroup and add * all parent programs. * Note that parent's F_ALLOW_OVERRIDE-type program is yielding * to programs in this cgroup */static int compute_effective_progs(struct cgroup *cgrp,				   enum bpf_attach_type type,				   struct bpf_prog_array **array){	enum bpf_cgroup_storage_type stype;	struct bpf_prog_array *progs;	struct bpf_prog_list *pl;	struct cgroup *p = cgrp;	int cnt = 0; 	/* count number of effective programs by walking parents */	do {		if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))			cnt += prog_list_length(&p->bpf.progs[type]);		p = cgroup_parent(p);	} while (p); 	progs = bpf_prog_array_alloc(cnt, GFP_KERNEL);	if (!progs)		return -ENOMEM; 	/* populate the array with effective progs */	cnt = 0;	p = cgrp;	do {		if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))			continue; 		list_for_each_entry(pl, &p->bpf.progs[type], node) {			if (!pl->prog)				continue; 			progs->items[cnt].prog = pl->prog;			for_each_cgroup_storage_type(stype)				progs->items[cnt].cgroup_storage[stype] =					pl->storage[stype];			cnt++;		}	} while ((p = cgroup_parent(p))); 	*array = progs;	return 0;} static void activate_effective_progs(struct cgroup *cgrp,				     enum bpf_attach_type type,				     struct bpf_prog_array *old_array){	rcu_swap_protected(cgrp->bpf.effective[type], old_array,			   lockdep_is_held(&cgroup_mutex));	/* free prog array after grace period, since __cgroup_bpf_run_*()	 * might be still walking the array	 */	bpf_prog_array_free(old_array);} /** * cgroup_bpf_inherit() - inherit effective programs from parent * @cgrp: the cgroup to modify */int cgroup_bpf_inherit(struct cgroup *cgrp){/* has to use marco instead of const int, since compiler thinks * that array below is variable length */#define	NR ARRAY_SIZE(cgrp->bpf.effective)	struct bpf_prog_array *arrays[NR] = {};	int ret, i; 	ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0,			      GFP_KERNEL);	if (ret)		return ret; 	for (i = 0; i < NR; i++)		INIT_LIST_HEAD(&cgrp->bpf.progs[i]); 	for (i = 0; i < NR; i++)		if (compute_effective_progs(cgrp, i, &arrays[i]))			goto cleanup; 	for (i = 0; i < NR; i++)		activate_effective_progs(cgrp, i, arrays[i]); 	return 0;cleanup:	for (i = 0; i < NR; i++)		bpf_prog_array_free(arrays[i]); 	percpu_ref_exit(&cgrp->bpf.refcnt); 	return -ENOMEM;} static int update_effective_progs(struct cgroup *cgrp,				  enum bpf_attach_type type){	struct cgroup_subsys_state *css;	int err; 	/* allocate and recompute effective prog arrays */	css_for_each_descendant_pre(css, &cgrp->self) {		struct cgroup *desc = container_of(css, struct cgroup, self); 		if (percpu_ref_is_zero(&desc->bpf.refcnt))			continue; 		err = compute_effective_progs(desc, type, &desc->bpf.inactive);		if (err)			goto cleanup;	} 	/* all allocations were successful. Activate all prog arrays */	css_for_each_descendant_pre(css, &cgrp->self) {		struct cgroup *desc = container_of(css, struct cgroup, self); 		if (percpu_ref_is_zero(&desc->bpf.refcnt)) {			if (unlikely(desc->bpf.inactive)) {				bpf_prog_array_free(desc->bpf.inactive);				desc->bpf.inactive = NULL;			}			continue;		} 		activate_effective_progs(desc, type, desc->bpf.inactive);		desc->bpf.inactive = NULL;	} 	return 0; cleanup:	/* oom while computing effective. Free all computed effective arrays	 * since they were not activated	 */	css_for_each_descendant_pre(css, &cgrp->self) {		struct cgroup *desc = container_of(css, struct cgroup, self); 		bpf_prog_array_free(desc->bpf.inactive);		desc->bpf.inactive = NULL;	} 	return err;} #define BPF_CGROUP_MAX_PROGS 64 /** * __cgroup_bpf_attach() - Attach the program to a cgroup, and *                         propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to attach * @type: Type of attach operation * @flags: Option flags * * Must be called with cgroup_mutex held. */int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,			enum bpf_attach_type type, u32 flags){	struct list_head *progs = &cgrp->bpf.progs[type];	struct bpf_prog *old_prog = NULL;	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE],		*old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL};	enum bpf_cgroup_storage_type stype;	struct bpf_prog_list *pl;	bool pl_was_allocated;	int err; 	if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI))		/* invalid combination */		return -EINVAL; 	if (!hierarchy_allows_attach(cgrp, type, flags))		return -EPERM; 	if (!list_empty(progs) && cgrp->bpf.flags[type] != flags)		/* Disallow attaching non-overridable on top		 * of existing overridable in this cgroup.		 * Disallow attaching multi-prog if overridable or none		 */		return -EPERM; 	if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)		return -E2BIG; 	for_each_cgroup_storage_type(stype) {		storage[stype] = bpf_cgroup_storage_alloc(prog, stype);		if (IS_ERR(storage[stype])) {			storage[stype] = NULL;			for_each_cgroup_storage_type(stype)				bpf_cgroup_storage_free(storage[stype]);			return -ENOMEM;		}	} 	if (flags & BPF_F_ALLOW_MULTI) {		list_for_each_entry(pl, progs, node) {			if (pl->prog == prog) {				/* disallow attaching the same prog twice */				for_each_cgroup_storage_type(stype)					bpf_cgroup_storage_free(storage[stype]);				return -EINVAL;			}		} 		pl = kmalloc(sizeof(*pl), GFP_KERNEL);		if (!pl) {			for_each_cgroup_storage_type(stype)				bpf_cgroup_storage_free(storage[stype]);			return -ENOMEM;		} 		pl_was_allocated = true;		pl->prog = prog;		for_each_cgroup_storage_type(stype)			pl->storage[stype] = storage[stype];		list_add_tail(&pl->node, progs);	} else {		if (list_empty(progs)) {			pl = kmalloc(sizeof(*pl), GFP_KERNEL);			if (!pl) {				for_each_cgroup_storage_type(stype)					bpf_cgroup_storage_free(storage[stype]);				return -ENOMEM;			}			pl_was_allocated = true;			list_add_tail(&pl->node, progs);		} else {			pl = list_first_entry(progs, typeof(*pl), node);			old_prog = pl->prog;			for_each_cgroup_storage_type(stype) {				old_storage[stype] = pl->storage[stype];				bpf_cgroup_storage_unlink(old_storage[stype]);			}			pl_was_allocated = false;		}		pl->prog = prog;		for_each_cgroup_storage_type(stype)			pl->storage[stype] = storage[stype];	} 	cgrp->bpf.flags[type] = flags; 	err = update_effective_progs(cgrp, type);	if (err)		goto cleanup; 	static_branch_inc(&cgroup_bpf_enabled_key);	for_each_cgroup_storage_type(stype) {		if (!old_storage[stype])			continue;		bpf_cgroup_storage_free(old_storage[stype]);	}	if (old_prog) {		bpf_prog_put(old_prog);		static_branch_dec(&cgroup_bpf_enabled_key);	}	for_each_cgroup_storage_type(stype)		bpf_cgroup_storage_link(storage[stype], cgrp, type);	return 0; cleanup:	/* and cleanup the prog list */	pl->prog = old_prog;	for_each_cgroup_storage_type(stype) {		bpf_cgroup_storage_free(pl->storage[stype]);		pl->storage[stype] = old_storage[stype];		bpf_cgroup_storage_link(old_storage[stype], cgrp, type);	}	if (pl_was_allocated) {		list_del(&pl->node);		kfree(pl);	}	return err;} /** * __cgroup_bpf_detach() - Detach the program from a cgroup, and *                         propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to detach or NULL * @type: Type of detach operation * * Must be called with cgroup_mutex held. */int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,			enum bpf_attach_type type){	struct list_head *progs = &cgrp->bpf.progs[type];	enum bpf_cgroup_storage_type stype;	u32 flags = cgrp->bpf.flags[type];	struct bpf_prog *old_prog = NULL;	struct bpf_prog_list *pl;	int err; 	if (flags & BPF_F_ALLOW_MULTI) {		if (!prog)			/* to detach MULTI prog the user has to specify valid FD			 * of the program to be detached			 */			return -EINVAL;	} else {		if (list_empty(progs))			/* report error when trying to detach and nothing is attached */			return -ENOENT;	} 	if (flags & BPF_F_ALLOW_MULTI) {		/* find the prog and detach it */		list_for_each_entry(pl, progs, node) {			if (pl->prog != prog)				continue;			old_prog = prog;			/* mark it deleted, so it's ignored while			 * recomputing effective			 */			pl->prog = NULL;			break;		}		if (!old_prog)			return -ENOENT;	} else {		/* to maintain backward compatibility NONE and OVERRIDE cgroups		 * allow detaching with invalid FD (prog==NULL)		 */		pl = list_first_entry(progs, typeof(*pl), node);		old_prog = pl->prog;		pl->prog = NULL;	} 	err = update_effective_progs(cgrp, type);	if (err)		goto cleanup; 	/* now can actually delete it from this cgroup list */	list_del(&pl->node);	for_each_cgroup_storage_type(stype) {		bpf_cgroup_storage_unlink(pl->storage[stype]);		bpf_cgroup_storage_free(pl->storage[stype]);	}	kfree(pl);	if (list_empty(progs))		/* last program was detached, reset flags to zero */		cgrp->bpf.flags[type] = 0; 	bpf_prog_put(old_prog);	static_branch_dec(&cgroup_bpf_enabled_key);	return 0; cleanup:	/* and restore back old_prog */	pl->prog = old_prog;	return err;} /* Must be called with cgroup_mutex held to avoid races. */int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,		       union bpf_attr __user *uattr){	__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);	enum bpf_attach_type type = attr->query.attach_type;	struct list_head *progs = &cgrp->bpf.progs[type];	u32 flags = cgrp->bpf.flags[type];	struct bpf_prog_array *effective;	int cnt, ret = 0, i; 

Lines 1–500 of 1552.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t