// SPDX-License-Identifier: GPL-2.0-only/* * Functions to manage eBPF programs attached to cgroups * * Copyright (c) 2016 Daniel Mack */ #include <linux/kernel.h>#include <linux/atomic.h>#include <linux/cgroup.h>#include <linux/filter.h>#include <linux/slab.h>#include <linux/sysctl.h>#include <linux/string.h>#include <linux/bpf.h>#include <linux/bpf-cgroup.h>#include <net/sock.h>#include <net/bpf_sk_storage.h> #include "../cgroup/cgroup-internal.h" DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);EXPORT_SYMBOL(cgroup_bpf_enabled_key); void cgroup_bpf_offline(struct cgroup *cgrp){ cgroup_get(cgrp); percpu_ref_kill(&cgrp->bpf.refcnt);} /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data * @work: work structure embedded into the cgroup to modify */static void cgroup_bpf_release(struct work_struct *work){ struct cgroup *cgrp = container_of(work, struct cgroup, bpf.release_work); enum bpf_cgroup_storage_type stype; struct bpf_prog_array *old_array; unsigned int type; mutex_lock(&cgroup_mutex); for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) { struct list_head *progs = &cgrp->bpf.progs[type]; struct bpf_prog_list *pl, *tmp; list_for_each_entry_safe(pl, tmp, progs, node) { list_del(&pl->node); bpf_prog_put(pl->prog); for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_unlink(pl->storage[stype]); bpf_cgroup_storage_free(pl->storage[stype]); } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key); } old_array = rcu_dereference_protected( cgrp->bpf.effective[type], lockdep_is_held(&cgroup_mutex)); bpf_prog_array_free(old_array); } mutex_unlock(&cgroup_mutex); percpu_ref_exit(&cgrp->bpf.refcnt); cgroup_put(cgrp);} /** * cgroup_bpf_release_fn() - callback used to schedule releasing * of bpf cgroup data * @ref: percpu ref counter structure */static void cgroup_bpf_release_fn(struct percpu_ref *ref){ struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release); queue_work(system_wq, &cgrp->bpf.release_work);} /* count number of elements in the list. * it's slow but the list cannot be long */static u32 prog_list_length(struct list_head *head){ struct bpf_prog_list *pl; u32 cnt = 0; list_for_each_entry(pl, head, node) { if (!pl->prog) continue; cnt++; } return cnt;} /* if parent has non-overridable prog attached, * disallow attaching new programs to the descendent cgroup. * if parent has overridable or multi-prog, allow attaching */static bool hierarchy_allows_attach(struct cgroup *cgrp, enum bpf_attach_type type, u32 new_flags){ struct cgroup *p; p = cgroup_parent(cgrp); if (!p) return true; do { u32 flags = p->bpf.flags[type]; u32 cnt; if (flags & BPF_F_ALLOW_MULTI) return true; cnt = prog_list_length(&p->bpf.progs[type]); WARN_ON_ONCE(cnt > 1); if (cnt == 1) return !!(flags & BPF_F_ALLOW_OVERRIDE); p = cgroup_parent(p); } while (p); return true;} /* compute a chain of effective programs for a given cgroup: * start from the list of programs in this cgroup and add * all parent programs. * Note that parent's F_ALLOW_OVERRIDE-type program is yielding * to programs in this cgroup */static int compute_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type, struct bpf_prog_array **array){ enum bpf_cgroup_storage_type stype; struct bpf_prog_array *progs; struct bpf_prog_list *pl; struct cgroup *p = cgrp; int cnt = 0; /* count number of effective programs by walking parents */ do { if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) cnt += prog_list_length(&p->bpf.progs[type]); p = cgroup_parent(p); } while (p); progs = bpf_prog_array_alloc(cnt, GFP_KERNEL); if (!progs) return -ENOMEM; /* populate the array with effective progs */ cnt = 0; p = cgrp; do { if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) continue; list_for_each_entry(pl, &p->bpf.progs[type], node) { if (!pl->prog) continue; progs->items[cnt].prog = pl->prog; for_each_cgroup_storage_type(stype) progs->items[cnt].cgroup_storage[stype] = pl->storage[stype]; cnt++; } } while ((p = cgroup_parent(p))); *array = progs; return 0;} static void activate_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type, struct bpf_prog_array *old_array){ rcu_swap_protected(cgrp->bpf.effective[type], old_array, lockdep_is_held(&cgroup_mutex)); /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ bpf_prog_array_free(old_array);} /** * cgroup_bpf_inherit() - inherit effective programs from parent * @cgrp: the cgroup to modify */int cgroup_bpf_inherit(struct cgroup *cgrp){/* has to use marco instead of const int, since compiler thinks * that array below is variable length */#define NR ARRAY_SIZE(cgrp->bpf.effective) struct bpf_prog_array *arrays[NR] = {}; int ret, i; ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, GFP_KERNEL); if (ret) return ret; for (i = 0; i < NR; i++) INIT_LIST_HEAD(&cgrp->bpf.progs[i]); for (i = 0; i < NR; i++) if (compute_effective_progs(cgrp, i, &arrays[i])) goto cleanup; for (i = 0; i < NR; i++) activate_effective_progs(cgrp, i, arrays[i]); return 0;cleanup: for (i = 0; i < NR; i++) bpf_prog_array_free(arrays[i]); percpu_ref_exit(&cgrp->bpf.refcnt); return -ENOMEM;} static int update_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type){ struct cgroup_subsys_state *css; int err; /* allocate and recompute effective prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; err = compute_effective_progs(desc, type, &desc->bpf.inactive); if (err) goto cleanup; } /* all allocations were successful. Activate all prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { bpf_prog_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } continue; } activate_effective_progs(desc, type, desc->bpf.inactive); desc->bpf.inactive = NULL; } return 0; cleanup: /* oom while computing effective. Free all computed effective arrays * since they were not activated */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); bpf_prog_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } return err;} #define BPF_CGROUP_MAX_PROGS 64 /** * __cgroup_bpf_attach() - Attach the program to a cgroup, and * propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to attach * @type: Type of attach operation * @flags: Option flags * * Must be called with cgroup_mutex held. */int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, enum bpf_attach_type type, u32 flags){ struct list_head *progs = &cgrp->bpf.progs[type]; struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE], *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL}; enum bpf_cgroup_storage_type stype; struct bpf_prog_list *pl; bool pl_was_allocated; int err; if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) /* invalid combination */ return -EINVAL; if (!hierarchy_allows_attach(cgrp, type, flags)) return -EPERM; if (!list_empty(progs) && cgrp->bpf.flags[type] != flags) /* Disallow attaching non-overridable on top * of existing overridable in this cgroup. * Disallow attaching multi-prog if overridable or none */ return -EPERM; if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) return -E2BIG; for_each_cgroup_storage_type(stype) { storage[stype] = bpf_cgroup_storage_alloc(prog, stype); if (IS_ERR(storage[stype])) { storage[stype] = NULL; for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } } if (flags & BPF_F_ALLOW_MULTI) { list_for_each_entry(pl, progs, node) { if (pl->prog == prog) { /* disallow attaching the same prog twice */ for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -EINVAL; } } pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } pl_was_allocated = true; pl->prog = prog; for_each_cgroup_storage_type(stype) pl->storage[stype] = storage[stype]; list_add_tail(&pl->node, progs); } else { if (list_empty(progs)) { pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } pl_was_allocated = true; list_add_tail(&pl->node, progs); } else { pl = list_first_entry(progs, typeof(*pl), node); old_prog = pl->prog; for_each_cgroup_storage_type(stype) { old_storage[stype] = pl->storage[stype]; bpf_cgroup_storage_unlink(old_storage[stype]); } pl_was_allocated = false; } pl->prog = prog; for_each_cgroup_storage_type(stype) pl->storage[stype] = storage[stype]; } cgrp->bpf.flags[type] = flags; err = update_effective_progs(cgrp, type); if (err) goto cleanup; static_branch_inc(&cgroup_bpf_enabled_key); for_each_cgroup_storage_type(stype) { if (!old_storage[stype]) continue; bpf_cgroup_storage_free(old_storage[stype]); } if (old_prog) { bpf_prog_put(old_prog); static_branch_dec(&cgroup_bpf_enabled_key); } for_each_cgroup_storage_type(stype) bpf_cgroup_storage_link(storage[stype], cgrp, type); return 0; cleanup: /* and cleanup the prog list */ pl->prog = old_prog; for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_free(pl->storage[stype]); pl->storage[stype] = old_storage[stype]; bpf_cgroup_storage_link(old_storage[stype], cgrp, type); } if (pl_was_allocated) { list_del(&pl->node); kfree(pl); } return err;} /** * __cgroup_bpf_detach() - Detach the program from a cgroup, and * propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to detach or NULL * @type: Type of detach operation * * Must be called with cgroup_mutex held. */int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, enum bpf_attach_type type){ struct list_head *progs = &cgrp->bpf.progs[type]; enum bpf_cgroup_storage_type stype; u32 flags = cgrp->bpf.flags[type]; struct bpf_prog *old_prog = NULL; struct bpf_prog_list *pl; int err; if (flags & BPF_F_ALLOW_MULTI) { if (!prog) /* to detach MULTI prog the user has to specify valid FD * of the program to be detached */ return -EINVAL; } else { if (list_empty(progs)) /* report error when trying to detach and nothing is attached */ return -ENOENT; } if (flags & BPF_F_ALLOW_MULTI) { /* find the prog and detach it */ list_for_each_entry(pl, progs, node) { if (pl->prog != prog) continue; old_prog = prog; /* mark it deleted, so it's ignored while * recomputing effective */ pl->prog = NULL; break; } if (!old_prog) return -ENOENT; } else { /* to maintain backward compatibility NONE and OVERRIDE cgroups * allow detaching with invalid FD (prog==NULL) */ pl = list_first_entry(progs, typeof(*pl), node); old_prog = pl->prog; pl->prog = NULL; } err = update_effective_progs(cgrp, type); if (err) goto cleanup; /* now can actually delete it from this cgroup list */ list_del(&pl->node); for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_unlink(pl->storage[stype]); bpf_cgroup_storage_free(pl->storage[stype]); } kfree(pl); if (list_empty(progs)) /* last program was detached, reset flags to zero */ cgrp->bpf.flags[type] = 0; bpf_prog_put(old_prog); static_branch_dec(&cgroup_bpf_enabled_key); return 0; cleanup: /* and restore back old_prog */ pl->prog = old_prog; return err;} /* Must be called with cgroup_mutex held to avoid races. */int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr){ __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); enum bpf_attach_type type = attr->query.attach_type; struct list_head *progs = &cgrp->bpf.progs[type]; u32 flags = cgrp->bpf.flags[type]; struct bpf_prog_array *effective; int cnt, ret = 0, i; Lines 1–500 of 1552. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Functions to manage eBPF programs attached to cgroups * * Copyright (c) 2016 Daniel Mack */ #include <linux/kernel.h> #include <linux/atomic.h> #include <linux/cgroup.h> #include <linux/filter.h> #include <linux/slab.h> #include <linux/sysctl.h> #include <linux/string.h> #include <linux/bpf.h> #include <linux/bpf-cgroup.h> #include <net/sock.h> #include <net/bpf_sk_storage.h> #include "../cgroup/cgroup-internal.h" DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key); EXPORT_SYMBOL(cgroup_bpf_enabled_key); void cgroup_bpf_offline(struct cgroup *cgrp) { cgroup_get(cgrp); percpu_ref_kill(&cgrp->bpf.refcnt); } /** * cgroup_bpf_release() - put references of all bpf programs and * release all cgroup bpf data * @work: work structure embedded into the cgroup to modify */ static void cgroup_bpf_release(struct work_struct *work) { struct cgroup *cgrp = container_of(work, struct cgroup, bpf.release_work); enum bpf_cgroup_storage_type stype; struct bpf_prog_array *old_array; unsigned int type; mutex_lock(&cgroup_mutex); for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) { struct list_head *progs = &cgrp->bpf.progs[type]; struct bpf_prog_list *pl, *tmp; list_for_each_entry_safe(pl, tmp, progs, node) { list_del(&pl->node); bpf_prog_put(pl->prog); for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_unlink(pl->storage[stype]); bpf_cgroup_storage_free(pl->storage[stype]); } kfree(pl); static_branch_dec(&cgroup_bpf_enabled_key); } old_array = rcu_dereference_protected( cgrp->bpf.effective[type], lockdep_is_held(&cgroup_mutex)); bpf_prog_array_free(old_array); } mutex_unlock(&cgroup_mutex); percpu_ref_exit(&cgrp->bpf.refcnt); cgroup_put(cgrp); } /** * cgroup_bpf_release_fn() - callback used to schedule releasing * of bpf cgroup data * @ref: percpu ref counter structure */ static void cgroup_bpf_release_fn(struct percpu_ref *ref) { struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release); queue_work(system_wq, &cgrp->bpf.release_work); } /* count number of elements in the list. * it's slow but the list cannot be long */ static u32 prog_list_length(struct list_head *head) { struct bpf_prog_list *pl; u32 cnt = 0; list_for_each_entry(pl, head, node) { if (!pl->prog) continue; cnt++; } return cnt; } /* if parent has non-overridable prog attached, * disallow attaching new programs to the descendent cgroup. * if parent has overridable or multi-prog, allow attaching */ static bool hierarchy_allows_attach(struct cgroup *cgrp, enum bpf_attach_type type, u32 new_flags) { struct cgroup *p; p = cgroup_parent(cgrp); if (!p) return true; do { u32 flags = p->bpf.flags[type]; u32 cnt; if (flags & BPF_F_ALLOW_MULTI) return true; cnt = prog_list_length(&p->bpf.progs[type]); WARN_ON_ONCE(cnt > 1); if (cnt == 1) return !!(flags & BPF_F_ALLOW_OVERRIDE); p = cgroup_parent(p); } while (p); return true; } /* compute a chain of effective programs for a given cgroup: * start from the list of programs in this cgroup and add * all parent programs. * Note that parent's F_ALLOW_OVERRIDE-type program is yielding * to programs in this cgroup */ static int compute_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type, struct bpf_prog_array **array) { enum bpf_cgroup_storage_type stype; struct bpf_prog_array *progs; struct bpf_prog_list *pl; struct cgroup *p = cgrp; int cnt = 0; /* count number of effective programs by walking parents */ do { if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) cnt += prog_list_length(&p->bpf.progs[type]); p = cgroup_parent(p); } while (p); progs = bpf_prog_array_alloc(cnt, GFP_KERNEL); if (!progs) return -ENOMEM; /* populate the array with effective progs */ cnt = 0; p = cgrp; do { if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) continue; list_for_each_entry(pl, &p->bpf.progs[type], node) { if (!pl->prog) continue; progs->items[cnt].prog = pl->prog; for_each_cgroup_storage_type(stype) progs->items[cnt].cgroup_storage[stype] = pl->storage[stype]; cnt++; } } while ((p = cgroup_parent(p))); *array = progs; return 0; } static void activate_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type, struct bpf_prog_array *old_array) { rcu_swap_protected(cgrp->bpf.effective[type], old_array, lockdep_is_held(&cgroup_mutex)); /* free prog array after grace period, since __cgroup_bpf_run_*() * might be still walking the array */ bpf_prog_array_free(old_array); } /** * cgroup_bpf_inherit() - inherit effective programs from parent * @cgrp: the cgroup to modify */ int cgroup_bpf_inherit(struct cgroup *cgrp) { /* has to use marco instead of const int, since compiler thinks * that array below is variable length */ #define NR ARRAY_SIZE(cgrp->bpf.effective) struct bpf_prog_array *arrays[NR] = {}; int ret, i; ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, GFP_KERNEL); if (ret) return ret; for (i = 0; i < NR; i++) INIT_LIST_HEAD(&cgrp->bpf.progs[i]); for (i = 0; i < NR; i++) if (compute_effective_progs(cgrp, i, &arrays[i])) goto cleanup; for (i = 0; i < NR; i++) activate_effective_progs(cgrp, i, arrays[i]); return 0; cleanup: for (i = 0; i < NR; i++) bpf_prog_array_free(arrays[i]); percpu_ref_exit(&cgrp->bpf.refcnt); return -ENOMEM; } static int update_effective_progs(struct cgroup *cgrp, enum bpf_attach_type type) { struct cgroup_subsys_state *css; int err; /* allocate and recompute effective prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); if (percpu_ref_is_zero(&desc->bpf.refcnt)) continue; err = compute_effective_progs(desc, type, &desc->bpf.inactive); if (err) goto cleanup; } /* all allocations were successful. Activate all prog arrays */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); if (percpu_ref_is_zero(&desc->bpf.refcnt)) { if (unlikely(desc->bpf.inactive)) { bpf_prog_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } continue; } activate_effective_progs(desc, type, desc->bpf.inactive); desc->bpf.inactive = NULL; } return 0; cleanup: /* oom while computing effective. Free all computed effective arrays * since they were not activated */ css_for_each_descendant_pre(css, &cgrp->self) { struct cgroup *desc = container_of(css, struct cgroup, self); bpf_prog_array_free(desc->bpf.inactive); desc->bpf.inactive = NULL; } return err; } #define BPF_CGROUP_MAX_PROGS 64 /** * __cgroup_bpf_attach() - Attach the program to a cgroup, and * propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to attach * @type: Type of attach operation * @flags: Option flags * * Must be called with cgroup_mutex held. */ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, enum bpf_attach_type type, u32 flags) { struct list_head *progs = &cgrp->bpf.progs[type]; struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE], *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL}; enum bpf_cgroup_storage_type stype; struct bpf_prog_list *pl; bool pl_was_allocated; int err; if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) /* invalid combination */ return -EINVAL; if (!hierarchy_allows_attach(cgrp, type, flags)) return -EPERM; if (!list_empty(progs) && cgrp->bpf.flags[type] != flags) /* Disallow attaching non-overridable on top * of existing overridable in this cgroup. * Disallow attaching multi-prog if overridable or none */ return -EPERM; if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) return -E2BIG; for_each_cgroup_storage_type(stype) { storage[stype] = bpf_cgroup_storage_alloc(prog, stype); if (IS_ERR(storage[stype])) { storage[stype] = NULL; for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } } if (flags & BPF_F_ALLOW_MULTI) { list_for_each_entry(pl, progs, node) { if (pl->prog == prog) { /* disallow attaching the same prog twice */ for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -EINVAL; } } pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } pl_was_allocated = true; pl->prog = prog; for_each_cgroup_storage_type(stype) pl->storage[stype] = storage[stype]; list_add_tail(&pl->node, progs); } else { if (list_empty(progs)) { pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { for_each_cgroup_storage_type(stype) bpf_cgroup_storage_free(storage[stype]); return -ENOMEM; } pl_was_allocated = true; list_add_tail(&pl->node, progs); } else { pl = list_first_entry(progs, typeof(*pl), node); old_prog = pl->prog; for_each_cgroup_storage_type(stype) { old_storage[stype] = pl->storage[stype]; bpf_cgroup_storage_unlink(old_storage[stype]); } pl_was_allocated = false; } pl->prog = prog; for_each_cgroup_storage_type(stype) pl->storage[stype] = storage[stype]; } cgrp->bpf.flags[type] = flags; err = update_effective_progs(cgrp, type); if (err) goto cleanup; static_branch_inc(&cgroup_bpf_enabled_key); for_each_cgroup_storage_type(stype) { if (!old_storage[stype]) continue; bpf_cgroup_storage_free(old_storage[stype]); } if (old_prog) { bpf_prog_put(old_prog); static_branch_dec(&cgroup_bpf_enabled_key); } for_each_cgroup_storage_type(stype) bpf_cgroup_storage_link(storage[stype], cgrp, type); return 0; cleanup: /* and cleanup the prog list */ pl->prog = old_prog; for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_free(pl->storage[stype]); pl->storage[stype] = old_storage[stype]; bpf_cgroup_storage_link(old_storage[stype], cgrp, type); } if (pl_was_allocated) { list_del(&pl->node); kfree(pl); } return err; } /** * __cgroup_bpf_detach() - Detach the program from a cgroup, and * propagate the change to descendants * @cgrp: The cgroup which descendants to traverse * @prog: A program to detach or NULL * @type: Type of detach operation * * Must be called with cgroup_mutex held. */ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, enum bpf_attach_type type) { struct list_head *progs = &cgrp->bpf.progs[type]; enum bpf_cgroup_storage_type stype; u32 flags = cgrp->bpf.flags[type]; struct bpf_prog *old_prog = NULL; struct bpf_prog_list *pl; int err; if (flags & BPF_F_ALLOW_MULTI) { if (!prog) /* to detach MULTI prog the user has to specify valid FD * of the program to be detached */ return -EINVAL; } else { if (list_empty(progs)) /* report error when trying to detach and nothing is attached */ return -ENOENT; } if (flags & BPF_F_ALLOW_MULTI) { /* find the prog and detach it */ list_for_each_entry(pl, progs, node) { if (pl->prog != prog) continue; old_prog = prog; /* mark it deleted, so it's ignored while * recomputing effective */ pl->prog = NULL; break; } if (!old_prog) return -ENOENT; } else { /* to maintain backward compatibility NONE and OVERRIDE cgroups * allow detaching with invalid FD (prog==NULL) */ pl = list_first_entry(progs, typeof(*pl), node); old_prog = pl->prog; pl->prog = NULL; } err = update_effective_progs(cgrp, type); if (err) goto cleanup; /* now can actually delete it from this cgroup list */ list_del(&pl->node); for_each_cgroup_storage_type(stype) { bpf_cgroup_storage_unlink(pl->storage[stype]); bpf_cgroup_storage_free(pl->storage[stype]); } kfree(pl); if (list_empty(progs)) /* last program was detached, reset flags to zero */ cgrp->bpf.flags[type] = 0; bpf_prog_put(old_prog); static_branch_dec(&cgroup_bpf_enabled_key); return 0; cleanup: /* and restore back old_prog */ pl->prog = old_prog; return err; } /* Must be called with cgroup_mutex held to avoid races. */ int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr) { __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); enum bpf_attach_type type = attr->query.attach_type; struct list_head *progs = &cgrp->bpf.progs[type]; u32 flags = cgrp->bpf.flags[type]; struct bpf_prog_array *effective; int cnt, ret = 0, i;