/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */#include <linux/bpf.h>#include <linux/bpf_trace.h>#include <linux/syscalls.h>#include <linux/slab.h>#include <linux/sched/signal.h>#include <linux/vmalloc.h>#include <linux/mmzone.h>#include <linux/anon_inodes.h>#include <linux/file.h>#include <linux/license.h>#include <linux/filter.h>#include <linux/version.h>#include <linux/kernel.h>#include <linux/idr.h>#include <linux/cred.h>#include <linux/timekeeping.h>#include <linux/ctype.h>#include <linux/nospec.h> #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) DEFINE_PER_CPU(int, bpf_prog_active);static DEFINE_IDR(prog_idr);static DEFINE_SPINLOCK(prog_idr_lock);static DEFINE_IDR(map_idr);static DEFINE_SPINLOCK(map_idr_lock); int sysctl_unprivileged_bpf_disabled __read_mostly; static const struct bpf_map_ops * const bpf_map_types[] = {#define BPF_PROG_TYPE(_id, _ops)#define BPF_MAP_TYPE(_id, _ops) \ [_id] = &_ops,#include <linux/bpf_types.h>#undef BPF_PROG_TYPE#undef BPF_MAP_TYPE}; /* * If we're handed a bigger struct than we know of, ensure all the unknown bits * are 0 - i.e. new user-space does not rely on any kernel feature extensions * we don't know about yet. * * There is a ToCToU between this function call and the following * copy_from_user() call. However, this is not a concern since this function is * meant to be a future-proofing of bits. */static int check_uarg_tail_zero(void __user *uaddr, size_t expected_size, size_t actual_size){ unsigned char __user *addr; unsigned char __user *end; unsigned char val; int err; if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ return -E2BIG; if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) return -EFAULT; if (actual_size <= expected_size) return 0; addr = uaddr + expected_size; end = uaddr + actual_size; for (; addr < end; addr++) { err = get_user(val, addr); if (err) return err; if (val) return -E2BIG; } return 0;} const struct bpf_map_ops bpf_map_offload_ops = { .map_alloc = bpf_map_offload_map_alloc, .map_free = bpf_map_offload_map_free,}; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr){ const struct bpf_map_ops *ops; u32 type = attr->map_type; struct bpf_map *map; int err; if (type >= ARRAY_SIZE(bpf_map_types)) return ERR_PTR(-EINVAL); type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); ops = bpf_map_types[type]; if (!ops) return ERR_PTR(-EINVAL); if (ops->map_alloc_check) { err = ops->map_alloc_check(attr); if (err) return ERR_PTR(err); } if (attr->map_ifindex) ops = &bpf_map_offload_ops; map = ops->map_alloc(attr); if (IS_ERR(map)) return map; map->ops = ops; map->map_type = type; return map;} void *bpf_map_area_alloc(size_t size, int numa_node){ /* We definitely need __GFP_NORETRY, so OOM killer doesn't * trigger under memory pressure as we really just want to * fail instead. */ const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; void *area; if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { area = kmalloc_node(size, GFP_USER | flags, numa_node); if (area != NULL) return area; } return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, __builtin_return_address(0));} void bpf_map_area_free(void *area){ kvfree(area);} void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr){ map->map_type = attr->map_type; map->key_size = attr->key_size; map->value_size = attr->value_size; map->max_entries = attr->max_entries; map->map_flags = attr->map_flags; map->numa_node = bpf_map_attr_numa_node(attr);} int bpf_map_precharge_memlock(u32 pages){ struct user_struct *user = get_current_user(); unsigned long memlock_limit, cur; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur = atomic_long_read(&user->locked_vm); free_uid(user); if (cur + pages > memlock_limit) return -EPERM; return 0;} static int bpf_map_charge_memlock(struct bpf_map *map){ struct user_struct *user = get_current_user(); unsigned long memlock_limit; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; atomic_long_add(map->pages, &user->locked_vm); if (atomic_long_read(&user->locked_vm) > memlock_limit) { atomic_long_sub(map->pages, &user->locked_vm); free_uid(user); return -EPERM; } map->user = user; return 0;} static void bpf_map_uncharge_memlock(struct bpf_map *map){ struct user_struct *user = map->user; atomic_long_sub(map->pages, &user->locked_vm); free_uid(user);} static int bpf_map_alloc_id(struct bpf_map *map){ int id; idr_preload(GFP_KERNEL); spin_lock_bh(&map_idr_lock); id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); if (id > 0) map->id = id; spin_unlock_bh(&map_idr_lock); idr_preload_end(); if (WARN_ON_ONCE(!id)) return -ENOSPC; return id > 0 ? 0 : id;} void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock){ unsigned long flags; /* Offloaded maps are removed from the IDR store when their device * disappears - even if someone holds an fd to them they are unusable, * the memory is gone, all ops will fail; they are simply waiting for * refcnt to drop to be freed. */ if (!map->id) return; if (do_idr_lock) spin_lock_irqsave(&map_idr_lock, flags); else __acquire(&map_idr_lock); idr_remove(&map_idr, map->id); map->id = 0; if (do_idr_lock) spin_unlock_irqrestore(&map_idr_lock, flags); else __release(&map_idr_lock);} /* called from workqueue */static void bpf_map_free_deferred(struct work_struct *work){ struct bpf_map *map = container_of(work, struct bpf_map, work); bpf_map_uncharge_memlock(map); security_bpf_map_free(map); /* implementation dependent freeing */ map->ops->map_free(map);} static void bpf_map_put_uref(struct bpf_map *map){ if (atomic_dec_and_test(&map->usercnt)) { if (map->ops->map_release_uref) map->ops->map_release_uref(map); }} /* decrement map refcnt and schedule it for freeing via workqueue * (unrelying map implementation ops->map_free() might sleep) */static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock){ if (atomic_dec_and_test(&map->refcnt)) { /* bpf_map_free_id() must be called first */ bpf_map_free_id(map, do_idr_lock); INIT_WORK(&map->work, bpf_map_free_deferred); schedule_work(&map->work); }} void bpf_map_put(struct bpf_map *map){ __bpf_map_put(map, true);} void bpf_map_put_with_uref(struct bpf_map *map){ bpf_map_put_uref(map); bpf_map_put(map);} static int bpf_map_release(struct inode *inode, struct file *filp){ struct bpf_map *map = filp->private_data; if (map->ops->map_release) map->ops->map_release(map, filp); bpf_map_put_with_uref(map); return 0;} #ifdef CONFIG_PROC_FSstatic void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp){ const struct bpf_map *map = filp->private_data; const struct bpf_array *array; u32 owner_prog_type = 0; u32 owner_jited = 0; if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { array = container_of(map, struct bpf_array, map); owner_prog_type = array->owner_prog_type; owner_jited = array->owner_jited; } seq_printf(m, "map_type:\t%u\n" "key_size:\t%u\n" "value_size:\t%u\n" "max_entries:\t%u\n" "map_flags:\t%#x\n" "memlock:\t%llu\n", map->map_type, map->key_size, map->value_size, map->max_entries, map->map_flags, map->pages * 1ULL << PAGE_SHIFT); if (owner_prog_type) { seq_printf(m, "owner_prog_type:\t%u\n", owner_prog_type); seq_printf(m, "owner_jited:\t%u\n", owner_jited); }}#endif static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, loff_t *ppos){ /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_READ. */ return -EINVAL;} static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, size_t siz, loff_t *ppos){ /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_WRITE. */ return -EINVAL;} const struct file_operations bpf_map_fops = {#ifdef CONFIG_PROC_FS .show_fdinfo = bpf_map_show_fdinfo,#endif .release = bpf_map_release, .read = bpf_dummy_read, .write = bpf_dummy_write,}; int bpf_map_new_fd(struct bpf_map *map, int flags){ int ret; ret = security_bpf_map(map, OPEN_FMODE(flags)); if (ret < 0) return ret; return anon_inode_getfd("bpf-map", &bpf_map_fops, map, flags | O_CLOEXEC);} int bpf_get_file_flag(int flags){ if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) return -EINVAL; if (flags & BPF_F_RDONLY) return O_RDONLY; if (flags & BPF_F_WRONLY) return O_WRONLY; return O_RDWR;} /* helper macro to check that unused fields 'union bpf_attr' are zero */#define CHECK_ATTR(CMD) \ memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ sizeof(attr->CMD##_LAST_FIELD), 0, \ sizeof(*attr) - \ offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ sizeof(attr->CMD##_LAST_FIELD)) != NULL /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. * Return 0 on success and < 0 on error. */static int bpf_obj_name_cpy(char *dst, const char *src){ const char *end = src + BPF_OBJ_NAME_LEN; memset(dst, 0, BPF_OBJ_NAME_LEN); /* Copy all isalnum() and '_' char */ while (src < end && *src) { if (!isalnum(*src) && *src != '_') return -EINVAL; *dst++ = *src++; } /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ if (src == end) return -EINVAL; return 0;} #define BPF_MAP_CREATE_LAST_FIELD map_ifindex/* called via syscall */static int map_create(union bpf_attr *attr){ int numa_node = bpf_map_attr_numa_node(attr); struct bpf_map *map; int f_flags; int err; err = CHECK_ATTR(BPF_MAP_CREATE); if (err) return -EINVAL; f_flags = bpf_get_file_flag(attr->map_flags); if (f_flags < 0) return f_flags; if (numa_node != NUMA_NO_NODE && ((unsigned int)numa_node >= nr_node_ids || !node_online(numa_node))) return -EINVAL; /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ map = find_and_alloc_map(attr); if (IS_ERR(map)) return PTR_ERR(map); err = bpf_obj_name_cpy(map->name, attr->map_name); if (err) goto free_map_nouncharge; atomic_set(&map->refcnt, 1); atomic_set(&map->usercnt, 1); err = security_bpf_map_alloc(map); if (err) goto free_map_nouncharge; err = bpf_map_charge_memlock(map); if (err) goto free_map_sec; err = bpf_map_alloc_id(map); if (err) goto free_map; err = bpf_map_new_fd(map, f_flags); if (err < 0) { /* failed to allocate fd. * bpf_map_put() is needed because the above * bpf_map_alloc_id() has published the map * to the userspace and the userspace may * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. */ bpf_map_put(map); return err; } trace_bpf_map_create(map, err); return err; free_map: bpf_map_uncharge_memlock(map);free_map_sec: security_bpf_map_free(map);free_map_nouncharge: map->ops->map_free(map); return err;} /* if error is returned, fd is released. * On success caller should complete fd access with matching fdput() */struct bpf_map *__bpf_map_get(struct fd f){ if (!f.file) return ERR_PTR(-EBADF); if (f.file->f_op != &bpf_map_fops) { fdput(f); return ERR_PTR(-EINVAL);Lines 1–500 of 2121. Load more
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/syscalls.h> #include <linux/slab.h> #include <linux/sched/signal.h> #include <linux/vmalloc.h> #include <linux/mmzone.h> #include <linux/anon_inodes.h> #include <linux/file.h> #include <linux/license.h> #include <linux/filter.h> #include <linux/version.h> #include <linux/kernel.h> #include <linux/idr.h> #include <linux/cred.h> #include <linux/timekeeping.h> #include <linux/ctype.h> #include <linux/nospec.h> #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) DEFINE_PER_CPU(int, bpf_prog_active); static DEFINE_IDR(prog_idr); static DEFINE_SPINLOCK(prog_idr_lock); static DEFINE_IDR(map_idr); static DEFINE_SPINLOCK(map_idr_lock); int sysctl_unprivileged_bpf_disabled __read_mostly; static const struct bpf_map_ops * const bpf_map_types[] = { #define BPF_PROG_TYPE(_id, _ops) #define BPF_MAP_TYPE(_id, _ops) \ [_id] = &_ops, #include <linux/bpf_types.h> #undef BPF_PROG_TYPE #undef BPF_MAP_TYPE }; /* * If we're handed a bigger struct than we know of, ensure all the unknown bits * are 0 - i.e. new user-space does not rely on any kernel feature extensions * we don't know about yet. * * There is a ToCToU between this function call and the following * copy_from_user() call. However, this is not a concern since this function is * meant to be a future-proofing of bits. */ static int check_uarg_tail_zero(void __user *uaddr, size_t expected_size, size_t actual_size) { unsigned char __user *addr; unsigned char __user *end; unsigned char val; int err; if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ return -E2BIG; if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) return -EFAULT; if (actual_size <= expected_size) return 0; addr = uaddr + expected_size; end = uaddr + actual_size; for (; addr < end; addr++) { err = get_user(val, addr); if (err) return err; if (val) return -E2BIG; } return 0; } const struct bpf_map_ops bpf_map_offload_ops = { .map_alloc = bpf_map_offload_map_alloc, .map_free = bpf_map_offload_map_free, }; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) { const struct bpf_map_ops *ops; u32 type = attr->map_type; struct bpf_map *map; int err; if (type >= ARRAY_SIZE(bpf_map_types)) return ERR_PTR(-EINVAL); type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); ops = bpf_map_types[type]; if (!ops) return ERR_PTR(-EINVAL); if (ops->map_alloc_check) { err = ops->map_alloc_check(attr); if (err) return ERR_PTR(err); } if (attr->map_ifindex) ops = &bpf_map_offload_ops; map = ops->map_alloc(attr); if (IS_ERR(map)) return map; map->ops = ops; map->map_type = type; return map; } void *bpf_map_area_alloc(size_t size, int numa_node) { /* We definitely need __GFP_NORETRY, so OOM killer doesn't * trigger under memory pressure as we really just want to * fail instead. */ const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; void *area; if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { area = kmalloc_node(size, GFP_USER | flags, numa_node); if (area != NULL) return area; } return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, __builtin_return_address(0)); } void bpf_map_area_free(void *area) { kvfree(area); } void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) { map->map_type = attr->map_type; map->key_size = attr->key_size; map->value_size = attr->value_size; map->max_entries = attr->max_entries; map->map_flags = attr->map_flags; map->numa_node = bpf_map_attr_numa_node(attr); } int bpf_map_precharge_memlock(u32 pages) { struct user_struct *user = get_current_user(); unsigned long memlock_limit, cur; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur = atomic_long_read(&user->locked_vm); free_uid(user); if (cur + pages > memlock_limit) return -EPERM; return 0; } static int bpf_map_charge_memlock(struct bpf_map *map) { struct user_struct *user = get_current_user(); unsigned long memlock_limit; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; atomic_long_add(map->pages, &user->locked_vm); if (atomic_long_read(&user->locked_vm) > memlock_limit) { atomic_long_sub(map->pages, &user->locked_vm); free_uid(user); return -EPERM; } map->user = user; return 0; } static void bpf_map_uncharge_memlock(struct bpf_map *map) { struct user_struct *user = map->user; atomic_long_sub(map->pages, &user->locked_vm); free_uid(user); } static int bpf_map_alloc_id(struct bpf_map *map) { int id; idr_preload(GFP_KERNEL); spin_lock_bh(&map_idr_lock); id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); if (id > 0) map->id = id; spin_unlock_bh(&map_idr_lock); idr_preload_end(); if (WARN_ON_ONCE(!id)) return -ENOSPC; return id > 0 ? 0 : id; } void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) { unsigned long flags; /* Offloaded maps are removed from the IDR store when their device * disappears - even if someone holds an fd to them they are unusable, * the memory is gone, all ops will fail; they are simply waiting for * refcnt to drop to be freed. */ if (!map->id) return; if (do_idr_lock) spin_lock_irqsave(&map_idr_lock, flags); else __acquire(&map_idr_lock); idr_remove(&map_idr, map->id); map->id = 0; if (do_idr_lock) spin_unlock_irqrestore(&map_idr_lock, flags); else __release(&map_idr_lock); } /* called from workqueue */ static void bpf_map_free_deferred(struct work_struct *work) { struct bpf_map *map = container_of(work, struct bpf_map, work); bpf_map_uncharge_memlock(map); security_bpf_map_free(map); /* implementation dependent freeing */ map->ops->map_free(map); } static void bpf_map_put_uref(struct bpf_map *map) { if (atomic_dec_and_test(&map->usercnt)) { if (map->ops->map_release_uref) map->ops->map_release_uref(map); } } /* decrement map refcnt and schedule it for freeing via workqueue * (unrelying map implementation ops->map_free() might sleep) */ static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) { if (atomic_dec_and_test(&map->refcnt)) { /* bpf_map_free_id() must be called first */ bpf_map_free_id(map, do_idr_lock); INIT_WORK(&map->work, bpf_map_free_deferred); schedule_work(&map->work); } } void bpf_map_put(struct bpf_map *map) { __bpf_map_put(map, true); } void bpf_map_put_with_uref(struct bpf_map *map) { bpf_map_put_uref(map); bpf_map_put(map); } static int bpf_map_release(struct inode *inode, struct file *filp) { struct bpf_map *map = filp->private_data; if (map->ops->map_release) map->ops->map_release(map, filp); bpf_map_put_with_uref(map); return 0; } #ifdef CONFIG_PROC_FS static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_map *map = filp->private_data; const struct bpf_array *array; u32 owner_prog_type = 0; u32 owner_jited = 0; if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { array = container_of(map, struct bpf_array, map); owner_prog_type = array->owner_prog_type; owner_jited = array->owner_jited; } seq_printf(m, "map_type:\t%u\n" "key_size:\t%u\n" "value_size:\t%u\n" "max_entries:\t%u\n" "map_flags:\t%#x\n" "memlock:\t%llu\n", map->map_type, map->key_size, map->value_size, map->max_entries, map->map_flags, map->pages * 1ULL << PAGE_SHIFT); if (owner_prog_type) { seq_printf(m, "owner_prog_type:\t%u\n", owner_prog_type); seq_printf(m, "owner_jited:\t%u\n", owner_jited); } } #endif static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, loff_t *ppos) { /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_READ. */ return -EINVAL; } static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, size_t siz, loff_t *ppos) { /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_WRITE. */ return -EINVAL; } const struct file_operations bpf_map_fops = { #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_map_show_fdinfo, #endif .release = bpf_map_release, .read = bpf_dummy_read, .write = bpf_dummy_write, }; int bpf_map_new_fd(struct bpf_map *map, int flags) { int ret; ret = security_bpf_map(map, OPEN_FMODE(flags)); if (ret < 0) return ret; return anon_inode_getfd("bpf-map", &bpf_map_fops, map, flags | O_CLOEXEC); } int bpf_get_file_flag(int flags) { if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) return -EINVAL; if (flags & BPF_F_RDONLY) return O_RDONLY; if (flags & BPF_F_WRONLY) return O_WRONLY; return O_RDWR; } /* helper macro to check that unused fields 'union bpf_attr' are zero */ #define CHECK_ATTR(CMD) \ memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ sizeof(attr->CMD##_LAST_FIELD), 0, \ sizeof(*attr) - \ offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ sizeof(attr->CMD##_LAST_FIELD)) != NULL /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. * Return 0 on success and < 0 on error. */ static int bpf_obj_name_cpy(char *dst, const char *src) { const char *end = src + BPF_OBJ_NAME_LEN; memset(dst, 0, BPF_OBJ_NAME_LEN); /* Copy all isalnum() and '_' char */ while (src < end && *src) { if (!isalnum(*src) && *src != '_') return -EINVAL; *dst++ = *src++; } /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ if (src == end) return -EINVAL; return 0; } #define BPF_MAP_CREATE_LAST_FIELD map_ifindex /* called via syscall */ static int map_create(union bpf_attr *attr) { int numa_node = bpf_map_attr_numa_node(attr); struct bpf_map *map; int f_flags; int err; err = CHECK_ATTR(BPF_MAP_CREATE); if (err) return -EINVAL; f_flags = bpf_get_file_flag(attr->map_flags); if (f_flags < 0) return f_flags; if (numa_node != NUMA_NO_NODE && ((unsigned int)numa_node >= nr_node_ids || !node_online(numa_node))) return -EINVAL; /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ map = find_and_alloc_map(attr); if (IS_ERR(map)) return PTR_ERR(map); err = bpf_obj_name_cpy(map->name, attr->map_name); if (err) goto free_map_nouncharge; atomic_set(&map->refcnt, 1); atomic_set(&map->usercnt, 1); err = security_bpf_map_alloc(map); if (err) goto free_map_nouncharge; err = bpf_map_charge_memlock(map); if (err) goto free_map_sec; err = bpf_map_alloc_id(map); if (err) goto free_map; err = bpf_map_new_fd(map, f_flags); if (err < 0) { /* failed to allocate fd. * bpf_map_put() is needed because the above * bpf_map_alloc_id() has published the map * to the userspace and the userspace may * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. */ bpf_map_put(map); return err; } trace_bpf_map_create(map, err); return err; free_map: bpf_map_uncharge_memlock(map); free_map_sec: security_bpf_map_free(map); free_map_nouncharge: map->ops->map_free(map); return err; } /* if error is returned, fd is released. * On success caller should complete fd access with matching fdput() */ struct bpf_map *__bpf_map_get(struct fd f) { if (!f.file) return ERR_PTR(-EBADF); if (f.file->f_op != &bpf_map_fops) { fdput(f); return ERR_PTR(-EINVAL);