/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */#include <linux/bpf.h>#include <linux/bpf_trace.h>#include <linux/bpf_lirc.h>#include <linux/btf.h>#include <linux/syscalls.h>#include <linux/slab.h>#include <linux/sched/signal.h>#include <linux/vmalloc.h>#include <linux/mmzone.h>#include <linux/anon_inodes.h>#include <linux/fdtable.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/license.h>#include <linux/filter.h>#include <linux/version.h>#include <linux/kernel.h>#include <linux/idr.h>#include <linux/cred.h>#include <linux/timekeeping.h>#include <linux/ctype.h>#include <linux/nospec.h> #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) DEFINE_PER_CPU(int, bpf_prog_active);static DEFINE_IDR(prog_idr);static DEFINE_SPINLOCK(prog_idr_lock);static DEFINE_IDR(map_idr);static DEFINE_SPINLOCK(map_idr_lock); int sysctl_unprivileged_bpf_disabled __read_mostly; static const struct bpf_map_ops * const bpf_map_types[] = {#define BPF_PROG_TYPE(_id, _ops)#define BPF_MAP_TYPE(_id, _ops) \ [_id] = &_ops,#include <linux/bpf_types.h>#undef BPF_PROG_TYPE#undef BPF_MAP_TYPE}; /* * If we're handed a bigger struct than we know of, ensure all the unknown bits * are 0 - i.e. new user-space does not rely on any kernel feature extensions * we don't know about yet. * * There is a ToCToU between this function call and the following * copy_from_user() call. However, this is not a concern since this function is * meant to be a future-proofing of bits. */int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, size_t actual_size){ unsigned char __user *addr; unsigned char __user *end; unsigned char val; int err; if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ return -E2BIG; if (unlikely(!access_ok(uaddr, actual_size))) return -EFAULT; if (actual_size <= expected_size) return 0; addr = uaddr + expected_size; end = uaddr + actual_size; for (; addr < end; addr++) { err = get_user(val, addr); if (err) return err; if (val) return -E2BIG; } return 0;} const struct bpf_map_ops bpf_map_offload_ops = { .map_alloc = bpf_map_offload_map_alloc, .map_free = bpf_map_offload_map_free, .map_check_btf = map_check_no_btf,}; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr){ const struct bpf_map_ops *ops; u32 type = attr->map_type; struct bpf_map *map; int err; if (type >= ARRAY_SIZE(bpf_map_types)) return ERR_PTR(-EINVAL); type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); ops = bpf_map_types[type]; if (!ops) return ERR_PTR(-EINVAL); if (ops->map_alloc_check) { err = ops->map_alloc_check(attr); if (err) return ERR_PTR(err); } if (attr->map_ifindex) ops = &bpf_map_offload_ops; map = ops->map_alloc(attr); if (IS_ERR(map)) return map; map->ops = ops; map->map_type = type; return map;} void *bpf_map_area_alloc(size_t size, int numa_node){ /* We definitely need __GFP_NORETRY, so OOM killer doesn't * trigger under memory pressure as we really just want to * fail instead. */ const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; void *area; if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { area = kmalloc_node(size, GFP_USER | flags, numa_node); if (area != NULL) return area; } return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, __builtin_return_address(0));} void bpf_map_area_free(void *area){ kvfree(area);} void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr){ map->map_type = attr->map_type; map->key_size = attr->key_size; map->value_size = attr->value_size; map->max_entries = attr->max_entries; map->map_flags = attr->map_flags; map->numa_node = bpf_map_attr_numa_node(attr);} int bpf_map_precharge_memlock(u32 pages){ struct user_struct *user = get_current_user(); unsigned long memlock_limit, cur; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur = atomic_long_read(&user->locked_vm); free_uid(user); if (cur + pages > memlock_limit) return -EPERM; return 0;} static int bpf_charge_memlock(struct user_struct *user, u32 pages){ unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { atomic_long_sub(pages, &user->locked_vm); return -EPERM; } return 0;} static void bpf_uncharge_memlock(struct user_struct *user, u32 pages){ atomic_long_sub(pages, &user->locked_vm);} static int bpf_map_init_memlock(struct bpf_map *map){ struct user_struct *user = get_current_user(); int ret; ret = bpf_charge_memlock(user, map->pages); if (ret) { free_uid(user); return ret; } map->user = user; return ret;} static void bpf_map_release_memlock(struct bpf_map *map){ struct user_struct *user = map->user; bpf_uncharge_memlock(user, map->pages); free_uid(user);} int bpf_map_charge_memlock(struct bpf_map *map, u32 pages){ int ret; ret = bpf_charge_memlock(map->user, pages); if (ret) return ret; map->pages += pages; return ret;} void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages){ bpf_uncharge_memlock(map->user, pages); map->pages -= pages;} static int bpf_map_alloc_id(struct bpf_map *map){ int id; idr_preload(GFP_KERNEL); spin_lock_bh(&map_idr_lock); id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); if (id > 0) map->id = id; spin_unlock_bh(&map_idr_lock); idr_preload_end(); if (WARN_ON_ONCE(!id)) return -ENOSPC; return id > 0 ? 0 : id;} void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock){ unsigned long flags; /* Offloaded maps are removed from the IDR store when their device * disappears - even if someone holds an fd to them they are unusable, * the memory is gone, all ops will fail; they are simply waiting for * refcnt to drop to be freed. */ if (!map->id) return; if (do_idr_lock) spin_lock_irqsave(&map_idr_lock, flags); else __acquire(&map_idr_lock); idr_remove(&map_idr, map->id); map->id = 0; if (do_idr_lock) spin_unlock_irqrestore(&map_idr_lock, flags); else __release(&map_idr_lock);} /* called from workqueue */static void bpf_map_free_deferred(struct work_struct *work){ struct bpf_map *map = container_of(work, struct bpf_map, work); bpf_map_release_memlock(map); security_bpf_map_free(map); /* implementation dependent freeing */ map->ops->map_free(map);} static void bpf_map_put_uref(struct bpf_map *map){ if (atomic_dec_and_test(&map->usercnt)) { if (map->ops->map_release_uref) map->ops->map_release_uref(map); }} /* decrement map refcnt and schedule it for freeing via workqueue * (unrelying map implementation ops->map_free() might sleep) */static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock){ if (atomic_dec_and_test(&map->refcnt)) { /* bpf_map_free_id() must be called first */ bpf_map_free_id(map, do_idr_lock); btf_put(map->btf); INIT_WORK(&map->work, bpf_map_free_deferred); schedule_work(&map->work); }} void bpf_map_put(struct bpf_map *map){ __bpf_map_put(map, true);}EXPORT_SYMBOL_GPL(bpf_map_put); void bpf_map_put_with_uref(struct bpf_map *map){ bpf_map_put_uref(map); bpf_map_put(map);} static int bpf_map_release(struct inode *inode, struct file *filp){ struct bpf_map *map = filp->private_data; if (map->ops->map_release) map->ops->map_release(map, filp); bpf_map_put_with_uref(map); return 0;} #ifdef CONFIG_PROC_FSstatic void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp){ const struct bpf_map *map = filp->private_data; const struct bpf_array *array; u32 owner_prog_type = 0; u32 owner_jited = 0; if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { array = container_of(map, struct bpf_array, map); owner_prog_type = array->owner_prog_type; owner_jited = array->owner_jited; } seq_printf(m, "map_type:\t%u\n" "key_size:\t%u\n" "value_size:\t%u\n" "max_entries:\t%u\n" "map_flags:\t%#x\n" "memlock:\t%llu\n" "map_id:\t%u\n", map->map_type, map->key_size, map->value_size, map->max_entries, map->map_flags, map->pages * 1ULL << PAGE_SHIFT, map->id); if (owner_prog_type) { seq_printf(m, "owner_prog_type:\t%u\n", owner_prog_type); seq_printf(m, "owner_jited:\t%u\n", owner_jited); }}#endif static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, loff_t *ppos){ /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_READ. */ return -EINVAL;} static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, size_t siz, loff_t *ppos){ /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_WRITE. */ return -EINVAL;} const struct file_operations bpf_map_fops = {#ifdef CONFIG_PROC_FS .show_fdinfo = bpf_map_show_fdinfo,#endif .release = bpf_map_release, .read = bpf_dummy_read, .write = bpf_dummy_write,}; int bpf_map_new_fd(struct bpf_map *map, int flags){ int ret; ret = security_bpf_map(map, OPEN_FMODE(flags)); if (ret < 0) return ret; return anon_inode_getfd("bpf-map", &bpf_map_fops, map, flags | O_CLOEXEC);} int bpf_get_file_flag(int flags){ if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) return -EINVAL; if (flags & BPF_F_RDONLY) return O_RDONLY; if (flags & BPF_F_WRONLY) return O_WRONLY; return O_RDWR;} /* helper macro to check that unused fields 'union bpf_attr' are zero */#define CHECK_ATTR(CMD) \ memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ sizeof(attr->CMD##_LAST_FIELD), 0, \ sizeof(*attr) - \ offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ sizeof(attr->CMD##_LAST_FIELD)) != NULL /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. * Return 0 on success and < 0 on error. */static int bpf_obj_name_cpy(char *dst, const char *src){ const char *end = src + BPF_OBJ_NAME_LEN; memset(dst, 0, BPF_OBJ_NAME_LEN); /* Copy all isalnum() and '_' char */ while (src < end && *src) { if (!isalnum(*src) && *src != '_') return -EINVAL; *dst++ = *src++; } /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ if (src == end) return -EINVAL; return 0;} int map_check_no_btf(const struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type){ return -ENOTSUPP;} static int map_check_btf(const struct bpf_map *map, const struct btf *btf, u32 btf_key_id, u32 btf_value_id){ const struct btf_type *key_type, *value_type; u32 key_size, value_size; int ret = 0; key_type = btf_type_id_size(btf, &btf_key_id, &key_size); if (!key_type || key_size != map->key_size) return -EINVAL; value_type = btf_type_id_size(btf, &btf_value_id, &value_size); if (!value_type || value_size != map->value_size) return -EINVAL; if (map->ops->map_check_btf) ret = map->ops->map_check_btf(map, btf, key_type, value_type); return ret;} #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id/* called via syscall */static int map_create(union bpf_attr *attr){ int numa_node = bpf_map_attr_numa_node(attr); struct bpf_map *map; int f_flags; int err; err = CHECK_ATTR(BPF_MAP_CREATE); if (err) return -EINVAL; f_flags = bpf_get_file_flag(attr->map_flags);Lines 1–500 of 2684. Load more
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com * * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. */ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/bpf_lirc.h> #include <linux/btf.h> #include <linux/syscalls.h> #include <linux/slab.h> #include <linux/sched/signal.h> #include <linux/vmalloc.h> #include <linux/mmzone.h> #include <linux/anon_inodes.h> #include <linux/fdtable.h> #include <linux/file.h> #include <linux/fs.h> #include <linux/license.h> #include <linux/filter.h> #include <linux/version.h> #include <linux/kernel.h> #include <linux/idr.h> #include <linux/cred.h> #include <linux/timekeeping.h> #include <linux/ctype.h> #include <linux/nospec.h> #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) DEFINE_PER_CPU(int, bpf_prog_active); static DEFINE_IDR(prog_idr); static DEFINE_SPINLOCK(prog_idr_lock); static DEFINE_IDR(map_idr); static DEFINE_SPINLOCK(map_idr_lock); int sysctl_unprivileged_bpf_disabled __read_mostly; static const struct bpf_map_ops * const bpf_map_types[] = { #define BPF_PROG_TYPE(_id, _ops) #define BPF_MAP_TYPE(_id, _ops) \ [_id] = &_ops, #include <linux/bpf_types.h> #undef BPF_PROG_TYPE #undef BPF_MAP_TYPE }; /* * If we're handed a bigger struct than we know of, ensure all the unknown bits * are 0 - i.e. new user-space does not rely on any kernel feature extensions * we don't know about yet. * * There is a ToCToU between this function call and the following * copy_from_user() call. However, this is not a concern since this function is * meant to be a future-proofing of bits. */ int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, size_t actual_size) { unsigned char __user *addr; unsigned char __user *end; unsigned char val; int err; if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ return -E2BIG; if (unlikely(!access_ok(uaddr, actual_size))) return -EFAULT; if (actual_size <= expected_size) return 0; addr = uaddr + expected_size; end = uaddr + actual_size; for (; addr < end; addr++) { err = get_user(val, addr); if (err) return err; if (val) return -E2BIG; } return 0; } const struct bpf_map_ops bpf_map_offload_ops = { .map_alloc = bpf_map_offload_map_alloc, .map_free = bpf_map_offload_map_free, .map_check_btf = map_check_no_btf, }; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) { const struct bpf_map_ops *ops; u32 type = attr->map_type; struct bpf_map *map; int err; if (type >= ARRAY_SIZE(bpf_map_types)) return ERR_PTR(-EINVAL); type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types)); ops = bpf_map_types[type]; if (!ops) return ERR_PTR(-EINVAL); if (ops->map_alloc_check) { err = ops->map_alloc_check(attr); if (err) return ERR_PTR(err); } if (attr->map_ifindex) ops = &bpf_map_offload_ops; map = ops->map_alloc(attr); if (IS_ERR(map)) return map; map->ops = ops; map->map_type = type; return map; } void *bpf_map_area_alloc(size_t size, int numa_node) { /* We definitely need __GFP_NORETRY, so OOM killer doesn't * trigger under memory pressure as we really just want to * fail instead. */ const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; void *area; if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { area = kmalloc_node(size, GFP_USER | flags, numa_node); if (area != NULL) return area; } return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, __builtin_return_address(0)); } void bpf_map_area_free(void *area) { kvfree(area); } void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) { map->map_type = attr->map_type; map->key_size = attr->key_size; map->value_size = attr->value_size; map->max_entries = attr->max_entries; map->map_flags = attr->map_flags; map->numa_node = bpf_map_attr_numa_node(attr); } int bpf_map_precharge_memlock(u32 pages) { struct user_struct *user = get_current_user(); unsigned long memlock_limit, cur; memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur = atomic_long_read(&user->locked_vm); free_uid(user); if (cur + pages > memlock_limit) return -EPERM; return 0; } static int bpf_charge_memlock(struct user_struct *user, u32 pages) { unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { atomic_long_sub(pages, &user->locked_vm); return -EPERM; } return 0; } static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) { atomic_long_sub(pages, &user->locked_vm); } static int bpf_map_init_memlock(struct bpf_map *map) { struct user_struct *user = get_current_user(); int ret; ret = bpf_charge_memlock(user, map->pages); if (ret) { free_uid(user); return ret; } map->user = user; return ret; } static void bpf_map_release_memlock(struct bpf_map *map) { struct user_struct *user = map->user; bpf_uncharge_memlock(user, map->pages); free_uid(user); } int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) { int ret; ret = bpf_charge_memlock(map->user, pages); if (ret) return ret; map->pages += pages; return ret; } void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) { bpf_uncharge_memlock(map->user, pages); map->pages -= pages; } static int bpf_map_alloc_id(struct bpf_map *map) { int id; idr_preload(GFP_KERNEL); spin_lock_bh(&map_idr_lock); id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); if (id > 0) map->id = id; spin_unlock_bh(&map_idr_lock); idr_preload_end(); if (WARN_ON_ONCE(!id)) return -ENOSPC; return id > 0 ? 0 : id; } void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) { unsigned long flags; /* Offloaded maps are removed from the IDR store when their device * disappears - even if someone holds an fd to them they are unusable, * the memory is gone, all ops will fail; they are simply waiting for * refcnt to drop to be freed. */ if (!map->id) return; if (do_idr_lock) spin_lock_irqsave(&map_idr_lock, flags); else __acquire(&map_idr_lock); idr_remove(&map_idr, map->id); map->id = 0; if (do_idr_lock) spin_unlock_irqrestore(&map_idr_lock, flags); else __release(&map_idr_lock); } /* called from workqueue */ static void bpf_map_free_deferred(struct work_struct *work) { struct bpf_map *map = container_of(work, struct bpf_map, work); bpf_map_release_memlock(map); security_bpf_map_free(map); /* implementation dependent freeing */ map->ops->map_free(map); } static void bpf_map_put_uref(struct bpf_map *map) { if (atomic_dec_and_test(&map->usercnt)) { if (map->ops->map_release_uref) map->ops->map_release_uref(map); } } /* decrement map refcnt and schedule it for freeing via workqueue * (unrelying map implementation ops->map_free() might sleep) */ static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) { if (atomic_dec_and_test(&map->refcnt)) { /* bpf_map_free_id() must be called first */ bpf_map_free_id(map, do_idr_lock); btf_put(map->btf); INIT_WORK(&map->work, bpf_map_free_deferred); schedule_work(&map->work); } } void bpf_map_put(struct bpf_map *map) { __bpf_map_put(map, true); } EXPORT_SYMBOL_GPL(bpf_map_put); void bpf_map_put_with_uref(struct bpf_map *map) { bpf_map_put_uref(map); bpf_map_put(map); } static int bpf_map_release(struct inode *inode, struct file *filp) { struct bpf_map *map = filp->private_data; if (map->ops->map_release) map->ops->map_release(map, filp); bpf_map_put_with_uref(map); return 0; } #ifdef CONFIG_PROC_FS static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_map *map = filp->private_data; const struct bpf_array *array; u32 owner_prog_type = 0; u32 owner_jited = 0; if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { array = container_of(map, struct bpf_array, map); owner_prog_type = array->owner_prog_type; owner_jited = array->owner_jited; } seq_printf(m, "map_type:\t%u\n" "key_size:\t%u\n" "value_size:\t%u\n" "max_entries:\t%u\n" "map_flags:\t%#x\n" "memlock:\t%llu\n" "map_id:\t%u\n", map->map_type, map->key_size, map->value_size, map->max_entries, map->map_flags, map->pages * 1ULL << PAGE_SHIFT, map->id); if (owner_prog_type) { seq_printf(m, "owner_prog_type:\t%u\n", owner_prog_type); seq_printf(m, "owner_jited:\t%u\n", owner_jited); } } #endif static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, loff_t *ppos) { /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_READ. */ return -EINVAL; } static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, size_t siz, loff_t *ppos) { /* We need this handler such that alloc_file() enables * f_mode with FMODE_CAN_WRITE. */ return -EINVAL; } const struct file_operations bpf_map_fops = { #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_map_show_fdinfo, #endif .release = bpf_map_release, .read = bpf_dummy_read, .write = bpf_dummy_write, }; int bpf_map_new_fd(struct bpf_map *map, int flags) { int ret; ret = security_bpf_map(map, OPEN_FMODE(flags)); if (ret < 0) return ret; return anon_inode_getfd("bpf-map", &bpf_map_fops, map, flags | O_CLOEXEC); } int bpf_get_file_flag(int flags) { if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) return -EINVAL; if (flags & BPF_F_RDONLY) return O_RDONLY; if (flags & BPF_F_WRONLY) return O_WRONLY; return O_RDWR; } /* helper macro to check that unused fields 'union bpf_attr' are zero */ #define CHECK_ATTR(CMD) \ memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ sizeof(attr->CMD##_LAST_FIELD), 0, \ sizeof(*attr) - \ offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ sizeof(attr->CMD##_LAST_FIELD)) != NULL /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. * Return 0 on success and < 0 on error. */ static int bpf_obj_name_cpy(char *dst, const char *src) { const char *end = src + BPF_OBJ_NAME_LEN; memset(dst, 0, BPF_OBJ_NAME_LEN); /* Copy all isalnum() and '_' char */ while (src < end && *src) { if (!isalnum(*src) && *src != '_') return -EINVAL; *dst++ = *src++; } /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ if (src == end) return -EINVAL; return 0; } int map_check_no_btf(const struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type) { return -ENOTSUPP; } static int map_check_btf(const struct bpf_map *map, const struct btf *btf, u32 btf_key_id, u32 btf_value_id) { const struct btf_type *key_type, *value_type; u32 key_size, value_size; int ret = 0; key_type = btf_type_id_size(btf, &btf_key_id, &key_size); if (!key_type || key_size != map->key_size) return -EINVAL; value_type = btf_type_id_size(btf, &btf_value_id, &value_size); if (!value_type || value_size != map->value_size) return -EINVAL; if (map->ops->map_check_btf) ret = map->ops->map_check_btf(map, btf, key_type, value_type); return ret; } #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id /* called via syscall */ static int map_create(union bpf_attr *attr) { int numa_node = bpf_map_attr_numa_node(attr); struct bpf_map *map; int f_flags; int err; err = CHECK_ATTR(BPF_MAP_CREATE); if (err) return -EINVAL; f_flags = bpf_get_file_flag(attr->map_flags);