// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */#include <linux/bpf.h>#include <linux/bpf-cgroup.h>#include <linux/bpf_trace.h>#include <linux/bpf_lirc.h>#include <linux/bpf_verifier.h>#include <linux/bsearch.h>#include <linux/btf.h>#include <linux/syscalls.h>#include <linux/slab.h>#include <linux/sched/signal.h>#include <linux/vmalloc.h>#include <linux/mmzone.h>#include <linux/anon_inodes.h>#include <linux/fdtable.h>#include <linux/file.h>#include <linux/fs.h>#include <linux/license.h>#include <linux/filter.h>#include <linux/kernel.h>#include <linux/idr.h>#include <linux/cred.h>#include <linux/timekeeping.h>#include <linux/ctype.h>#include <linux/nospec.h>#include <linux/audit.h>#include <uapi/linux/btf.h>#include <linux/pgtable.h>#include <linux/bpf_lsm.h>#include <linux/poll.h>#include <linux/sort.h>#include <linux/bpf-netns.h>#include <linux/rcupdate_trace.h>#include <linux/memcontrol.h>#include <linux/trace_events.h>#include <net/netfilter/nf_bpf_link.h> #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \			  (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \			  (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)#define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \			IS_FD_HASH(map)) #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY) DEFINE_PER_CPU(int, bpf_prog_active);static DEFINE_IDR(prog_idr);static DEFINE_SPINLOCK(prog_idr_lock);static DEFINE_IDR(map_idr);static DEFINE_SPINLOCK(map_idr_lock);static DEFINE_IDR(link_idr);static DEFINE_SPINLOCK(link_idr_lock); int sysctl_unprivileged_bpf_disabled __read_mostly =	IS_BUILTIN(CONFIG_BPF_UNPRIV_DEFAULT_OFF) ? 2 : 0; static const struct bpf_map_ops * const bpf_map_types[] = {#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)#define BPF_MAP_TYPE(_id, _ops) \	[_id] = &_ops,#define BPF_LINK_TYPE(_id, _name)#include <linux/bpf_types.h>#undef BPF_PROG_TYPE#undef BPF_MAP_TYPE#undef BPF_LINK_TYPE}; /* * If we're handed a bigger struct than we know of, ensure all the unknown bits * are 0 - i.e. new user-space does not rely on any kernel feature extensions * we don't know about yet. * * There is a ToCToU between this function call and the following * copy_from_user() call. However, this is not a concern since this function is * meant to be a future-proofing of bits. */int bpf_check_uarg_tail_zero(bpfptr_t uaddr,			     size_t expected_size,			     size_t actual_size){	int res; 	if (unlikely(actual_size > PAGE_SIZE))	/* silly large */		return -E2BIG; 	if (actual_size <= expected_size)		return 0; 	if (uaddr.is_kernel)		res = memchr_inv(uaddr.kernel + expected_size, 0,				 actual_size - expected_size) == NULL;	else		res = check_zeroed_user(uaddr.user + expected_size,					actual_size - expected_size);	if (res < 0)		return res;	return res ? 0 : -E2BIG;} const struct bpf_map_ops bpf_map_offload_ops = {	.map_meta_equal = bpf_map_meta_equal,	.map_alloc = bpf_map_offload_map_alloc,	.map_free = bpf_map_offload_map_free,	.map_check_btf = map_check_no_btf,	.map_mem_usage = bpf_map_offload_map_mem_usage,}; static struct bpf_map *find_and_alloc_map(union bpf_attr *attr){	const struct bpf_map_ops *ops;	u32 type = attr->map_type;	struct bpf_map *map;	int err; 	if (type >= ARRAY_SIZE(bpf_map_types))		return ERR_PTR(-EINVAL);	type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));	ops = bpf_map_types[type];	if (!ops)		return ERR_PTR(-EINVAL); 	if (ops->map_alloc_check) {		err = ops->map_alloc_check(attr);		if (err)			return ERR_PTR(err);	}	if (attr->map_ifindex)		ops = &bpf_map_offload_ops;	if (!ops->map_mem_usage)		return ERR_PTR(-EINVAL);	map = ops->map_alloc(attr);	if (IS_ERR(map))		return map;	map->ops = ops;	map->map_type = type;	return map;} static void bpf_map_write_active_inc(struct bpf_map *map){	atomic64_inc(&map->writecnt);} static void bpf_map_write_active_dec(struct bpf_map *map){	atomic64_dec(&map->writecnt);} bool bpf_map_write_active(const struct bpf_map *map){	return atomic64_read(&map->writecnt) != 0;} static u32 bpf_map_value_size(const struct bpf_map *map){	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||	    map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)		return round_up(map->value_size, 8) * num_possible_cpus();	else if (IS_FD_MAP(map))		return sizeof(u32);	else		return  map->value_size;} static void maybe_wait_bpf_programs(struct bpf_map *map){	/* Wait for any running BPF programs to complete so that	 * userspace, when we return to it, knows that all programs	 * that could be running use the new map value.	 */	if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||	    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)		synchronize_rcu();} static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,				void *key, void *value, __u64 flags){	int err; 	/* Need to create a kthread, thus must support schedule */	if (bpf_map_is_offloaded(map)) {		return bpf_map_offload_update_elem(map, key, value, flags);	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||		   map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {		return map->ops->map_update_elem(map, key, value, flags);	} else if (map->map_type == BPF_MAP_TYPE_SOCKHASH ||		   map->map_type == BPF_MAP_TYPE_SOCKMAP) {		return sock_map_update_elem_sys(map, key, value, flags);	} else if (IS_FD_PROG_ARRAY(map)) {		return bpf_fd_array_map_update_elem(map, map_file, key, value,						    flags);	} 	bpf_disable_instrumentation();	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {		err = bpf_percpu_hash_update(map, key, value, flags);	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {		err = bpf_percpu_array_update(map, key, value, flags);	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {		err = bpf_percpu_cgroup_storage_update(map, key, value,						       flags);	} else if (IS_FD_ARRAY(map)) {		rcu_read_lock();		err = bpf_fd_array_map_update_elem(map, map_file, key, value,						   flags);		rcu_read_unlock();	} else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {		rcu_read_lock();		err = bpf_fd_htab_map_update_elem(map, map_file, key, value,						  flags);		rcu_read_unlock();	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {		/* rcu_read_lock() is not needed */		err = bpf_fd_reuseport_array_update_elem(map, key, value,							 flags);	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||		   map->map_type == BPF_MAP_TYPE_STACK ||		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {		err = map->ops->map_push_elem(map, value, flags);	} else {		rcu_read_lock();		err = map->ops->map_update_elem(map, key, value, flags);		rcu_read_unlock();	}	bpf_enable_instrumentation();	maybe_wait_bpf_programs(map); 	return err;} static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,			      __u64 flags){	void *ptr;	int err; 	if (bpf_map_is_offloaded(map))		return bpf_map_offload_lookup_elem(map, key, value); 	bpf_disable_instrumentation();	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {		err = bpf_percpu_hash_copy(map, key, value);	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {		err = bpf_percpu_array_copy(map, key, value);	} else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {		err = bpf_percpu_cgroup_storage_copy(map, key, value);	} else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {		err = bpf_stackmap_copy(map, key, value);	} else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {		err = bpf_fd_array_map_lookup_elem(map, key, value);	} else if (IS_FD_HASH(map)) {		err = bpf_fd_htab_map_lookup_elem(map, key, value);	} else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {		err = bpf_fd_reuseport_array_lookup_elem(map, key, value);	} else if (map->map_type == BPF_MAP_TYPE_QUEUE ||		   map->map_type == BPF_MAP_TYPE_STACK ||		   map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {		err = map->ops->map_peek_elem(map, value);	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {		/* struct_ops map requires directly updating "value" */		err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);	} else {		rcu_read_lock();		if (map->ops->map_lookup_elem_sys_only)			ptr = map->ops->map_lookup_elem_sys_only(map, key);		else			ptr = map->ops->map_lookup_elem(map, key);		if (IS_ERR(ptr)) {			err = PTR_ERR(ptr);		} else if (!ptr) {			err = -ENOENT;		} else {			err = 0;			if (flags & BPF_F_LOCK)				/* lock 'ptr' and copy everything but lock */				copy_map_value_locked(map, value, ptr, true);			else				copy_map_value(map, value, ptr);			/* mask lock and timer, since value wasn't zero inited */			check_and_init_map_value(map, value);		}		rcu_read_unlock();	} 	bpf_enable_instrumentation();	maybe_wait_bpf_programs(map); 	return err;} /* Please, do not use this function outside from the map creation path * (e.g. in map update path) without taking care of setting the active * memory cgroup (see at bpf_map_kmalloc_node() for example). */static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable){	/* We really just want to fail instead of triggering OOM killer	 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,	 * which is used for lower order allocation requests.	 *	 * It has been observed that higher order allocation requests done by	 * vmalloc with __GFP_NORETRY being set might fail due to not trying	 * to reclaim memory from the page cache, thus we set	 * __GFP_RETRY_MAYFAIL to avoid such situations.	 */ 	gfp_t gfp = bpf_memcg_flags(__GFP_NOWARN | __GFP_ZERO);	unsigned int flags = 0;	unsigned long align = 1;	void *area; 	if (size >= SIZE_MAX)		return NULL; 	/* kmalloc()'ed memory can't be mmap()'ed */	if (mmapable) {		BUG_ON(!PAGE_ALIGNED(size));		align = SHMLBA;		flags = VM_USERMAP;	} else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {		area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,				    numa_node);		if (area != NULL)			return area;	} 	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,			gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,			flags, numa_node, __builtin_return_address(0));} void *bpf_map_area_alloc(u64 size, int numa_node){	return __bpf_map_area_alloc(size, numa_node, false);} void *bpf_map_area_mmapable_alloc(u64 size, int numa_node){	return __bpf_map_area_alloc(size, numa_node, true);} void bpf_map_area_free(void *area){	kvfree(area);} static u32 bpf_map_flags_retain_permanent(u32 flags){	/* Some map creation flags are not tied to the map object but	 * rather to the map fd instead, so they have no meaning upon	 * map object inspection since multiple file descriptors with	 * different (access) properties can exist here. Thus, given	 * this has zero meaning for the map itself, lets clear these	 * from here.	 */	return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);} void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr){	map->map_type = attr->map_type;	map->key_size = attr->key_size;	map->value_size = attr->value_size;	map->max_entries = attr->max_entries;	map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);	map->numa_node = bpf_map_attr_numa_node(attr);	map->map_extra = attr->map_extra;} static int bpf_map_alloc_id(struct bpf_map *map){	int id; 	idr_preload(GFP_KERNEL);	spin_lock_bh(&map_idr_lock);	id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);	if (id > 0)		map->id = id;	spin_unlock_bh(&map_idr_lock);	idr_preload_end(); 	if (WARN_ON_ONCE(!id))		return -ENOSPC; 	return id > 0 ? 0 : id;} void bpf_map_free_id(struct bpf_map *map){	unsigned long flags; 	/* Offloaded maps are removed from the IDR store when their device	 * disappears - even if someone holds an fd to them they are unusable,	 * the memory is gone, all ops will fail; they are simply waiting for	 * refcnt to drop to be freed.	 */	if (!map->id)		return; 	spin_lock_irqsave(&map_idr_lock, flags); 	idr_remove(&map_idr, map->id);	map->id = 0; 	spin_unlock_irqrestore(&map_idr_lock, flags);} #ifdef CONFIG_MEMCG_KMEMstatic void bpf_map_save_memcg(struct bpf_map *map){	/* Currently if a map is created by a process belonging to the root	 * memory cgroup, get_obj_cgroup_from_current() will return NULL.	 * So we have to check map->objcg for being NULL each time it's	 * being used.	 */	if (memcg_bpf_enabled())		map->objcg = get_obj_cgroup_from_current();} static void bpf_map_release_memcg(struct bpf_map *map){	if (map->objcg)		obj_cgroup_put(map->objcg);} static struct mem_cgroup *bpf_map_get_memcg(const struct bpf_map *map){	if (map->objcg)		return get_mem_cgroup_from_objcg(map->objcg); 	return root_mem_cgroup;} void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,			   int node){	struct mem_cgroup *memcg, *old_memcg;	void *ptr; 	memcg = bpf_map_get_memcg(map);	old_memcg = set_active_memcg(memcg);	ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);	set_active_memcg(old_memcg);	mem_cgroup_put(memcg); 	return ptr;} void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags){	struct mem_cgroup *memcg, *old_memcg;	void *ptr; 	memcg = bpf_map_get_memcg(map);	old_memcg = set_active_memcg(memcg);	ptr = kzalloc(size, flags | __GFP_ACCOUNT);	set_active_memcg(old_memcg);	mem_cgroup_put(memcg); 	return ptr;} void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,		       gfp_t flags){	struct mem_cgroup *memcg, *old_memcg;	void *ptr; 	memcg = bpf_map_get_memcg(map);	old_memcg = set_active_memcg(memcg);	ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);	set_active_memcg(old_memcg);	mem_cgroup_put(memcg); 	return ptr;} void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,				    size_t align, gfp_t flags){	struct mem_cgroup *memcg, *old_memcg;	void __percpu *ptr; 	memcg = bpf_map_get_memcg(map);	old_memcg = set_active_memcg(memcg);	ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);	set_active_memcg(old_memcg);	mem_cgroup_put(memcg); 	return ptr;}

Lines 1–500 of 5427. Load more