// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */#include <linux/bpf.h>#include <linux/rcupdate.h>#include <linux/random.h>#include <linux/smp.h>#include <linux/topology.h>#include <linux/ktime.h>#include <linux/sched.h>#include <linux/uidgid.h>#include <linux/filter.h>#include <linux/ctype.h> #include "../../lib/kstrtox.h" /* If kernel subsystem is allowing eBPF programs to call this function, * inside its own verifier_ops->get_func_proto() callback it should return * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments * * Different map implementations will rely on rcu in map methods * lookup/update/delete, therefore eBPF programs must run under rcu lock * if program is allowed to access maps, so check rcu_read_lock_held in * all three functions. */BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key){	WARN_ON_ONCE(!rcu_read_lock_held());	return (unsigned long) map->ops->map_lookup_elem(map, key);} const struct bpf_func_proto bpf_map_lookup_elem_proto = {	.func		= bpf_map_lookup_elem,	.gpl_only	= false,	.pkt_access	= true,	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_MAP_KEY,}; BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,	   void *, value, u64, flags){	WARN_ON_ONCE(!rcu_read_lock_held());	return map->ops->map_update_elem(map, key, value, flags);} const struct bpf_func_proto bpf_map_update_elem_proto = {	.func		= bpf_map_update_elem,	.gpl_only	= false,	.pkt_access	= true,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_MAP_KEY,	.arg3_type	= ARG_PTR_TO_MAP_VALUE,	.arg4_type	= ARG_ANYTHING,}; BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key){	WARN_ON_ONCE(!rcu_read_lock_held());	return map->ops->map_delete_elem(map, key);} const struct bpf_func_proto bpf_map_delete_elem_proto = {	.func		= bpf_map_delete_elem,	.gpl_only	= false,	.pkt_access	= true,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_MAP_KEY,}; BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags){	return map->ops->map_push_elem(map, value, flags);} const struct bpf_func_proto bpf_map_push_elem_proto = {	.func		= bpf_map_push_elem,	.gpl_only	= false,	.pkt_access	= true,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_MAP_VALUE,	.arg3_type	= ARG_ANYTHING,}; BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value){	return map->ops->map_pop_elem(map, value);} const struct bpf_func_proto bpf_map_pop_elem_proto = {	.func		= bpf_map_pop_elem,	.gpl_only	= false,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_UNINIT_MAP_VALUE,}; BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value){	return map->ops->map_peek_elem(map, value);} const struct bpf_func_proto bpf_map_peek_elem_proto = {	.func		= bpf_map_pop_elem,	.gpl_only	= false,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_PTR_TO_UNINIT_MAP_VALUE,}; const struct bpf_func_proto bpf_get_prandom_u32_proto = {	.func		= bpf_user_rnd_u32,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; BPF_CALL_0(bpf_get_smp_processor_id){	return smp_processor_id();} const struct bpf_func_proto bpf_get_smp_processor_id_proto = {	.func		= bpf_get_smp_processor_id,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; BPF_CALL_0(bpf_get_numa_node_id){	return numa_node_id();} const struct bpf_func_proto bpf_get_numa_node_id_proto = {	.func		= bpf_get_numa_node_id,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; BPF_CALL_0(bpf_ktime_get_ns){	/* NMI safe access to clock monotonic */	return ktime_get_mono_fast_ns();} const struct bpf_func_proto bpf_ktime_get_ns_proto = {	.func		= bpf_ktime_get_ns,	.gpl_only	= true,	.ret_type	= RET_INTEGER,}; BPF_CALL_0(bpf_get_current_pid_tgid){	struct task_struct *task = current; 	if (unlikely(!task))		return -EINVAL; 	return (u64) task->tgid << 32 | task->pid;} const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {	.func		= bpf_get_current_pid_tgid,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; BPF_CALL_0(bpf_get_current_uid_gid){	struct task_struct *task = current;	kuid_t uid;	kgid_t gid; 	if (unlikely(!task))		return -EINVAL; 	current_uid_gid(&uid, &gid);	return (u64) from_kgid(&init_user_ns, gid) << 32 |		     from_kuid(&init_user_ns, uid);} const struct bpf_func_proto bpf_get_current_uid_gid_proto = {	.func		= bpf_get_current_uid_gid,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size){	struct task_struct *task = current; 	if (unlikely(!task))		goto err_clear; 	strncpy(buf, task->comm, size); 	/* Verifier guarantees that size > 0. For task->comm exceeding	 * size, guarantee that buf is %NUL-terminated. Unconditionally	 * done here to save the size test.	 */	buf[size - 1] = 0;	return 0;err_clear:	memset(buf, 0, size);	return -EINVAL;} const struct bpf_func_proto bpf_get_current_comm_proto = {	.func		= bpf_get_current_comm,	.gpl_only	= false,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,	.arg2_type	= ARG_CONST_SIZE,}; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) static inline void __bpf_spin_lock(struct bpf_spin_lock *lock){	arch_spinlock_t *l = (void *)lock;	union {		__u32 val;		arch_spinlock_t lock;	} u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED }; 	compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");	BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));	BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));	arch_spin_lock(l);} static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock){	arch_spinlock_t *l = (void *)lock; 	arch_spin_unlock(l);} #else static inline void __bpf_spin_lock(struct bpf_spin_lock *lock){	atomic_t *l = (void *)lock; 	BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));	do {		atomic_cond_read_relaxed(l, !VAL);	} while (atomic_xchg(l, 1));} static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock){	atomic_t *l = (void *)lock; 	atomic_set_release(l, 0);} #endif static DEFINE_PER_CPU(unsigned long, irqsave_flags); notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock){	unsigned long flags; 	local_irq_save(flags);	__bpf_spin_lock(lock);	__this_cpu_write(irqsave_flags, flags);	return 0;} const struct bpf_func_proto bpf_spin_lock_proto = {	.func		= bpf_spin_lock,	.gpl_only	= false,	.ret_type	= RET_VOID,	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,}; notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock){	unsigned long flags; 	flags = __this_cpu_read(irqsave_flags);	__bpf_spin_unlock(lock);	local_irq_restore(flags);	return 0;} const struct bpf_func_proto bpf_spin_unlock_proto = {	.func		= bpf_spin_unlock,	.gpl_only	= false,	.ret_type	= RET_VOID,	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,}; void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,			   bool lock_src){	struct bpf_spin_lock *lock; 	if (lock_src)		lock = src + map->spin_lock_off;	else		lock = dst + map->spin_lock_off;	preempt_disable();	____bpf_spin_lock(lock);	copy_map_value(map, dst, src);	____bpf_spin_unlock(lock);	preempt_enable();} #ifdef CONFIG_CGROUPSBPF_CALL_0(bpf_get_current_cgroup_id){	struct cgroup *cgrp = task_dfl_cgroup(current); 	return cgrp->kn->id.id;} const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {	.func		= bpf_get_current_cgroup_id,	.gpl_only	= false,	.ret_type	= RET_INTEGER,}; #ifdef CONFIG_CGROUP_BPFDECLARE_PER_CPU(struct bpf_cgroup_storage*,		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags){	/* flags argument is not used now,	 * but provides an ability to extend the API.	 * verifier checks that its value is correct.	 */	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);	struct bpf_cgroup_storage *storage;	void *ptr; 	storage = this_cpu_read(bpf_cgroup_storage[stype]); 	if (stype == BPF_CGROUP_STORAGE_SHARED)		ptr = &READ_ONCE(storage->buf)->data[0];	else		ptr = this_cpu_ptr(storage->percpu_buf); 	return (unsigned long)ptr;} const struct bpf_func_proto bpf_get_local_storage_proto = {	.func		= bpf_get_local_storage,	.gpl_only	= false,	.ret_type	= RET_PTR_TO_MAP_VALUE,	.arg1_type	= ARG_CONST_MAP_PTR,	.arg2_type	= ARG_ANYTHING,};#endif #define BPF_STRTOX_BASE_MASK 0x1F static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags,			  unsigned long long *res, bool *is_negative){	unsigned int base = flags & BPF_STRTOX_BASE_MASK;	const char *cur_buf = buf;	size_t cur_len = buf_len;	unsigned int consumed;	size_t val_len;	char str[64]; 	if (!buf || !buf_len || !res || !is_negative)		return -EINVAL; 	if (base != 0 && base != 8 && base != 10 && base != 16)		return -EINVAL; 	if (flags & ~BPF_STRTOX_BASE_MASK)		return -EINVAL; 	while (cur_buf < buf + buf_len && isspace(*cur_buf))		++cur_buf; 	*is_negative = (cur_buf < buf + buf_len && *cur_buf == '-');	if (*is_negative)		++cur_buf; 	consumed = cur_buf - buf;	cur_len -= consumed;	if (!cur_len)		return -EINVAL; 	cur_len = min(cur_len, sizeof(str) - 1);	memcpy(str, cur_buf, cur_len);	str[cur_len] = '\0';	cur_buf = str; 	cur_buf = _parse_integer_fixup_radix(cur_buf, &base);	val_len = _parse_integer(cur_buf, base, res); 	if (val_len & KSTRTOX_OVERFLOW)		return -ERANGE; 	if (val_len == 0)		return -EINVAL; 	cur_buf += val_len;	consumed += cur_buf - str; 	return consumed;} static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags,			 long long *res){	unsigned long long _res;	bool is_negative;	int err; 	err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);	if (err < 0)		return err;	if (is_negative) {		if ((long long)-_res > 0)			return -ERANGE;		*res = -_res;	} else {		if ((long long)_res < 0)			return -ERANGE;		*res = _res;	}	return err;} BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags,	   long *, res){	long long _res;	int err; 	err = __bpf_strtoll(buf, buf_len, flags, &_res);	if (err < 0)		return err;	if (_res != (long)_res)		return -ERANGE;	*res = _res;	return err;} const struct bpf_func_proto bpf_strtol_proto = {	.func		= bpf_strtol,	.gpl_only	= false,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_PTR_TO_MEM,	.arg2_type	= ARG_CONST_SIZE,	.arg3_type	= ARG_ANYTHING,	.arg4_type	= ARG_PTR_TO_LONG,}; BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags,	   unsigned long *, res){	unsigned long long _res;	bool is_negative;	int err; 	err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);	if (err < 0)		return err;	if (is_negative)		return -EINVAL;	if (_res != (unsigned long)_res)		return -ERANGE;	*res = _res;	return err;} const struct bpf_func_proto bpf_strtoul_proto = {	.func		= bpf_strtoul,	.gpl_only	= false,	.ret_type	= RET_INTEGER,	.arg1_type	= ARG_PTR_TO_MEM,	.arg2_type	= ARG_CONST_SIZE,	.arg3_type	= ARG_ANYTHING,	.arg4_type	= ARG_PTR_TO_LONG,};#endif