/* SPDX-License-Identifier: GPL-2.0-only *//* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */#ifndef _LINUX_BPF_H#define _LINUX_BPF_H 1 #include <uapi/linux/bpf.h>#include <uapi/linux/filter.h> #include <linux/workqueue.h>#include <linux/file.h>#include <linux/percpu.h>#include <linux/err.h>#include <linux/rbtree_latch.h>#include <linux/numa.h>#include <linux/mm_types.h>#include <linux/wait.h>#include <linux/refcount.h>#include <linux/mutex.h>#include <linux/module.h>#include <linux/kallsyms.h>#include <linux/capability.h>#include <linux/sched/mm.h>#include <linux/slab.h>#include <linux/percpu-refcount.h>#include <linux/stddef.h>#include <linux/bpfptr.h>#include <linux/btf.h>#include <linux/rcupdate_trace.h>#include <linux/static_call.h> struct bpf_verifier_env;struct bpf_verifier_log;struct perf_event;struct bpf_prog;struct bpf_prog_aux;struct bpf_map;struct sock;struct seq_file;struct btf;struct btf_type;struct exception_table_entry;struct seq_operations;struct bpf_iter_aux_info;struct bpf_local_storage;struct bpf_local_storage_map;struct kobject;struct mem_cgroup;struct module;struct bpf_func_state;struct ftrace_ops;struct cgroup; extern struct idr btf_idr;extern spinlock_t btf_idr_lock;extern struct kobject *btf_kobj;extern struct bpf_mem_alloc bpf_global_ma;extern bool bpf_global_ma_set; typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,					struct bpf_iter_aux_info *aux);typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);typedef unsigned int (*bpf_func_t)(const void *,				   const struct bpf_insn *);struct bpf_iter_seq_info {	const struct seq_operations *seq_ops;	bpf_iter_init_seq_priv_t init_seq_private;	bpf_iter_fini_seq_priv_t fini_seq_private;	u32 seq_priv_size;}; /* map is generic key/value storage optionally accessible by eBPF programs */struct bpf_map_ops {	/* funcs callable from userspace (via syscall) */	int (*map_alloc_check)(union bpf_attr *attr);	struct bpf_map *(*map_alloc)(union bpf_attr *attr);	void (*map_release)(struct bpf_map *map, struct file *map_file);	void (*map_free)(struct bpf_map *map);	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);	void (*map_release_uref)(struct bpf_map *map);	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);	int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,				union bpf_attr __user *uattr);	int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,					  void *value, u64 flags);	int (*map_lookup_and_delete_batch)(struct bpf_map *map,					   const union bpf_attr *attr,					   union bpf_attr __user *uattr);	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,				const union bpf_attr *attr,				union bpf_attr __user *uattr);	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,				union bpf_attr __user *uattr); 	/* funcs callable from userspace and from eBPF programs */	void *(*map_lookup_elem)(struct bpf_map *map, void *key);	int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);	int (*map_delete_elem)(struct bpf_map *map, void *key);	int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);	int (*map_pop_elem)(struct bpf_map *map, void *value);	int (*map_peek_elem)(struct bpf_map *map, void *value);	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); 	/* funcs called by prog_array and perf_event_array map */	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,				int fd);	void (*map_fd_put_ptr)(void *ptr);	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);	u32 (*map_fd_sys_lookup_elem)(void *ptr);	void (*map_seq_show_elem)(struct bpf_map *map, void *key,				  struct seq_file *m);	int (*map_check_btf)(const struct bpf_map *map,			     const struct btf *btf,			     const struct btf_type *key_type,			     const struct btf_type *value_type); 	/* Prog poke tracking helpers. */	int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);	void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);	void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,			     struct bpf_prog *new); 	/* Direct value access helpers. */	int (*map_direct_value_addr)(const struct bpf_map *map,				     u64 *imm, u32 off);	int (*map_direct_value_meta)(const struct bpf_map *map,				     u64 imm, u32 *off);	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,			     struct poll_table_struct *pts); 	/* Functions called by bpf_local_storage maps */	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,					void *owner, u32 size);	void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,					   void *owner, u32 size);	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 	/* Misc helpers.*/	int (*map_redirect)(struct bpf_map *map, u64 key, u64 flags); 	/* map_meta_equal must be implemented for maps that can be	 * used as an inner map.  It is a runtime check to ensure	 * an inner map can be inserted to an outer map.	 *	 * Some properties of the inner map has been used during the	 * verification time.  When inserting an inner map at the runtime,	 * map_meta_equal has to ensure the inserting map has the same	 * properties that the verifier has used earlier.	 */	bool (*map_meta_equal)(const struct bpf_map *meta0,			       const struct bpf_map *meta1);  	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,					      struct bpf_func_state *caller,					      struct bpf_func_state *callee);	int (*map_for_each_callback)(struct bpf_map *map,				     bpf_callback_t callback_fn,				     void *callback_ctx, u64 flags); 	/* BTF id of struct allocated by map_alloc */	int *map_btf_id; 	/* bpf_iter info used to open a seq_file */	const struct bpf_iter_seq_info *iter_seq_info;}; enum {	/* Support at most 10 fields in a BTF type */	BTF_FIELDS_MAX	   = 10,}; enum btf_field_type {	BPF_SPIN_LOCK  = (1 << 0),	BPF_TIMER      = (1 << 1),	BPF_KPTR_UNREF = (1 << 2),	BPF_KPTR_REF   = (1 << 3),	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF,	BPF_LIST_HEAD  = (1 << 4),	BPF_LIST_NODE  = (1 << 5),}; struct btf_field_kptr {	struct btf *btf;	struct module *module;	btf_dtor_kfunc_t dtor;	u32 btf_id;}; struct btf_field_list_head {	struct btf *btf;	u32 value_btf_id;	u32 node_offset;	struct btf_record *value_rec;}; struct btf_field {	u32 offset;	enum btf_field_type type;	union {		struct btf_field_kptr kptr;		struct btf_field_list_head list_head;	};}; struct btf_record {	u32 cnt;	u32 field_mask;	int spin_lock_off;	int timer_off;	struct btf_field fields[];}; struct btf_field_offs {	u32 cnt;	u32 field_off[BTF_FIELDS_MAX];	u8 field_sz[BTF_FIELDS_MAX];}; struct bpf_map {	/* The first two cachelines with read-mostly members of which some	 * are also accessed in fast-path (e.g. ops, max_entries).	 */	const struct bpf_map_ops *ops ____cacheline_aligned;	struct bpf_map *inner_map_meta;#ifdef CONFIG_SECURITY	void *security;#endif	enum bpf_map_type map_type;	u32 key_size;	u32 value_size;	u32 max_entries;	u64 map_extra; /* any per-map-type extra fields */	u32 map_flags;	u32 id;	struct btf_record *record;	int numa_node;	u32 btf_key_type_id;	u32 btf_value_type_id;	u32 btf_vmlinux_value_type_id;	struct btf *btf;#ifdef CONFIG_MEMCG_KMEM	struct obj_cgroup *objcg;#endif	char name[BPF_OBJ_NAME_LEN];	struct btf_field_offs *field_offs;	/* The 3rd and 4th cacheline with misc members to avoid false sharing	 * particularly with refcounting.	 */	atomic64_t refcnt ____cacheline_aligned;	atomic64_t usercnt;	struct work_struct work;	struct mutex freeze_mutex;	atomic64_t writecnt;	/* 'Ownership' of program-containing map is claimed by the first program	 * that is going to use this map or by the first program which FD is	 * stored in the map to make sure that all callers and callees have the	 * same prog type, JITed flag and xdp_has_frags flag.	 */	struct {		spinlock_t lock;		enum bpf_prog_type type;		bool jited;		bool xdp_has_frags;	} owner;	bool bypass_spec_v1;	bool frozen; /* write-once; write-protected by freeze_mutex */}; static inline const char *btf_field_type_name(enum btf_field_type type){	switch (type) {	case BPF_SPIN_LOCK:		return "bpf_spin_lock";	case BPF_TIMER:		return "bpf_timer";	case BPF_KPTR_UNREF:	case BPF_KPTR_REF:		return "kptr";	case BPF_LIST_HEAD:		return "bpf_list_head";	case BPF_LIST_NODE:		return "bpf_list_node";	default:		WARN_ON_ONCE(1);		return "unknown";	}} static inline u32 btf_field_type_size(enum btf_field_type type){	switch (type) {	case BPF_SPIN_LOCK:		return sizeof(struct bpf_spin_lock);	case BPF_TIMER:		return sizeof(struct bpf_timer);	case BPF_KPTR_UNREF:	case BPF_KPTR_REF:		return sizeof(u64);	case BPF_LIST_HEAD:		return sizeof(struct bpf_list_head);	case BPF_LIST_NODE:		return sizeof(struct bpf_list_node);	default:		WARN_ON_ONCE(1);		return 0;	}} static inline u32 btf_field_type_align(enum btf_field_type type){	switch (type) {	case BPF_SPIN_LOCK:		return __alignof__(struct bpf_spin_lock);	case BPF_TIMER:		return __alignof__(struct bpf_timer);	case BPF_KPTR_UNREF:	case BPF_KPTR_REF:		return __alignof__(u64);	case BPF_LIST_HEAD:		return __alignof__(struct bpf_list_head);	case BPF_LIST_NODE:		return __alignof__(struct bpf_list_node);	default:		WARN_ON_ONCE(1);		return 0;	}} static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type){	if (IS_ERR_OR_NULL(rec))		return false;	return rec->field_mask & type;} static inline void bpf_obj_init(const struct btf_field_offs *foffs, void *obj){	int i; 	if (!foffs)		return;	for (i = 0; i < foffs->cnt; i++)		memset(obj + foffs->field_off[i], 0, foffs->field_sz[i]);} static inline void check_and_init_map_value(struct bpf_map *map, void *dst){	bpf_obj_init(map->field_offs, dst);} /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and * forced to use 'long' read/writes to try to atomically copy long counters. * Best-effort only.  No barriers here, since it _will_ race with concurrent * updates from BPF programs. Called from bpf syscall and mostly used with * size 8 or 16 bytes, so ask compiler to inline it. */static inline void bpf_long_memcpy(void *dst, const void *src, u32 size){	const long *lsrc = src;	long *ldst = dst; 	size /= sizeof(long);	while (size--)		*ldst++ = *lsrc++;} /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */static inline void bpf_obj_memcpy(struct btf_field_offs *foffs,				  void *dst, void *src, u32 size,				  bool long_memcpy){	u32 curr_off = 0;	int i; 	if (likely(!foffs)) {		if (long_memcpy)			bpf_long_memcpy(dst, src, round_up(size, 8));		else			memcpy(dst, src, size);		return;	} 	for (i = 0; i < foffs->cnt; i++) {		u32 next_off = foffs->field_off[i];		u32 sz = next_off - curr_off; 		memcpy(dst + curr_off, src + curr_off, sz);		curr_off += foffs->field_sz[i] + sz;	}	memcpy(dst + curr_off, src + curr_off, size - curr_off);} static inline void copy_map_value(struct bpf_map *map, void *dst, void *src){	bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, false);} static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src){	bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, true);} static inline void bpf_obj_memzero(struct btf_field_offs *foffs, void *dst, u32 size){	u32 curr_off = 0;	int i; 	if (likely(!foffs)) {		memset(dst, 0, size);		return;	} 	for (i = 0; i < foffs->cnt; i++) {		u32 next_off = foffs->field_off[i];		u32 sz = next_off - curr_off; 		memset(dst + curr_off, 0, sz);		curr_off += foffs->field_sz[i] + sz;	}	memset(dst + curr_off, 0, size - curr_off);} static inline void zero_map_value(struct bpf_map *map, void *dst){	bpf_obj_memzero(map->field_offs, dst, map->value_size);} void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,			   bool lock_src);void bpf_timer_cancel_and_free(void *timer);void bpf_list_head_free(const struct btf_field *field, void *list_head,			struct bpf_spin_lock *spin_lock); int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); struct bpf_offload_dev;struct bpf_offloaded_map; struct bpf_map_dev_ops {	int (*map_get_next_key)(struct bpf_offloaded_map *map,				void *key, void *next_key);	int (*map_lookup_elem)(struct bpf_offloaded_map *map,			       void *key, void *value);	int (*map_update_elem)(struct bpf_offloaded_map *map,			       void *key, void *value, u64 flags);	int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);}; struct bpf_offloaded_map {	struct bpf_map map;	struct net_device *netdev;	const struct bpf_map_dev_ops *dev_ops;	void *dev_priv;	struct list_head offloads;}; static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map){	return container_of(map, struct bpf_offloaded_map, map);} static inline bool bpf_map_offload_neutral(const struct bpf_map *map){	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;} static inline bool bpf_map_support_seq_show(const struct bpf_map *map){	return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&		map->ops->map_seq_show_elem;} int map_check_no_btf(const struct bpf_map *map,		     const struct btf *btf,		     const struct btf_type *key_type,		     const struct btf_type *value_type); bool bpf_map_meta_equal(const struct bpf_map *meta0,			const struct bpf_map *meta1); extern const struct bpf_map_ops bpf_map_offload_ops; /* bpf_type_flag contains a set of flags that are applicable to the values of * arg_type, ret_type and reg_type. For example, a pointer value may be null, * or a memory is read-only. We classify types into two categories: base types * and extended types. Extended types are base types combined with a type flag. * * Currently there are no more than 32 base types in arg_type, ret_type and * reg_types. */#define BPF_BASE_TYPE_BITS	8 enum bpf_type_flag {	/* PTR may be NULL. */	PTR_MAYBE_NULL		= BIT(0 + BPF_BASE_TYPE_BITS), 	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is

Lines 1–500 of 2855. Load more