// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* * Common eBPF ELF object loading operations. * * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> * Copyright (C) 2015 Huawei Inc. * Copyright (C) 2017 Nicira, Inc. * Copyright (C) 2019 Isovalent, Inc. */ #ifndef _GNU_SOURCE#define _GNU_SOURCE#endif#include <stdlib.h>#include <stdio.h>#include <stdarg.h>#include <libgen.h>#include <inttypes.h>#include <limits.h>#include <string.h>#include <unistd.h>#include <endian.h>#include <fcntl.h>#include <errno.h>#include <ctype.h>#include <asm/unistd.h>#include <linux/err.h>#include <linux/kernel.h>#include <linux/bpf.h>#include <linux/btf.h>#include <linux/filter.h>#include <linux/limits.h>#include <linux/perf_event.h>#include <linux/bpf_perf_event.h>#include <linux/ring_buffer.h>#include <sys/epoll.h>#include <sys/ioctl.h>#include <sys/mman.h>#include <sys/stat.h>#include <sys/types.h>#include <sys/vfs.h>#include <sys/utsname.h>#include <sys/resource.h>#include <libelf.h>#include <gelf.h>#include <zlib.h> #include "libbpf.h"#include "bpf.h"#include "btf.h"#include "libbpf_internal.h"#include "hashmap.h"#include "bpf_gen_internal.h"#include "zip.h" #ifndef BPF_FS_MAGIC#define BPF_FS_MAGIC		0xcafe4a11#endif #define MAX_EVENT_NAME_LEN	64 #define BPF_FS_DEFAULT_PATH "/sys/fs/bpf" #define BPF_INSN_SZ (sizeof(struct bpf_insn)) /* vsprintf() in __base_pr() uses nonliteral format string. It may break * compilation if user enables corresponding warning. Disable it explicitly. */#pragma GCC diagnostic ignored "-Wformat-nonliteral" #define __printf(a, b)	__attribute__((format(printf, a, b))) static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);static int map_set_def_max_entries(struct bpf_map *map); static const char * const attach_type_name[] = {	[BPF_CGROUP_INET_INGRESS]	= "cgroup_inet_ingress",	[BPF_CGROUP_INET_EGRESS]	= "cgroup_inet_egress",	[BPF_CGROUP_INET_SOCK_CREATE]	= "cgroup_inet_sock_create",	[BPF_CGROUP_INET_SOCK_RELEASE]	= "cgroup_inet_sock_release",	[BPF_CGROUP_SOCK_OPS]		= "cgroup_sock_ops",	[BPF_CGROUP_DEVICE]		= "cgroup_device",	[BPF_CGROUP_INET4_BIND]		= "cgroup_inet4_bind",	[BPF_CGROUP_INET6_BIND]		= "cgroup_inet6_bind",	[BPF_CGROUP_INET4_CONNECT]	= "cgroup_inet4_connect",	[BPF_CGROUP_INET6_CONNECT]	= "cgroup_inet6_connect",	[BPF_CGROUP_UNIX_CONNECT]       = "cgroup_unix_connect",	[BPF_CGROUP_INET4_POST_BIND]	= "cgroup_inet4_post_bind",	[BPF_CGROUP_INET6_POST_BIND]	= "cgroup_inet6_post_bind",	[BPF_CGROUP_INET4_GETPEERNAME]	= "cgroup_inet4_getpeername",	[BPF_CGROUP_INET6_GETPEERNAME]	= "cgroup_inet6_getpeername",	[BPF_CGROUP_UNIX_GETPEERNAME]	= "cgroup_unix_getpeername",	[BPF_CGROUP_INET4_GETSOCKNAME]	= "cgroup_inet4_getsockname",	[BPF_CGROUP_INET6_GETSOCKNAME]	= "cgroup_inet6_getsockname",	[BPF_CGROUP_UNIX_GETSOCKNAME]	= "cgroup_unix_getsockname",	[BPF_CGROUP_UDP4_SENDMSG]	= "cgroup_udp4_sendmsg",	[BPF_CGROUP_UDP6_SENDMSG]	= "cgroup_udp6_sendmsg",	[BPF_CGROUP_UNIX_SENDMSG]	= "cgroup_unix_sendmsg",	[BPF_CGROUP_SYSCTL]		= "cgroup_sysctl",	[BPF_CGROUP_UDP4_RECVMSG]	= "cgroup_udp4_recvmsg",	[BPF_CGROUP_UDP6_RECVMSG]	= "cgroup_udp6_recvmsg",	[BPF_CGROUP_UNIX_RECVMSG]	= "cgroup_unix_recvmsg",	[BPF_CGROUP_GETSOCKOPT]		= "cgroup_getsockopt",	[BPF_CGROUP_SETSOCKOPT]		= "cgroup_setsockopt",	[BPF_SK_SKB_STREAM_PARSER]	= "sk_skb_stream_parser",	[BPF_SK_SKB_STREAM_VERDICT]	= "sk_skb_stream_verdict",	[BPF_SK_SKB_VERDICT]		= "sk_skb_verdict",	[BPF_SK_MSG_VERDICT]		= "sk_msg_verdict",	[BPF_LIRC_MODE2]		= "lirc_mode2",	[BPF_FLOW_DISSECTOR]		= "flow_dissector",	[BPF_TRACE_RAW_TP]		= "trace_raw_tp",	[BPF_TRACE_FENTRY]		= "trace_fentry",	[BPF_TRACE_FEXIT]		= "trace_fexit",	[BPF_MODIFY_RETURN]		= "modify_return",	[BPF_TRACE_FSESSION]		= "trace_fsession",	[BPF_LSM_MAC]			= "lsm_mac",	[BPF_LSM_CGROUP]		= "lsm_cgroup",	[BPF_SK_LOOKUP]			= "sk_lookup",	[BPF_TRACE_ITER]		= "trace_iter",	[BPF_XDP_DEVMAP]		= "xdp_devmap",	[BPF_XDP_CPUMAP]		= "xdp_cpumap",	[BPF_XDP]			= "xdp",	[BPF_SK_REUSEPORT_SELECT]	= "sk_reuseport_select",	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]	= "sk_reuseport_select_or_migrate",	[BPF_PERF_EVENT]		= "perf_event",	[BPF_TRACE_KPROBE_MULTI]	= "trace_kprobe_multi",	[BPF_STRUCT_OPS]		= "struct_ops",	[BPF_NETFILTER]			= "netfilter",	[BPF_TCX_INGRESS]		= "tcx_ingress",	[BPF_TCX_EGRESS]		= "tcx_egress",	[BPF_TRACE_UPROBE_MULTI]	= "trace_uprobe_multi",	[BPF_NETKIT_PRIMARY]		= "netkit_primary",	[BPF_NETKIT_PEER]		= "netkit_peer",	[BPF_TRACE_KPROBE_SESSION]	= "trace_kprobe_session",	[BPF_TRACE_UPROBE_SESSION]	= "trace_uprobe_session",	[BPF_TRACE_FENTRY_MULTI]	= "trace_fentry_multi",	[BPF_TRACE_FEXIT_MULTI]		= "trace_fexit_multi",	[BPF_TRACE_FSESSION_MULTI]	= "trace_fsession_multi",}; static const char * const link_type_name[] = {	[BPF_LINK_TYPE_UNSPEC]			= "unspec",	[BPF_LINK_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",	[BPF_LINK_TYPE_TRACING]			= "tracing",	[BPF_LINK_TYPE_CGROUP]			= "cgroup",	[BPF_LINK_TYPE_ITER]			= "iter",	[BPF_LINK_TYPE_NETNS]			= "netns",	[BPF_LINK_TYPE_XDP]			= "xdp",	[BPF_LINK_TYPE_PERF_EVENT]		= "perf_event",	[BPF_LINK_TYPE_KPROBE_MULTI]		= "kprobe_multi",	[BPF_LINK_TYPE_STRUCT_OPS]		= "struct_ops",	[BPF_LINK_TYPE_NETFILTER]		= "netfilter",	[BPF_LINK_TYPE_TCX]			= "tcx",	[BPF_LINK_TYPE_UPROBE_MULTI]		= "uprobe_multi",	[BPF_LINK_TYPE_NETKIT]			= "netkit",	[BPF_LINK_TYPE_SOCKMAP]			= "sockmap",	[BPF_LINK_TYPE_TRACING_MULTI]		= "tracing_multi",}; static const char * const map_type_name[] = {	[BPF_MAP_TYPE_UNSPEC]			= "unspec",	[BPF_MAP_TYPE_HASH]			= "hash",	[BPF_MAP_TYPE_ARRAY]			= "array",	[BPF_MAP_TYPE_PROG_ARRAY]		= "prog_array",	[BPF_MAP_TYPE_PERF_EVENT_ARRAY]		= "perf_event_array",	[BPF_MAP_TYPE_PERCPU_HASH]		= "percpu_hash",	[BPF_MAP_TYPE_PERCPU_ARRAY]		= "percpu_array",	[BPF_MAP_TYPE_STACK_TRACE]		= "stack_trace",	[BPF_MAP_TYPE_CGROUP_ARRAY]		= "cgroup_array",	[BPF_MAP_TYPE_LRU_HASH]			= "lru_hash",	[BPF_MAP_TYPE_LRU_PERCPU_HASH]		= "lru_percpu_hash",	[BPF_MAP_TYPE_LPM_TRIE]			= "lpm_trie",	[BPF_MAP_TYPE_ARRAY_OF_MAPS]		= "array_of_maps",	[BPF_MAP_TYPE_HASH_OF_MAPS]		= "hash_of_maps",	[BPF_MAP_TYPE_DEVMAP]			= "devmap",	[BPF_MAP_TYPE_DEVMAP_HASH]		= "devmap_hash",	[BPF_MAP_TYPE_SOCKMAP]			= "sockmap",	[BPF_MAP_TYPE_CPUMAP]			= "cpumap",	[BPF_MAP_TYPE_XSKMAP]			= "xskmap",	[BPF_MAP_TYPE_SOCKHASH]			= "sockhash",	[BPF_MAP_TYPE_CGROUP_STORAGE]		= "cgroup_storage",	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]	= "reuseport_sockarray",	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]	= "percpu_cgroup_storage",	[BPF_MAP_TYPE_QUEUE]			= "queue",	[BPF_MAP_TYPE_STACK]			= "stack",	[BPF_MAP_TYPE_SK_STORAGE]		= "sk_storage",	[BPF_MAP_TYPE_STRUCT_OPS]		= "struct_ops",	[BPF_MAP_TYPE_RINGBUF]			= "ringbuf",	[BPF_MAP_TYPE_INODE_STORAGE]		= "inode_storage",	[BPF_MAP_TYPE_TASK_STORAGE]		= "task_storage",	[BPF_MAP_TYPE_BLOOM_FILTER]		= "bloom_filter",	[BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",	[BPF_MAP_TYPE_CGRP_STORAGE]		= "cgrp_storage",	[BPF_MAP_TYPE_ARENA]			= "arena",	[BPF_MAP_TYPE_INSN_ARRAY]		= "insn_array",	[BPF_MAP_TYPE_RHASH]			= "rhash",}; static const char * const prog_type_name[] = {	[BPF_PROG_TYPE_UNSPEC]			= "unspec",	[BPF_PROG_TYPE_SOCKET_FILTER]		= "socket_filter",	[BPF_PROG_TYPE_KPROBE]			= "kprobe",	[BPF_PROG_TYPE_SCHED_CLS]		= "sched_cls",	[BPF_PROG_TYPE_SCHED_ACT]		= "sched_act",	[BPF_PROG_TYPE_TRACEPOINT]		= "tracepoint",	[BPF_PROG_TYPE_XDP]			= "xdp",	[BPF_PROG_TYPE_PERF_EVENT]		= "perf_event",	[BPF_PROG_TYPE_CGROUP_SKB]		= "cgroup_skb",	[BPF_PROG_TYPE_CGROUP_SOCK]		= "cgroup_sock",	[BPF_PROG_TYPE_LWT_IN]			= "lwt_in",	[BPF_PROG_TYPE_LWT_OUT]			= "lwt_out",	[BPF_PROG_TYPE_LWT_XMIT]		= "lwt_xmit",	[BPF_PROG_TYPE_SOCK_OPS]		= "sock_ops",	[BPF_PROG_TYPE_SK_SKB]			= "sk_skb",	[BPF_PROG_TYPE_CGROUP_DEVICE]		= "cgroup_device",	[BPF_PROG_TYPE_SK_MSG]			= "sk_msg",	[BPF_PROG_TYPE_RAW_TRACEPOINT]		= "raw_tracepoint",	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR]	= "cgroup_sock_addr",	[BPF_PROG_TYPE_LWT_SEG6LOCAL]		= "lwt_seg6local",	[BPF_PROG_TYPE_LIRC_MODE2]		= "lirc_mode2",	[BPF_PROG_TYPE_SK_REUSEPORT]		= "sk_reuseport",	[BPF_PROG_TYPE_FLOW_DISSECTOR]		= "flow_dissector",	[BPF_PROG_TYPE_CGROUP_SYSCTL]		= "cgroup_sysctl",	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE]	= "raw_tracepoint_writable",	[BPF_PROG_TYPE_CGROUP_SOCKOPT]		= "cgroup_sockopt",	[BPF_PROG_TYPE_TRACING]			= "tracing",	[BPF_PROG_TYPE_STRUCT_OPS]		= "struct_ops",	[BPF_PROG_TYPE_EXT]			= "ext",	[BPF_PROG_TYPE_LSM]			= "lsm",	[BPF_PROG_TYPE_SK_LOOKUP]		= "sk_lookup",	[BPF_PROG_TYPE_SYSCALL]			= "syscall",	[BPF_PROG_TYPE_NETFILTER]		= "netfilter",}; static int __base_pr(enum libbpf_print_level level, const char *format,		     va_list args){	const char *env_var = "LIBBPF_LOG_LEVEL";	static enum libbpf_print_level min_level = LIBBPF_INFO;	static bool initialized; 	if (!initialized) {		char *verbosity; 		initialized = true;		verbosity = getenv(env_var);		if (verbosity) {			if (strcasecmp(verbosity, "warn") == 0)				min_level = LIBBPF_WARN;			else if (strcasecmp(verbosity, "debug") == 0)				min_level = LIBBPF_DEBUG;			else if (strcasecmp(verbosity, "info") == 0)				min_level = LIBBPF_INFO;			else				fprintf(stderr, "libbpf: unrecognized '%s' envvar value: '%s', should be one of 'warn', 'debug', or 'info'.\n",					env_var, verbosity);		}	} 	/* if too verbose, skip logging  */	if (level > min_level)		return 0; 	return vfprintf(stderr, format, args);} static libbpf_print_fn_t __libbpf_pr = __base_pr; libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn){	libbpf_print_fn_t old_print_fn; 	old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); 	return old_print_fn;} __printf(2, 3)void libbpf_print(enum libbpf_print_level level, const char *format, ...){	va_list args;	int old_errno;	libbpf_print_fn_t print_fn; 	print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);	if (!print_fn)		return; 	old_errno = errno; 	va_start(args, format);	print_fn(level, format, args);	va_end(args); 	errno = old_errno;} static void pr_perm_msg(int err){	struct rlimit limit;	char buf[100]; 	if (err != -EPERM || geteuid() != 0)		return; 	err = getrlimit(RLIMIT_MEMLOCK, &limit);	if (err)		return; 	if (limit.rlim_cur == RLIM_INFINITY)		return; 	if (limit.rlim_cur < 1024)		snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);	else if (limit.rlim_cur < 1024*1024)		snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);	else		snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 	pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",		buf);} /* Copied from tools/perf/util/util.h */#ifndef zfree# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })#endif #ifndef zclose# define zclose(fd) ({			\	int ___err = 0;			\	if ((fd) >= 0)			\		___err = close((fd));	\	fd = -1;			\	___err; })#endif static inline __u64 ptr_to_u64(const void *ptr){	return (__u64) (unsigned long) ptr;} int libbpf_set_strict_mode(enum libbpf_strict_mode mode){	/* as of v1.0 libbpf_set_strict_mode() is a no-op */	return 0;} __u32 libbpf_major_version(void){	return LIBBPF_MAJOR_VERSION;} __u32 libbpf_minor_version(void){	return LIBBPF_MINOR_VERSION;} const char *libbpf_version_string(void){#define __S(X) #X#define _S(X) __S(X)	return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);#undef _S#undef __S} enum reloc_type {	RELO_LD64,	RELO_CALL,	RELO_DATA,	RELO_EXTERN_LD64,	RELO_EXTERN_CALL,	RELO_SUBPROG_ADDR,	RELO_CORE,	RELO_INSN_ARRAY,}; struct reloc_desc {	enum reloc_type type;	int insn_idx;	union {		const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */		struct {			int map_idx;			unsigned int sym_off;			/*			 * The following two fields can be unionized, as the			 * ext_idx field is used for extern symbols, and the			 * sym_size is used for jump tables, which are never			 * extern			 */			union {				int ext_idx;				int sym_size;			};		};	};}; /* stored as sec_def->cookie for all libbpf-supported SEC()s */enum sec_def_flags {	SEC_NONE = 0,	/* expected_attach_type is optional, if kernel doesn't support that */	SEC_EXP_ATTACH_OPT = 1,	/* legacy, only used by libbpf_get_type_names() and	 * libbpf_attach_type_by_name(), not used by libbpf itself at all.	 * This used to be associated with cgroup (and few other) BPF programs	 * that were attachable through BPF_PROG_ATTACH command. Pretty	 * meaningless nowadays, though.	 */	SEC_ATTACHABLE = 2,	SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,	/* attachment target is specified through BTF ID in either kernel or	 * other BPF program's BTF object	 */	SEC_ATTACH_BTF = 4,	/* BPF program type allows sleeping/blocking in kernel */	SEC_SLEEPABLE = 8,	/* BPF program support non-linear XDP buffer */	SEC_XDP_FRAGS = 16,	/* Setup proper attach type for usdt probes. */	SEC_USDT = 32,}; struct bpf_sec_def {	char *sec;	enum bpf_prog_type prog_type;	enum bpf_attach_type expected_attach_type;	long cookie;	int handler_id; 	libbpf_prog_setup_fn_t prog_setup_fn;	libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;	libbpf_prog_attach_fn_t prog_attach_fn;}; struct bpf_light_subprog {	__u32 sec_insn_off;	__u32 sub_insn_off;}; /* * bpf_prog should be a better name but it has been used in * linux/filter.h. */struct bpf_program {	char *name;	char *sec_name;	size_t sec_idx;	const struct bpf_sec_def *sec_def;	/* this program's instruction offset (in number of instructions)	 * within its containing ELF section	 */	size_t sec_insn_off;	/* number of original instructions in ELF section belonging to this	 * program, not taking into account subprogram instructions possible	 * appended later during relocation	 */	size_t sec_insn_cnt;	/* Offset (in number of instructions) of the start of instruction	 * belonging to this BPF program  within its containing main BPF	 * program. For the entry-point (main) BPF program, this is always	 * zero. For a sub-program, this gets reset before each of main BPF	 * programs are processed and relocated and is used to determined	 * whether sub-program was already appended to the main program, and	 * if yes, at which instruction offset.	 */	size_t sub_insn_off; 	/* instructions that belong to BPF program; insns[0] is located at	 * sec_insn_off instruction within its ELF section in ELF file, so	 * when mapping ELF file instruction index to the local instruction,	 * one needs to subtract sec_insn_off; and vice versa.	 */	struct bpf_insn *insns;	/* actual number of instruction in this BPF program's image; for	 * entry-point BPF programs this includes the size of main program	 * itself plus all the used sub-programs, appended at the end	 */	size_t insns_cnt; 	struct reloc_desc *reloc_desc;	int nr_reloc; 	/* BPF verifier log settings */	char *log_buf;	size_t log_size;	__u32 log_level; 	struct bpf_object *obj; 	int fd;	bool autoload;	bool autoattach;	bool sym_global;	bool mark_btf_static;

Lines 1–500 of 15223. Load more