// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* * common eBPF ELF operations. * * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> * Copyright (C) 2015 Huawei Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License (not later!) * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see <http://www.gnu.org/licenses> */ #include <stdlib.h>#include <string.h>#include <memory.h>#include <unistd.h>#include <asm/unistd.h>#include <errno.h>#include <linux/bpf.h>#include <linux/filter.h>#include <linux/kernel.h>#include <limits.h>#include <sys/resource.h>#include "bpf.h"#include "libbpf.h"#include "libbpf_internal.h" /* * When building perf, unistd.h is overridden. __NR_bpf is * required to be defined explicitly. */#ifndef __NR_bpf# if defined(__i386__)# define __NR_bpf 357# elif defined(__x86_64__)# define __NR_bpf 321# elif defined(__aarch64__)# define __NR_bpf 280# elif defined(__sparc__)# define __NR_bpf 349# elif defined(__s390__)# define __NR_bpf 351# elif defined(__arc__)# define __NR_bpf 280# elif defined(__mips__) && defined(_ABIO32)# define __NR_bpf 4355# elif defined(__mips__) && defined(_ABIN32)# define __NR_bpf 6319# elif defined(__mips__) && defined(_ABI64)# define __NR_bpf 5315# elif defined(__loongarch__)# define __NR_bpf 280# else# error __NR_bpf not defined. libbpf does not support your arch.# endif#endif static inline __u64 ptr_to_u64(const void *ptr){ return (__u64) (unsigned long) ptr;} static inline int sys_bpf_ext(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size, struct bpf_common_attr *attr_common, unsigned int size_common){ cmd = attr_common ? (cmd | BPF_COMMON_ATTRS) : (cmd & ~BPF_COMMON_ATTRS); return syscall(__NR_bpf, cmd, attr, size, attr_common, size_common);} static inline int sys_bpf_ext_fd(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size, struct bpf_common_attr *attr_common, unsigned int size_common){ int fd; fd = sys_bpf_ext(cmd, attr, size, attr_common, size_common); return ensure_good_fd(fd);} int probe_sys_bpf_ext(void){ const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); union bpf_attr attr; int fd; memset(&attr, 0, attr_sz); fd = syscall(__NR_bpf, BPF_PROG_LOAD | BPF_COMMON_ATTRS, &attr, attr_sz, NULL, sizeof(struct bpf_common_attr)); if (fd >= 0) { close(fd); return -EINVAL; } return errno == EFAULT ? 1 : 0;} static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size){ return syscall(__NR_bpf, cmd, attr, size);} static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size){ int fd; fd = sys_bpf(cmd, attr, size); return ensure_good_fd(fd);} int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts){ int fd; do { fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size); } while (fd < 0 && errno == EAGAIN && --attempts > 0); return fd;} /* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to * memcg-based memory accounting for BPF maps and progs. This was done in [0]. * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF. * * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") */int probe_memcg_account(int token_fd){ const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); struct bpf_insn insns[] = { BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), BPF_EXIT_INSN(), }; size_t insn_cnt = ARRAY_SIZE(insns); union bpf_attr attr; int prog_fd; /* attempt loading freplace trying to use custom BTF */ memset(&attr, 0, attr_sz); attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; attr.insns = ptr_to_u64(insns); attr.insn_cnt = insn_cnt; attr.license = ptr_to_u64("GPL"); attr.prog_token_fd = token_fd; if (token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); if (prog_fd >= 0) { close(prog_fd); return 1; } return 0;} static bool memlock_bumped;static rlim_t memlock_rlim = RLIM_INFINITY; int libbpf_set_memlock_rlim(size_t memlock_bytes){ if (memlock_bumped) return libbpf_err(-EBUSY); memlock_rlim = memlock_bytes; return 0;} int bump_rlimit_memlock(void){ struct rlimit rlim; /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT)) return 0; memlock_bumped = true; /* zero memlock_rlim disables auto-bumping RLIMIT_MEMLOCK */ if (memlock_rlim == 0) return 0; rlim.rlim_cur = rlim.rlim_max = memlock_rlim; if (setrlimit(RLIMIT_MEMLOCK, &rlim)) return -errno; return 0;} int bpf_map_create(enum bpf_map_type map_type, const char *map_name, __u32 key_size, __u32 value_size, __u32 max_entries, const struct bpf_map_create_opts *opts){ const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); const size_t attr_common_sz = sizeof(struct bpf_common_attr); struct bpf_common_attr attr_common; struct bpf_log_opts *log_opts; union bpf_attr attr; int fd; bump_rlimit_memlock(); memset(&attr, 0, attr_sz); if (!OPTS_VALID(opts, bpf_map_create_opts)) return libbpf_err(-EINVAL); attr.map_type = map_type; if (map_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; attr.max_entries = max_entries; attr.btf_fd = OPTS_GET(opts, btf_fd, 0); attr.btf_key_type_id = OPTS_GET(opts, btf_key_type_id, 0); attr.btf_value_type_id = OPTS_GET(opts, btf_value_type_id, 0); attr.btf_vmlinux_value_type_id = OPTS_GET(opts, btf_vmlinux_value_type_id, 0); attr.value_type_btf_obj_fd = OPTS_GET(opts, value_type_btf_obj_fd, 0); attr.inner_map_fd = OPTS_GET(opts, inner_map_fd, 0); attr.map_flags = OPTS_GET(opts, map_flags, 0); attr.map_extra = OPTS_GET(opts, map_extra, 0); attr.numa_node = OPTS_GET(opts, numa_node, 0); attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0); attr.map_token_fd = OPTS_GET(opts, token_fd, 0); attr.excl_prog_hash = ptr_to_u64(OPTS_GET(opts, excl_prog_hash, NULL)); attr.excl_prog_hash_size = OPTS_GET(opts, excl_prog_hash_size, 0); log_opts = OPTS_GET(opts, log_opts, NULL); if (!OPTS_VALID(log_opts, bpf_log_opts)) return libbpf_err(-EINVAL); if (log_opts && feat_supported(NULL, FEAT_BPF_SYSCALL_COMMON_ATTRS)) { memset(&attr_common, 0, attr_common_sz); attr_common.log_buf = ptr_to_u64(OPTS_GET(log_opts, buf, NULL)); attr_common.log_size = OPTS_GET(log_opts, size, 0); attr_common.log_level = OPTS_GET(log_opts, level, 0); fd = sys_bpf_ext_fd(BPF_MAP_CREATE, &attr, attr_sz, &attr_common, attr_common_sz); OPTS_SET(log_opts, true_size, attr_common.log_true_size); } else { fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); OPTS_SET(log_opts, true_size, 0); } return libbpf_err_errno(fd);} static void *alloc_zero_tailing_info(const void *orecord, __u32 cnt, __u32 actual_rec_size, __u32 expected_rec_size){ __u64 info_len = (__u64)actual_rec_size * cnt; void *info, *nrecord; int i; info = malloc(info_len); if (!info) return NULL; /* zero out bytes kernel does not understand */ nrecord = info; for (i = 0; i < cnt; i++) { memcpy(nrecord, orecord, expected_rec_size); memset(nrecord + expected_rec_size, 0, actual_rec_size - expected_rec_size); orecord += actual_rec_size; nrecord += actual_rec_size; } return info;} int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name, const char *license, const struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *opts){ const size_t attr_sz = offsetofend(union bpf_attr, keyring_id); void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; __u32 func_info_rec_size, line_info_rec_size; int fd, attempts; union bpf_attr attr; char *log_buf; bump_rlimit_memlock(); if (!OPTS_VALID(opts, bpf_prog_load_opts)) return libbpf_err(-EINVAL); attempts = OPTS_GET(opts, attempts, 0); if (attempts < 0) return libbpf_err(-EINVAL); if (attempts == 0) attempts = PROG_LOAD_ATTEMPTS; memset(&attr, 0, attr_sz); attr.prog_type = prog_type; attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); attr.prog_flags = OPTS_GET(opts, prog_flags, 0); attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); attr.kern_version = OPTS_GET(opts, kern_version, 0); attr.prog_token_fd = OPTS_GET(opts, token_fd, 0); if (prog_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); if (insn_cnt > UINT_MAX) return libbpf_err(-E2BIG); attr.insns = ptr_to_u64(insns); attr.insn_cnt = (__u32)insn_cnt; attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); if (attach_prog_fd && attach_btf_obj_fd) return libbpf_err(-EINVAL); attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); if (attach_prog_fd) attr.attach_prog_fd = attach_prog_fd; else attr.attach_btf_obj_fd = attach_btf_obj_fd; log_buf = OPTS_GET(opts, log_buf, NULL); log_size = OPTS_GET(opts, log_size, 0); log_level = OPTS_GET(opts, log_level, 0); if (!!log_buf != !!log_size) return libbpf_err(-EINVAL); func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); func_info = OPTS_GET(opts, func_info, NULL); attr.func_info_rec_size = func_info_rec_size; attr.func_info = ptr_to_u64(func_info); attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0); line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); line_info = OPTS_GET(opts, line_info, NULL); attr.line_info_rec_size = line_info_rec_size; attr.line_info = ptr_to_u64(line_info); attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0); attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); if (log_level) { attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_size; attr.log_level = log_level; } fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); if (fd >= 0) return fd; /* After bpf_prog_load, the kernel may modify certain attributes * to give user space a hint how to deal with loading failure. * Check to see whether we can make some changes and load again. */ while (errno == E2BIG && (!finfo || !linfo)) { if (!finfo && attr.func_info_cnt && attr.func_info_rec_size < func_info_rec_size) { /* try with corrected func info records */ finfo = alloc_zero_tailing_info(func_info, attr.func_info_cnt, func_info_rec_size, attr.func_info_rec_size); if (!finfo) { errno = E2BIG; goto done; } attr.func_info = ptr_to_u64(finfo); attr.func_info_rec_size = func_info_rec_size; } else if (!linfo && attr.line_info_cnt && attr.line_info_rec_size < line_info_rec_size) { linfo = alloc_zero_tailing_info(line_info, attr.line_info_cnt, line_info_rec_size, attr.line_info_rec_size); if (!linfo) { errno = E2BIG; goto done; } attr.line_info = ptr_to_u64(linfo); attr.line_info_rec_size = line_info_rec_size; } else { break; } fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); if (fd >= 0) goto done; } if (log_level == 0 && log_buf) { /* log_level == 0 with non-NULL log_buf requires retrying on error * with log_level == 1 and log_buf/log_buf_size set, to get details of * failure */ attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_size; attr.log_level = 1; fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); }done: /* free() doesn't affect errno, so we don't need to restore it */ free(finfo); free(linfo); return libbpf_err_errno(fd);} int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags){ const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret);} int bpf_map_lookup_elem(int fd, const void *key, void *value){ const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret);} int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags){ const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret);} int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value){ const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd;Lines 1–500 of 1484. Load more
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* * common eBPF ELF operations. * * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> * Copyright (C) 2015 Huawei Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License (not later!) * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see <http://www.gnu.org/licenses> */ #include <stdlib.h> #include <string.h> #include <memory.h> #include <unistd.h> #include <asm/unistd.h> #include <errno.h> #include <linux/bpf.h> #include <linux/filter.h> #include <linux/kernel.h> #include <limits.h> #include <sys/resource.h> #include "bpf.h" #include "libbpf.h" #include "libbpf_internal.h" /* * When building perf, unistd.h is overridden. __NR_bpf is * required to be defined explicitly. */ #ifndef __NR_bpf # if defined(__i386__) # define __NR_bpf 357 # elif defined(__x86_64__) # define __NR_bpf 321 # elif defined(__aarch64__) # define __NR_bpf 280 # elif defined(__sparc__) # define __NR_bpf 349 # elif defined(__s390__) # define __NR_bpf 351 # elif defined(__arc__) # define __NR_bpf 280 # elif defined(__mips__) && defined(_ABIO32) # define __NR_bpf 4355 # elif defined(__mips__) && defined(_ABIN32) # define __NR_bpf 6319 # elif defined(__mips__) && defined(_ABI64) # define __NR_bpf 5315 # elif defined(__loongarch__) # define __NR_bpf 280 # else # error __NR_bpf not defined. libbpf does not support your arch. # endif #endif static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; } static inline int sys_bpf_ext(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size, struct bpf_common_attr *attr_common, unsigned int size_common) { cmd = attr_common ? (cmd | BPF_COMMON_ATTRS) : (cmd & ~BPF_COMMON_ATTRS); return syscall(__NR_bpf, cmd, attr, size, attr_common, size_common); } static inline int sys_bpf_ext_fd(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size, struct bpf_common_attr *attr_common, unsigned int size_common) { int fd; fd = sys_bpf_ext(cmd, attr, size, attr_common, size_common); return ensure_good_fd(fd); } int probe_sys_bpf_ext(void) { const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); union bpf_attr attr; int fd; memset(&attr, 0, attr_sz); fd = syscall(__NR_bpf, BPF_PROG_LOAD | BPF_COMMON_ATTRS, &attr, attr_sz, NULL, sizeof(struct bpf_common_attr)); if (fd >= 0) { close(fd); return -EINVAL; } return errno == EFAULT ? 1 : 0; } static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size) { return syscall(__NR_bpf, cmd, attr, size); } static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size) { int fd; fd = sys_bpf(cmd, attr, size); return ensure_good_fd(fd); } int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) { int fd; do { fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size); } while (fd < 0 && errno == EAGAIN && --attempts > 0); return fd; } /* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to * memcg-based memory accounting for BPF maps and progs. This was done in [0]. * We use the support for bpf_ktime_get_coarse_ns() helper, which was added in * the same 5.11 Linux release ([1]), to detect memcg-based accounting for BPF. * * [0] https://lore.kernel.org/bpf/20201201215900.3569844-1-guro@fb.com/ * [1] d05512618056 ("bpf: Add bpf_ktime_get_coarse_ns helper") */ int probe_memcg_account(int token_fd) { const size_t attr_sz = offsetofend(union bpf_attr, prog_token_fd); struct bpf_insn insns[] = { BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), BPF_EXIT_INSN(), }; size_t insn_cnt = ARRAY_SIZE(insns); union bpf_attr attr; int prog_fd; /* attempt loading freplace trying to use custom BTF */ memset(&attr, 0, attr_sz); attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; attr.insns = ptr_to_u64(insns); attr.insn_cnt = insn_cnt; attr.license = ptr_to_u64("GPL"); attr.prog_token_fd = token_fd; if (token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); if (prog_fd >= 0) { close(prog_fd); return 1; } return 0; } static bool memlock_bumped; static rlim_t memlock_rlim = RLIM_INFINITY; int libbpf_set_memlock_rlim(size_t memlock_bytes) { if (memlock_bumped) return libbpf_err(-EBUSY); memlock_rlim = memlock_bytes; return 0; } int bump_rlimit_memlock(void) { struct rlimit rlim; /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ if (memlock_bumped || feat_supported(NULL, FEAT_MEMCG_ACCOUNT)) return 0; memlock_bumped = true; /* zero memlock_rlim disables auto-bumping RLIMIT_MEMLOCK */ if (memlock_rlim == 0) return 0; rlim.rlim_cur = rlim.rlim_max = memlock_rlim; if (setrlimit(RLIMIT_MEMLOCK, &rlim)) return -errno; return 0; } int bpf_map_create(enum bpf_map_type map_type, const char *map_name, __u32 key_size, __u32 value_size, __u32 max_entries, const struct bpf_map_create_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, excl_prog_hash_size); const size_t attr_common_sz = sizeof(struct bpf_common_attr); struct bpf_common_attr attr_common; struct bpf_log_opts *log_opts; union bpf_attr attr; int fd; bump_rlimit_memlock(); memset(&attr, 0, attr_sz); if (!OPTS_VALID(opts, bpf_map_create_opts)) return libbpf_err(-EINVAL); attr.map_type = map_type; if (map_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; attr.max_entries = max_entries; attr.btf_fd = OPTS_GET(opts, btf_fd, 0); attr.btf_key_type_id = OPTS_GET(opts, btf_key_type_id, 0); attr.btf_value_type_id = OPTS_GET(opts, btf_value_type_id, 0); attr.btf_vmlinux_value_type_id = OPTS_GET(opts, btf_vmlinux_value_type_id, 0); attr.value_type_btf_obj_fd = OPTS_GET(opts, value_type_btf_obj_fd, 0); attr.inner_map_fd = OPTS_GET(opts, inner_map_fd, 0); attr.map_flags = OPTS_GET(opts, map_flags, 0); attr.map_extra = OPTS_GET(opts, map_extra, 0); attr.numa_node = OPTS_GET(opts, numa_node, 0); attr.map_ifindex = OPTS_GET(opts, map_ifindex, 0); attr.map_token_fd = OPTS_GET(opts, token_fd, 0); attr.excl_prog_hash = ptr_to_u64(OPTS_GET(opts, excl_prog_hash, NULL)); attr.excl_prog_hash_size = OPTS_GET(opts, excl_prog_hash_size, 0); log_opts = OPTS_GET(opts, log_opts, NULL); if (!OPTS_VALID(log_opts, bpf_log_opts)) return libbpf_err(-EINVAL); if (log_opts && feat_supported(NULL, FEAT_BPF_SYSCALL_COMMON_ATTRS)) { memset(&attr_common, 0, attr_common_sz); attr_common.log_buf = ptr_to_u64(OPTS_GET(log_opts, buf, NULL)); attr_common.log_size = OPTS_GET(log_opts, size, 0); attr_common.log_level = OPTS_GET(log_opts, level, 0); fd = sys_bpf_ext_fd(BPF_MAP_CREATE, &attr, attr_sz, &attr_common, attr_common_sz); OPTS_SET(log_opts, true_size, attr_common.log_true_size); } else { fd = sys_bpf_fd(BPF_MAP_CREATE, &attr, attr_sz); OPTS_SET(log_opts, true_size, 0); } return libbpf_err_errno(fd); } static void * alloc_zero_tailing_info(const void *orecord, __u32 cnt, __u32 actual_rec_size, __u32 expected_rec_size) { __u64 info_len = (__u64)actual_rec_size * cnt; void *info, *nrecord; int i; info = malloc(info_len); if (!info) return NULL; /* zero out bytes kernel does not understand */ nrecord = info; for (i = 0; i < cnt; i++) { memcpy(nrecord, orecord, expected_rec_size); memset(nrecord + expected_rec_size, 0, actual_rec_size - expected_rec_size); orecord += actual_rec_size; nrecord += actual_rec_size; } return info; } int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name, const char *license, const struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *opts) { const size_t attr_sz = offsetofend(union bpf_attr, keyring_id); void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; __u32 func_info_rec_size, line_info_rec_size; int fd, attempts; union bpf_attr attr; char *log_buf; bump_rlimit_memlock(); if (!OPTS_VALID(opts, bpf_prog_load_opts)) return libbpf_err(-EINVAL); attempts = OPTS_GET(opts, attempts, 0); if (attempts < 0) return libbpf_err(-EINVAL); if (attempts == 0) attempts = PROG_LOAD_ATTEMPTS; memset(&attr, 0, attr_sz); attr.prog_type = prog_type; attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); attr.prog_flags = OPTS_GET(opts, prog_flags, 0); attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); attr.kern_version = OPTS_GET(opts, kern_version, 0); attr.prog_token_fd = OPTS_GET(opts, token_fd, 0); if (prog_name && feat_supported(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); if (insn_cnt > UINT_MAX) return libbpf_err(-E2BIG); attr.insns = ptr_to_u64(insns); attr.insn_cnt = (__u32)insn_cnt; attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); if (attach_prog_fd && attach_btf_obj_fd) return libbpf_err(-EINVAL); attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); if (attach_prog_fd) attr.attach_prog_fd = attach_prog_fd; else attr.attach_btf_obj_fd = attach_btf_obj_fd; log_buf = OPTS_GET(opts, log_buf, NULL); log_size = OPTS_GET(opts, log_size, 0); log_level = OPTS_GET(opts, log_level, 0); if (!!log_buf != !!log_size) return libbpf_err(-EINVAL); func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); func_info = OPTS_GET(opts, func_info, NULL); attr.func_info_rec_size = func_info_rec_size; attr.func_info = ptr_to_u64(func_info); attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0); line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); line_info = OPTS_GET(opts, line_info, NULL); attr.line_info_rec_size = line_info_rec_size; attr.line_info = ptr_to_u64(line_info); attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0); attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); if (log_level) { attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_size; attr.log_level = log_level; } fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); if (fd >= 0) return fd; /* After bpf_prog_load, the kernel may modify certain attributes * to give user space a hint how to deal with loading failure. * Check to see whether we can make some changes and load again. */ while (errno == E2BIG && (!finfo || !linfo)) { if (!finfo && attr.func_info_cnt && attr.func_info_rec_size < func_info_rec_size) { /* try with corrected func info records */ finfo = alloc_zero_tailing_info(func_info, attr.func_info_cnt, func_info_rec_size, attr.func_info_rec_size); if (!finfo) { errno = E2BIG; goto done; } attr.func_info = ptr_to_u64(finfo); attr.func_info_rec_size = func_info_rec_size; } else if (!linfo && attr.line_info_cnt && attr.line_info_rec_size < line_info_rec_size) { linfo = alloc_zero_tailing_info(line_info, attr.line_info_cnt, line_info_rec_size, attr.line_info_rec_size); if (!linfo) { errno = E2BIG; goto done; } attr.line_info = ptr_to_u64(linfo); attr.line_info_rec_size = line_info_rec_size; } else { break; } fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); if (fd >= 0) goto done; } if (log_level == 0 && log_buf) { /* log_level == 0 with non-NULL log_buf requires retrying on error * with log_level == 1 and log_buf/log_buf_size set, to get details of * failure */ attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_size; attr.log_level = 1; fd = sys_bpf_prog_load(&attr, attr_sz, attempts); OPTS_SET(opts, log_true_size, attr.log_true_size); } done: /* free() doesn't affect errno, so we don't need to restore it */ free(finfo); free(linfo); return libbpf_err_errno(fd); } int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags) { const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_elem(int fd, const void *key, void *value) { const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags) { const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value) { const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; memset(&attr, 0, attr_sz); attr.map_fd = fd;