// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)/* Copyright (c) 2018 Facebook */ #include <byteswap.h>#include <endian.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <fcntl.h>#include <unistd.h>#include <errno.h>#include <sys/utsname.h>#include <sys/param.h>#include <sys/stat.h>#include <sys/mman.h>#include <linux/kernel.h>#include <linux/err.h>#include <linux/btf.h>#include <gelf.h>#include "btf.h"#include "bpf.h"#include "libbpf.h"#include "libbpf_internal.h"#include "hashmap.h"#include "strset.h" #define BTF_MAX_NR_TYPES 0x7fffffffU#define BTF_MAX_STR_OFFSET 0x7fffffffU static struct btf_type btf_void; /* * Describe how kinds are laid out; some have a singular element following the "struct btf_type", * some have BTF_INFO_VLEN(t->info) elements. Specify sizes for both. Flags are currently unused. * Kind layout can be optionally added to the BTF representation in a dedicated section to * facilitate parsing. New kinds must be added here. */static struct btf_layout layouts[NR_BTF_KINDS] = {/* singular element size vlen element(s) size flags */[BTF_KIND_UNKN] = { 0, 0, 0 },[BTF_KIND_INT] = { sizeof(__u32), 0, 0 },[BTF_KIND_PTR] = { 0, 0, 0 },[BTF_KIND_ARRAY] = { sizeof(struct btf_array), 0, 0 },[BTF_KIND_STRUCT] = { 0, sizeof(struct btf_member), 0 },[BTF_KIND_UNION] = { 0, sizeof(struct btf_member), 0 },[BTF_KIND_ENUM] = { 0, sizeof(struct btf_enum), 0 },[BTF_KIND_FWD] = { 0, 0, 0 },[BTF_KIND_TYPEDEF] = { 0, 0, 0 },[BTF_KIND_VOLATILE] = { 0, 0, 0 },[BTF_KIND_CONST] = { 0, 0, 0 },[BTF_KIND_RESTRICT] = { 0, 0, 0 },[BTF_KIND_FUNC] = { 0, 0, 0 },[BTF_KIND_FUNC_PROTO] = { 0, sizeof(struct btf_param), 0 },[BTF_KIND_VAR] = { sizeof(struct btf_var), 0, 0 },[BTF_KIND_DATASEC] = { 0, sizeof(struct btf_var_secinfo), 0 },[BTF_KIND_FLOAT] = { 0, 0, 0 },[BTF_KIND_DECL_TAG] = { sizeof(struct btf_decl_tag), 0, 0 },[BTF_KIND_TYPE_TAG] = { 0, 0, 0 },[BTF_KIND_ENUM64] = { 0, sizeof(struct btf_enum64), 0 },}; struct btf { /* raw BTF data in native endianness */ void *raw_data; /* raw BTF data in non-native endianness */ void *raw_data_swapped; __u32 raw_size; /* whether target endianness differs from the native one */ bool swapped_endian; /* * When BTF is loaded from an ELF or raw memory it is stored * in a contiguous memory block. The type_data, layout and strs_data * point inside that memory region to their respective parts of BTF * representation: * * +----------------------------------------+---------------+ * | Header | Types | Optional layout | Strings | * +--------------------------------------------------------+ * ^ ^ ^ ^ * | | | | * raw_data | | | * types_data-+ | | * layout---------------+ | * strs_data--------------------------------+ * * A separate struct btf_header is embedded as btf->hdr, * and header information is copied into it. This allows us * to handle header data for various header formats; the original, * the extended header with layout info, etc. * * If BTF data is later modified, e.g., due to types added or * removed, BTF deduplication performed, etc, this contiguous * representation is broken up into four independent memory * regions. * * raw_data is nulled out at that point, but can be later allocated * and cached again if user calls btf__raw_data(), at which point * raw_data will contain a contiguous copy of header, types, optional * layout and strings. layout optionally points to a * btf_layout array - this allows us to encode information about * the kinds known at encoding time. If layout is NULL no * layout information is encoded. * * +----------+ +---------+ +-----------+ +-----------+ * | Header | | Types | | Layout | | Strings | * +----------+ +---------+ +-----------+ +-----------+ * ^ ^ ^ ^ * | | | | * hdr | | | * types_data----+ | | * layout---------------------+ | * strset__data(strs_set)---------------------+ * * +----------+---------+-------------------+-----------+ * | Header | Types | Optional Layout | Strings | * raw_data----->+----------+---------+-------------------+-----------+ */ struct btf_header hdr; void *types_data; size_t types_data_cap; /* used size stored in hdr->type_len */ /* type ID to `struct btf_type *` lookup index * type_offs[0] corresponds to the first non-VOID type: * - for base BTF it's type [1]; * - for split BTF it's the first non-base BTF type. */ __u32 *type_offs; size_t type_offs_cap; /* number of types in this BTF instance: * - doesn't include special [0] void type; * - for split BTF counts number of types added on top of base BTF. */ __u32 nr_types; /* the start IDs of named types in sorted BTF */ int named_start_id; /* if not NULL, points to the base BTF on top of which the current * split BTF is based */ struct btf *base_btf; /* BTF type ID of the first type in this BTF instance: * - for base BTF it's equal to 1; * - for split BTF it's equal to biggest type ID of base BTF plus 1. */ int start_id; /* logical string offset of this BTF instance: * - for base BTF it's equal to 0; * - for split BTF it's equal to total size of base BTF's string section size. */ int start_str_off; /* only one of strs_data or strs_set can be non-NULL, depending on * whether BTF is in a modifiable state (strs_set is used) or not * (strs_data points inside raw_data) */ void *strs_data; /* a set of unique strings */ struct strset *strs_set; /* whether strings are already deduplicated */ bool strs_deduped; /* whether base_btf should be freed in btf_free for this instance */ bool owns_base; /* whether raw_data is a (read-only) mmap */ bool raw_data_is_mmap; /* is BTF modifiable? i.e. is it split into separate sections as described above? */ bool modifiable; /* does BTF have header information we do not support? If so, disallow * modification. */ bool has_hdr_extra; /* Points either at raw kind layout data in parsed BTF (if present), or * at an allocated kind layout array when BTF is modifiable. */ void *layout; /* BTF object FD, if loaded into kernel */ int fd; /* Pointer size (in bytes) for a target architecture of this BTF */ int ptr_sz;}; static inline __u64 ptr_to_u64(const void *ptr){ return (__u64) (unsigned long) ptr;} /* Ensure given dynamically allocated memory region pointed to by *data* with * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements * are already used. At most *max_cnt* elements can be ever allocated. * If necessary, memory is reallocated and all existing data is copied over, * new pointer to the memory region is stored at *data, new memory region * capacity (in number of elements) is stored in *cap. * On success, memory pointer to the beginning of unused memory is returned. * On error, NULL is returned. */void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t cur_cnt, size_t max_cnt, size_t add_cnt){ size_t new_cnt; void *new_data; if (cur_cnt + add_cnt <= *cap_cnt) return *data + cur_cnt * elem_sz; /* requested more than the set limit */ if (cur_cnt + add_cnt > max_cnt) return NULL; new_cnt = *cap_cnt; new_cnt += new_cnt / 4; /* expand by 25% */ if (new_cnt < 16) /* but at least 16 elements */ new_cnt = 16; if (new_cnt > max_cnt) /* but not exceeding a set limit */ new_cnt = max_cnt; if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ new_cnt = cur_cnt + add_cnt; new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); if (!new_data) return NULL; /* zero out newly allocated portion of memory */ memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); *data = new_data; *cap_cnt = new_cnt; return new_data + cur_cnt * elem_sz;} /* Ensure given dynamically allocated memory region has enough allocated space * to accommodate *need_cnt* elements of size *elem_sz* bytes each */int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt){ void *p; if (need_cnt <= *cap_cnt) return 0; p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); if (!p) return -ENOMEM; return 0;} static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt){ return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), btf->nr_types, BTF_MAX_NR_TYPES, add_cnt);} static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off){ __u32 *p; p = btf_add_type_offs_mem(btf, 1); if (!p) return -ENOMEM; *p = type_off; return 0;} static void btf_bswap_hdr(struct btf_header *h, __u32 hdr_len){ h->magic = bswap_16(h->magic); h->hdr_len = bswap_32(h->hdr_len); h->type_off = bswap_32(h->type_off); h->type_len = bswap_32(h->type_len); h->str_off = bswap_32(h->str_off); h->str_len = bswap_32(h->str_len); /* May be operating on raw data with hdr_len that does not include below fields */ if (hdr_len >= sizeof(struct btf_header)) { h->layout_off = bswap_32(h->layout_off); h->layout_len = bswap_32(h->layout_len); }} static int btf_parse_hdr(struct btf *btf){ struct btf_header *hdr = btf->raw_data; __u32 hdr_len, meta_left; if (btf->raw_size < offsetofend(struct btf_header, str_len)) { pr_debug("BTF header not found\n"); return -EINVAL; } hdr_len = hdr->hdr_len; if (hdr->magic == bswap_16(BTF_MAGIC)) { btf->swapped_endian = true; hdr_len = bswap_32(hdr->hdr_len); if (hdr_len < offsetofend(struct btf_header, str_len)) { pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", hdr_len); return -ENOTSUP; } } else if (hdr->magic != BTF_MAGIC) { pr_debug("Invalid BTF magic: %x\n", hdr->magic); return -EINVAL; } if (btf->raw_size < hdr_len) { pr_debug("BTF header len %u larger than data size %u\n", hdr_len, btf->raw_size); return -EINVAL; } if (btf->swapped_endian) btf_bswap_hdr(hdr, hdr_len); memcpy(&btf->hdr, hdr, min((size_t)hdr_len, sizeof(struct btf_header))); /* If unknown header data is found, modification is prohibited in * btf_ensure_modifiable(). */ if (hdr_len > sizeof(struct btf_header)) { __u8 *h = (__u8 *)hdr; __u32 i; for (i = sizeof(struct btf_header); i < hdr_len; i++) { if (!h[i]) continue; btf->has_hdr_extra = true; pr_debug("Unknown BTF header data at offset %u; modification is disallowed\n", i); break; } } meta_left = btf->raw_size - hdr_len; if (meta_left < (long long)btf->hdr.str_off + btf->hdr.str_len) { pr_debug("Invalid BTF total size: %u\n", btf->raw_size); return -EINVAL; } if ((long long)btf->hdr.type_off + btf->hdr.type_len > btf->hdr.str_off) { pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", btf->hdr.type_off, btf->hdr.type_len, btf->hdr.str_off, btf->hdr.str_len); return -EINVAL; } if (btf->hdr.type_off % 4) { pr_debug("BTF type section is not aligned to 4 bytes\n"); return -EINVAL; } if (btf->hdr.layout_len == 0) return 0; /* optional layout section sits between types and strings */ if (btf->hdr.layout_off % 4) { pr_debug("BTF layout section is not aligned to 4 bytes\n"); return -EINVAL; } if (btf->hdr.layout_off < (long long)btf->hdr.type_off + btf->hdr.type_len) { pr_debug("Invalid BTF data sections layout: type data at %u + %u, layout data at %u + %u\n", btf->hdr.type_off, btf->hdr.type_len, btf->hdr.layout_off, btf->hdr.layout_len); return -EINVAL; } if ((long long)btf->hdr.layout_off + btf->hdr.layout_len > btf->hdr.str_off || btf->hdr.layout_off > btf->hdr.str_off) { pr_debug("Invalid BTF data sections layout: layout data at %u + %u, strings data at %u\n", btf->hdr.layout_off, btf->hdr.layout_len, btf->hdr.str_off); return -EINVAL; } return 0;} static int btf_parse_str_sec(struct btf *btf){ const char *start = btf->strs_data; const char *end = start + btf->hdr.str_len; if (btf->base_btf && btf->hdr.str_len == 0) return 0; if (!btf->hdr.str_len || btf->hdr.str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { pr_debug("Invalid BTF string section\n"); return -EINVAL; } if (!btf->base_btf && start[0]) { pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); return -EINVAL; } return 0;} static int btf_parse_layout_sec(struct btf *btf){ if (!btf->hdr.layout_len) return 0; if (btf->hdr.layout_len % sizeof(struct btf_layout) != 0) { pr_debug("Invalid BTF kind layout section\n"); return -EINVAL; } btf->layout = btf->raw_data + btf->hdr.hdr_len + btf->hdr.layout_off; if (btf->swapped_endian) { struct btf_layout *l, *end = btf->layout + btf->hdr.layout_len; for (l = btf->layout; l < end; l++) l->flags = bswap_16(l->flags); } return 0;} /* for unknown kinds, consult kind layout. */static int btf_type_size_unknown(const struct btf *btf, const struct btf_type *t){ __u32 l_cnt = btf->hdr.layout_len / sizeof(struct btf_layout); struct btf_layout *l = btf->layout; __u32 vlen = btf_vlen(t); __u32 kind = btf_kind(t); /* Fall back to base BTF if needed as they share layout information */ if (!l) { struct btf *base_btf = btf->base_btf; if (base_btf) { l = base_btf->layout; l_cnt = base_btf->hdr.layout_len / sizeof(struct btf_layout); } } if (!l || kind >= l_cnt) { pr_debug("Unsupported BTF_KIND: %u\n", btf_kind(t)); return -EINVAL; } if (l[kind].info_sz % 4) { pr_debug("Unsupported info_sz %u for kind %u\n", l[kind].info_sz, kind); return -EINVAL; } if (l[kind].elem_sz % 4) { pr_debug("Unsupported elem_sz %u for kind %u\n", l[kind].elem_sz, kind); return -EINVAL; } return sizeof(struct btf_type) + l[kind].info_sz + vlen * l[kind].elem_sz;} static int btf_type_size(const struct btf *btf, const struct btf_type *t){ const int base_size = sizeof(struct btf_type); __u32 vlen = btf_vlen(t); switch (btf_kind(t)) { case BTF_KIND_FWD: case BTF_KIND_CONST: case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_PTR: case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: case BTF_KIND_TYPE_TAG: return base_size; case BTF_KIND_INT: return base_size + sizeof(__u32); case BTF_KIND_ENUM: return base_size + vlen * sizeof(struct btf_enum); case BTF_KIND_ENUM64: return base_size + vlen * sizeof(struct btf_enum64); case BTF_KIND_ARRAY: return base_size + sizeof(struct btf_array); case BTF_KIND_STRUCT: case BTF_KIND_UNION: return base_size + vlen * sizeof(struct btf_member); case BTF_KIND_FUNC_PROTO: return base_size + vlen * sizeof(struct btf_param); case BTF_KIND_VAR: return base_size + sizeof(struct btf_var); case BTF_KIND_DATASEC: return base_size + vlen * sizeof(struct btf_var_secinfo); case BTF_KIND_DECL_TAG: return base_size + sizeof(struct btf_decl_tag); default: return btf_type_size_unknown(btf, t); }} static void btf_bswap_type_base(struct btf_type *t){ t->name_off = bswap_32(t->name_off); t->info = bswap_32(t->info); t->type = bswap_32(t->type);} Lines 1–500 of 6376. Load more
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) /* Copyright (c) 2018 Facebook */ #include <byteswap.h> #include <endian.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> #include <sys/utsname.h> #include <sys/param.h> #include <sys/stat.h> #include <sys/mman.h> #include <linux/kernel.h> #include <linux/err.h> #include <linux/btf.h> #include <gelf.h> #include "btf.h" #include "bpf.h" #include "libbpf.h" #include "libbpf_internal.h" #include "hashmap.h" #include "strset.h" #define BTF_MAX_NR_TYPES 0x7fffffffU #define BTF_MAX_STR_OFFSET 0x7fffffffU static struct btf_type btf_void; /* * Describe how kinds are laid out; some have a singular element following the "struct btf_type", * some have BTF_INFO_VLEN(t->info) elements. Specify sizes for both. Flags are currently unused. * Kind layout can be optionally added to the BTF representation in a dedicated section to * facilitate parsing. New kinds must be added here. */ static struct btf_layout layouts[NR_BTF_KINDS] = { /* singular element size vlen element(s) size flags */ [BTF_KIND_UNKN] = { 0, 0, 0 }, [BTF_KIND_INT] = { sizeof(__u32), 0, 0 }, [BTF_KIND_PTR] = { 0, 0, 0 }, [BTF_KIND_ARRAY] = { sizeof(struct btf_array), 0, 0 }, [BTF_KIND_STRUCT] = { 0, sizeof(struct btf_member), 0 }, [BTF_KIND_UNION] = { 0, sizeof(struct btf_member), 0 }, [BTF_KIND_ENUM] = { 0, sizeof(struct btf_enum), 0 }, [BTF_KIND_FWD] = { 0, 0, 0 }, [BTF_KIND_TYPEDEF] = { 0, 0, 0 }, [BTF_KIND_VOLATILE] = { 0, 0, 0 }, [BTF_KIND_CONST] = { 0, 0, 0 }, [BTF_KIND_RESTRICT] = { 0, 0, 0 }, [BTF_KIND_FUNC] = { 0, 0, 0 }, [BTF_KIND_FUNC_PROTO] = { 0, sizeof(struct btf_param), 0 }, [BTF_KIND_VAR] = { sizeof(struct btf_var), 0, 0 }, [BTF_KIND_DATASEC] = { 0, sizeof(struct btf_var_secinfo), 0 }, [BTF_KIND_FLOAT] = { 0, 0, 0 }, [BTF_KIND_DECL_TAG] = { sizeof(struct btf_decl_tag), 0, 0 }, [BTF_KIND_TYPE_TAG] = { 0, 0, 0 }, [BTF_KIND_ENUM64] = { 0, sizeof(struct btf_enum64), 0 }, }; struct btf { /* raw BTF data in native endianness */ void *raw_data; /* raw BTF data in non-native endianness */ void *raw_data_swapped; __u32 raw_size; /* whether target endianness differs from the native one */ bool swapped_endian; /* * When BTF is loaded from an ELF or raw memory it is stored * in a contiguous memory block. The type_data, layout and strs_data * point inside that memory region to their respective parts of BTF * representation: * * +----------------------------------------+---------------+ * | Header | Types | Optional layout | Strings | * +--------------------------------------------------------+ * ^ ^ ^ ^ * | | | | * raw_data | | | * types_data-+ | | * layout---------------+ | * strs_data--------------------------------+ * * A separate struct btf_header is embedded as btf->hdr, * and header information is copied into it. This allows us * to handle header data for various header formats; the original, * the extended header with layout info, etc. * * If BTF data is later modified, e.g., due to types added or * removed, BTF deduplication performed, etc, this contiguous * representation is broken up into four independent memory * regions. * * raw_data is nulled out at that point, but can be later allocated * and cached again if user calls btf__raw_data(), at which point * raw_data will contain a contiguous copy of header, types, optional * layout and strings. layout optionally points to a * btf_layout array - this allows us to encode information about * the kinds known at encoding time. If layout is NULL no * layout information is encoded. * * +----------+ +---------+ +-----------+ +-----------+ * | Header | | Types | | Layout | | Strings | * +----------+ +---------+ +-----------+ +-----------+ * ^ ^ ^ ^ * | | | | * hdr | | | * types_data----+ | | * layout---------------------+ | * strset__data(strs_set)---------------------+ * * +----------+---------+-------------------+-----------+ * | Header | Types | Optional Layout | Strings | * raw_data----->+----------+---------+-------------------+-----------+ */ struct btf_header hdr; void *types_data; size_t types_data_cap; /* used size stored in hdr->type_len */ /* type ID to `struct btf_type *` lookup index * type_offs[0] corresponds to the first non-VOID type: * - for base BTF it's type [1]; * - for split BTF it's the first non-base BTF type. */ __u32 *type_offs; size_t type_offs_cap; /* number of types in this BTF instance: * - doesn't include special [0] void type; * - for split BTF counts number of types added on top of base BTF. */ __u32 nr_types; /* the start IDs of named types in sorted BTF */ int named_start_id; /* if not NULL, points to the base BTF on top of which the current * split BTF is based */ struct btf *base_btf; /* BTF type ID of the first type in this BTF instance: * - for base BTF it's equal to 1; * - for split BTF it's equal to biggest type ID of base BTF plus 1. */ int start_id; /* logical string offset of this BTF instance: * - for base BTF it's equal to 0; * - for split BTF it's equal to total size of base BTF's string section size. */ int start_str_off; /* only one of strs_data or strs_set can be non-NULL, depending on * whether BTF is in a modifiable state (strs_set is used) or not * (strs_data points inside raw_data) */ void *strs_data; /* a set of unique strings */ struct strset *strs_set; /* whether strings are already deduplicated */ bool strs_deduped; /* whether base_btf should be freed in btf_free for this instance */ bool owns_base; /* whether raw_data is a (read-only) mmap */ bool raw_data_is_mmap; /* is BTF modifiable? i.e. is it split into separate sections as described above? */ bool modifiable; /* does BTF have header information we do not support? If so, disallow * modification. */ bool has_hdr_extra; /* Points either at raw kind layout data in parsed BTF (if present), or * at an allocated kind layout array when BTF is modifiable. */ void *layout; /* BTF object FD, if loaded into kernel */ int fd; /* Pointer size (in bytes) for a target architecture of this BTF */ int ptr_sz; }; static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; } /* Ensure given dynamically allocated memory region pointed to by *data* with * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements * are already used. At most *max_cnt* elements can be ever allocated. * If necessary, memory is reallocated and all existing data is copied over, * new pointer to the memory region is stored at *data, new memory region * capacity (in number of elements) is stored in *cap. * On success, memory pointer to the beginning of unused memory is returned. * On error, NULL is returned. */ void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t cur_cnt, size_t max_cnt, size_t add_cnt) { size_t new_cnt; void *new_data; if (cur_cnt + add_cnt <= *cap_cnt) return *data + cur_cnt * elem_sz; /* requested more than the set limit */ if (cur_cnt + add_cnt > max_cnt) return NULL; new_cnt = *cap_cnt; new_cnt += new_cnt / 4; /* expand by 25% */ if (new_cnt < 16) /* but at least 16 elements */ new_cnt = 16; if (new_cnt > max_cnt) /* but not exceeding a set limit */ new_cnt = max_cnt; if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */ new_cnt = cur_cnt + add_cnt; new_data = libbpf_reallocarray(*data, new_cnt, elem_sz); if (!new_data) return NULL; /* zero out newly allocated portion of memory */ memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz); *data = new_data; *cap_cnt = new_cnt; return new_data + cur_cnt * elem_sz; } /* Ensure given dynamically allocated memory region has enough allocated space * to accommodate *need_cnt* elements of size *elem_sz* bytes each */ int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) { void *p; if (need_cnt <= *cap_cnt) return 0; p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); if (!p) return -ENOMEM; return 0; } static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt) { return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), btf->nr_types, BTF_MAX_NR_TYPES, add_cnt); } static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off) { __u32 *p; p = btf_add_type_offs_mem(btf, 1); if (!p) return -ENOMEM; *p = type_off; return 0; } static void btf_bswap_hdr(struct btf_header *h, __u32 hdr_len) { h->magic = bswap_16(h->magic); h->hdr_len = bswap_32(h->hdr_len); h->type_off = bswap_32(h->type_off); h->type_len = bswap_32(h->type_len); h->str_off = bswap_32(h->str_off); h->str_len = bswap_32(h->str_len); /* May be operating on raw data with hdr_len that does not include below fields */ if (hdr_len >= sizeof(struct btf_header)) { h->layout_off = bswap_32(h->layout_off); h->layout_len = bswap_32(h->layout_len); } } static int btf_parse_hdr(struct btf *btf) { struct btf_header *hdr = btf->raw_data; __u32 hdr_len, meta_left; if (btf->raw_size < offsetofend(struct btf_header, str_len)) { pr_debug("BTF header not found\n"); return -EINVAL; } hdr_len = hdr->hdr_len; if (hdr->magic == bswap_16(BTF_MAGIC)) { btf->swapped_endian = true; hdr_len = bswap_32(hdr->hdr_len); if (hdr_len < offsetofend(struct btf_header, str_len)) { pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n", hdr_len); return -ENOTSUP; } } else if (hdr->magic != BTF_MAGIC) { pr_debug("Invalid BTF magic: %x\n", hdr->magic); return -EINVAL; } if (btf->raw_size < hdr_len) { pr_debug("BTF header len %u larger than data size %u\n", hdr_len, btf->raw_size); return -EINVAL; } if (btf->swapped_endian) btf_bswap_hdr(hdr, hdr_len); memcpy(&btf->hdr, hdr, min((size_t)hdr_len, sizeof(struct btf_header))); /* If unknown header data is found, modification is prohibited in * btf_ensure_modifiable(). */ if (hdr_len > sizeof(struct btf_header)) { __u8 *h = (__u8 *)hdr; __u32 i; for (i = sizeof(struct btf_header); i < hdr_len; i++) { if (!h[i]) continue; btf->has_hdr_extra = true; pr_debug("Unknown BTF header data at offset %u; modification is disallowed\n", i); break; } } meta_left = btf->raw_size - hdr_len; if (meta_left < (long long)btf->hdr.str_off + btf->hdr.str_len) { pr_debug("Invalid BTF total size: %u\n", btf->raw_size); return -EINVAL; } if ((long long)btf->hdr.type_off + btf->hdr.type_len > btf->hdr.str_off) { pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n", btf->hdr.type_off, btf->hdr.type_len, btf->hdr.str_off, btf->hdr.str_len); return -EINVAL; } if (btf->hdr.type_off % 4) { pr_debug("BTF type section is not aligned to 4 bytes\n"); return -EINVAL; } if (btf->hdr.layout_len == 0) return 0; /* optional layout section sits between types and strings */ if (btf->hdr.layout_off % 4) { pr_debug("BTF layout section is not aligned to 4 bytes\n"); return -EINVAL; } if (btf->hdr.layout_off < (long long)btf->hdr.type_off + btf->hdr.type_len) { pr_debug("Invalid BTF data sections layout: type data at %u + %u, layout data at %u + %u\n", btf->hdr.type_off, btf->hdr.type_len, btf->hdr.layout_off, btf->hdr.layout_len); return -EINVAL; } if ((long long)btf->hdr.layout_off + btf->hdr.layout_len > btf->hdr.str_off || btf->hdr.layout_off > btf->hdr.str_off) { pr_debug("Invalid BTF data sections layout: layout data at %u + %u, strings data at %u\n", btf->hdr.layout_off, btf->hdr.layout_len, btf->hdr.str_off); return -EINVAL; } return 0; } static int btf_parse_str_sec(struct btf *btf) { const char *start = btf->strs_data; const char *end = start + btf->hdr.str_len; if (btf->base_btf && btf->hdr.str_len == 0) return 0; if (!btf->hdr.str_len || btf->hdr.str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) { pr_debug("Invalid BTF string section\n"); return -EINVAL; } if (!btf->base_btf && start[0]) { pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n"); return -EINVAL; } return 0; } static int btf_parse_layout_sec(struct btf *btf) { if (!btf->hdr.layout_len) return 0; if (btf->hdr.layout_len % sizeof(struct btf_layout) != 0) { pr_debug("Invalid BTF kind layout section\n"); return -EINVAL; } btf->layout = btf->raw_data + btf->hdr.hdr_len + btf->hdr.layout_off; if (btf->swapped_endian) { struct btf_layout *l, *end = btf->layout + btf->hdr.layout_len; for (l = btf->layout; l < end; l++) l->flags = bswap_16(l->flags); } return 0; } /* for unknown kinds, consult kind layout. */ static int btf_type_size_unknown(const struct btf *btf, const struct btf_type *t) { __u32 l_cnt = btf->hdr.layout_len / sizeof(struct btf_layout); struct btf_layout *l = btf->layout; __u32 vlen = btf_vlen(t); __u32 kind = btf_kind(t); /* Fall back to base BTF if needed as they share layout information */ if (!l) { struct btf *base_btf = btf->base_btf; if (base_btf) { l = base_btf->layout; l_cnt = base_btf->hdr.layout_len / sizeof(struct btf_layout); } } if (!l || kind >= l_cnt) { pr_debug("Unsupported BTF_KIND: %u\n", btf_kind(t)); return -EINVAL; } if (l[kind].info_sz % 4) { pr_debug("Unsupported info_sz %u for kind %u\n", l[kind].info_sz, kind); return -EINVAL; } if (l[kind].elem_sz % 4) { pr_debug("Unsupported elem_sz %u for kind %u\n", l[kind].elem_sz, kind); return -EINVAL; } return sizeof(struct btf_type) + l[kind].info_sz + vlen * l[kind].elem_sz; } static int btf_type_size(const struct btf *btf, const struct btf_type *t) { const int base_size = sizeof(struct btf_type); __u32 vlen = btf_vlen(t); switch (btf_kind(t)) { case BTF_KIND_FWD: case BTF_KIND_CONST: case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_PTR: case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: case BTF_KIND_TYPE_TAG: return base_size; case BTF_KIND_INT: return base_size + sizeof(__u32); case BTF_KIND_ENUM: return base_size + vlen * sizeof(struct btf_enum); case BTF_KIND_ENUM64: return base_size + vlen * sizeof(struct btf_enum64); case BTF_KIND_ARRAY: return base_size + sizeof(struct btf_array); case BTF_KIND_STRUCT: case BTF_KIND_UNION: return base_size + vlen * sizeof(struct btf_member); case BTF_KIND_FUNC_PROTO: return base_size + vlen * sizeof(struct btf_param); case BTF_KIND_VAR: return base_size + sizeof(struct btf_var); case BTF_KIND_DATASEC: return base_size + vlen * sizeof(struct btf_var_secinfo); case BTF_KIND_DECL_TAG: return base_size + sizeof(struct btf_decl_tag); default: return btf_type_size_unknown(btf, t); } } static void btf_bswap_type_base(struct btf_type *t) { t->name_off = bswap_32(t->name_off); t->info = bswap_32(t->info); t->type = bswap_32(t->type); }