// SPDX-License-Identifier: GPL-2.0-or-later/* * elf.c - ELF access library * * Adapted from kpatch (https://github.com/dynup/kpatch): * Copyright (C) 2013-2015 Josh Poimboeuf <jpoimboe@redhat.com> * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> */ #include <sys/types.h>#include <sys/stat.h>#include <sys/mman.h>#include <fcntl.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <unistd.h>#include <errno.h>#include <ctype.h>#include <linux/align.h>#include <linux/kernel.h>#include <linux/interval_tree_generic.h>#include <linux/log2.h>#include <objtool/builtin.h>#include <objtool/elf.h>#include <objtool/warn.h> static ssize_t demangled_name_len(const char *name); static inline u32 str_hash(const char *str){ return jhash(str, strlen(str), 0);} static inline u32 str_hash_demangled(const char *str){ return jhash(str, demangled_name_len(str), 0);} #define __elf_table(name) (elf->name##_hash)#define __elf_bits(name) (elf->name##_bits) #define __elf_table_entry(name, key) \ __elf_table(name)[hash_min(key, __elf_bits(name))] #define elf_hash_add(name, node, key) \({ \ struct elf_hash_node *__node = node; \ __node->next = __elf_table_entry(name, key); \ __elf_table_entry(name, key) = __node; \}) static inline void __elf_hash_del(struct elf_hash_node *node, struct elf_hash_node **head){ struct elf_hash_node *cur, *prev; if (node == *head) { *head = node->next; return; } for (prev = NULL, cur = *head; cur; prev = cur, cur = cur->next) { if (cur == node) { prev->next = cur->next; break; } }} #define elf_hash_del(name, node, key) \ __elf_hash_del(node, &__elf_table_entry(name, key)) #define elf_list_entry(ptr, type, member) \({ \ typeof(ptr) __ptr = (ptr); \ __ptr ? container_of(__ptr, type, member) : NULL; \}) #define elf_hash_for_each_possible(name, obj, member, key) \ for (obj = elf_list_entry(__elf_table_entry(name, key), typeof(*obj), member); \ obj; \ obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) #define elf_alloc_hash(name, size) \({ \ __elf_bits(name) = max(10, ilog2(size)); \ __elf_table(name) = mmap(NULL, sizeof(struct elf_hash_node *) << __elf_bits(name), \ PROT_READ|PROT_WRITE, \ MAP_PRIVATE|MAP_ANON, -1, 0); \ if (__elf_table(name) == (void *)-1L) { \ ERROR_GLIBC("mmap fail " #name); \ __elf_table(name) = NULL; \ } \ __elf_table(name); \}) static inline unsigned long __sym_start(struct symbol *s){ return s->offset;} static inline unsigned long __sym_last(struct symbol *s){ return s->offset + (s->len ? s->len - 1 : 0);} INTERVAL_TREE_DEFINE(struct symbol, node, unsigned long, __subtree_last, __sym_start, __sym_last, static inline __maybe_unused, __sym) #define __sym_for_each(_iter, _tree, _start, _end) \ for (_iter = __sym_iter_first((_tree), (_start), (_end)); \ _iter; _iter = __sym_iter_next(_iter, (_start), (_end))) struct symbol_hole { unsigned long key; const struct symbol *sym;}; /* * Find the last symbol before @offset. */static int symbol_hole_by_offset(const void *key, const struct rb_node *node){ const struct symbol *s = rb_entry(node, struct symbol, node); struct symbol_hole *sh = (void *)key; if (sh->key < s->offset) return -1; if (sh->key >= s->offset + s->len) { sh->sym = s; return 1; } return 0;} struct section *find_section_by_name(const struct elf *elf, const char *name){ struct section *sec; elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) { if (!strcmp(sec->name, name)) return sec; } return NULL;} static struct section *find_section_by_index(struct elf *elf, unsigned int idx){ struct section *sec; elf_hash_for_each_possible(section, sec, hash, idx) { if (sec->idx == idx) return sec; } return NULL;} static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx){ struct symbol *sym; elf_hash_for_each_possible(symbol, sym, hash, idx) { if (sym->idx == idx) return sym; } return NULL;} struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset){ struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *sym; __sym_for_each(sym, tree, offset, offset) { if (sym->offset == offset && !is_sec_sym(sym)) return sym->alias; } return NULL;} struct symbol *find_func_by_offset(struct section *sec, unsigned long offset){ struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *func; __sym_for_each(func, tree, offset, offset) { if (func->offset == offset && is_func_sym(func)) return func->alias; } return NULL;} struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset){ struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *sym = NULL, *tmp; __sym_for_each(tmp, tree, offset, offset) { if (tmp->len) { if (!sym) { sym = tmp; continue; } if (sym->offset != tmp->offset || sym->len != tmp->len) { /* * In the rare case of overlapping symbols, * pick the smaller one. * * TODO: outlaw overlapping symbols */ if (tmp->len < sym->len) sym = tmp; } } } return sym ? sym->alias : NULL;} /* * Returns size of hole starting at @offset. */int find_symbol_hole_containing(const struct section *sec, unsigned long offset){ struct symbol_hole hole = { .key = offset, .sym = NULL, }; struct rb_node *n; struct symbol *s; /* * Find the rightmost symbol for which @offset is after it. */ n = rb_find(&hole, &sec->symbol_tree.rb_root, symbol_hole_by_offset); /* found a symbol that contains @offset */ if (n) return 0; /* not a hole */ /* * @offset >= sym->offset + sym->len, find symbol after it. * When hole.sym is empty, use the first node to compute the hole. * If there is no symbol in the section, the first node will be NULL, * in which case, -1 is returned to skip the whole section. */ if (hole.sym) n = rb_next(&hole.sym->node); else n = rb_first_cached(&sec->symbol_tree); if (!n) return -1; /* until end of address space */ /* hole until start of next symbol */ s = rb_entry(n, struct symbol, node); return s->offset - offset;} struct symbol *find_func_containing(struct section *sec, unsigned long offset){ struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *func; __sym_for_each(func, tree, offset, offset) { if (is_func_sym(func)) return func->alias; } return NULL;} struct symbol *find_symbol_by_name(const struct elf *elf, const char *name){ struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { if (!strcmp(sym->name, name)) return sym; } return NULL;} /* Find local symbol with matching STT_FILE */static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf, struct symbol *file, const char *name){ struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { if (sym->bind == STB_LOCAL && sym->file == file && !strcmp(sym->name, name)) { return sym; } } return NULL;} struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name){ struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { if (!strcmp(sym->name, name) && !is_local_sym(sym)) return sym; } return NULL;} void iterate_global_symbol_by_demangled_name(const struct elf *elf, const char *demangled_name, void (*process)(struct symbol *sym, void *data), void *data){ struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(demangled_name)) { if (!strcmp(sym->demangled_name, demangled_name) && !is_local_sym(sym)) process(sym, data); }} struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec, unsigned long offset, unsigned int len){ struct reloc *reloc, *r = NULL; struct section *rsec; unsigned long o; rsec = sec->rsec; if (!rsec) return NULL; for_offset_range(o, offset, offset + len) { elf_hash_for_each_possible(reloc, reloc, hash, sec_offset_hash(rsec, o)) { if (reloc->sec != rsec) continue; if (reloc_offset(reloc) >= offset && reloc_offset(reloc) < offset + len) { if (!r || reloc_offset(reloc) < reloc_offset(r)) r = reloc; } } if (r) return r; } return NULL;} struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset){ return find_reloc_by_dest_range(elf, sec, offset, 1);} static bool is_dwarf_section(struct section *sec){ return !strncmp(sec->name, ".debug_", 7);} static int read_sections(struct elf *elf){ Elf_Scn *s = NULL; struct section *sec; size_t shstrndx, sections_nr; int i; if (elf_getshdrnum(elf->elf, §ions_nr)) { ERROR_ELF("elf_getshdrnum"); return -1; } if (elf_getshdrstrndx(elf->elf, &shstrndx)) { ERROR_ELF("elf_getshdrstrndx"); return -1; } if (!elf_alloc_hash(section, sections_nr) || !elf_alloc_hash(section_name, sections_nr)) return -1; elf->section_data = calloc(sections_nr, sizeof(*sec)); if (!elf->section_data) { ERROR_GLIBC("calloc"); return -1; } for (i = 0; i < sections_nr; i++) { sec = &elf->section_data[i]; INIT_LIST_HEAD(&sec->symbol_list); s = elf_getscn(elf->elf, i); if (!s) { ERROR_ELF("elf_getscn"); return -1; } sec->idx = elf_ndxscn(s); if (!gelf_getshdr(s, &sec->sh)) { ERROR_ELF("gelf_getshdr"); return -1; } sec->name = elf_strptr(elf->elf, shstrndx, sec->sh.sh_name); if (!sec->name) { ERROR_ELF("elf_strptr"); return -1; } if (sec_size(sec) != 0 && !is_dwarf_section(sec)) { sec->data = elf_getdata(s, NULL); if (!sec->data) { ERROR_ELF("elf_getdata"); return -1; } if (sec->data->d_off != 0 || sec->data->d_size != sec_size(sec)) { ERROR("unexpected data attributes for %s", sec->name); return -1; } } list_add_tail(&sec->list, &elf->sections); elf_hash_add(section, &sec->hash, sec->idx); elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name)); if (is_reloc_sec(sec)) elf->num_relocs += sec_num_entries(sec); } if (opts.stats) { printf("nr_sections: %lu\n", (unsigned long)sections_nr); printf("section_bits: %d\n", elf->section_bits); } /* sanity check, one more call to elf_nextscn() should return NULL */ if (elf_nextscn(elf->elf, s)) { ERROR("section entry mismatch"); return -1; } return 0;} /* * Returns desired length of the demangled name. * If name doesn't need demangling, return strlen(name). */static ssize_t demangled_name_len(const char *name){ ssize_t idx; const char *p; p = strstr(name, ".llvm."); if (p) return p - name; if (!strstarts(name, "__UNIQUE_ID_") && !strchr(name, '.')) return strlen(name); for (idx = strlen(name) - 1; idx >= 0; idx--) { char c = name[idx]; if (!isdigit(c) && c != '.' && c != '_') break; } if (idx <= 0) return strlen(name); return idx + 1;} /* * Remove number suffix of a symbol. * * Specifically, remove trailing numbers for "__UNIQUE_ID_" symbols and * symbols with '.'. * * With CONFIG_LTO_CLANG_THIN, it is possible to have nested __UNIQUE_ID_, * such as * * __UNIQUE_ID_addressable___UNIQUE_ID_pci_invalid_bar_694_695 *Lines 1–500 of 1838. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * elf.c - ELF access library * * Adapted from kpatch (https://github.com/dynup/kpatch): * Copyright (C) 2013-2015 Josh Poimboeuf <jpoimboe@redhat.com> * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com> */ #include <sys/types.h> #include <sys/stat.h> #include <sys/mman.h> #include <fcntl.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <unistd.h> #include <errno.h> #include <ctype.h> #include <linux/align.h> #include <linux/kernel.h> #include <linux/interval_tree_generic.h> #include <linux/log2.h> #include <objtool/builtin.h> #include <objtool/elf.h> #include <objtool/warn.h> static ssize_t demangled_name_len(const char *name); static inline u32 str_hash(const char *str) { return jhash(str, strlen(str), 0); } static inline u32 str_hash_demangled(const char *str) { return jhash(str, demangled_name_len(str), 0); } #define __elf_table(name) (elf->name##_hash) #define __elf_bits(name) (elf->name##_bits) #define __elf_table_entry(name, key) \ __elf_table(name)[hash_min(key, __elf_bits(name))] #define elf_hash_add(name, node, key) \ ({ \ struct elf_hash_node *__node = node; \ __node->next = __elf_table_entry(name, key); \ __elf_table_entry(name, key) = __node; \ }) static inline void __elf_hash_del(struct elf_hash_node *node, struct elf_hash_node **head) { struct elf_hash_node *cur, *prev; if (node == *head) { *head = node->next; return; } for (prev = NULL, cur = *head; cur; prev = cur, cur = cur->next) { if (cur == node) { prev->next = cur->next; break; } } } #define elf_hash_del(name, node, key) \ __elf_hash_del(node, &__elf_table_entry(name, key)) #define elf_list_entry(ptr, type, member) \ ({ \ typeof(ptr) __ptr = (ptr); \ __ptr ? container_of(__ptr, type, member) : NULL; \ }) #define elf_hash_for_each_possible(name, obj, member, key) \ for (obj = elf_list_entry(__elf_table_entry(name, key), typeof(*obj), member); \ obj; \ obj = elf_list_entry(obj->member.next, typeof(*(obj)), member)) #define elf_alloc_hash(name, size) \ ({ \ __elf_bits(name) = max(10, ilog2(size)); \ __elf_table(name) = mmap(NULL, sizeof(struct elf_hash_node *) << __elf_bits(name), \ PROT_READ|PROT_WRITE, \ MAP_PRIVATE|MAP_ANON, -1, 0); \ if (__elf_table(name) == (void *)-1L) { \ ERROR_GLIBC("mmap fail " #name); \ __elf_table(name) = NULL; \ } \ __elf_table(name); \ }) static inline unsigned long __sym_start(struct symbol *s) { return s->offset; } static inline unsigned long __sym_last(struct symbol *s) { return s->offset + (s->len ? s->len - 1 : 0); } INTERVAL_TREE_DEFINE(struct symbol, node, unsigned long, __subtree_last, __sym_start, __sym_last, static inline __maybe_unused, __sym) #define __sym_for_each(_iter, _tree, _start, _end) \ for (_iter = __sym_iter_first((_tree), (_start), (_end)); \ _iter; _iter = __sym_iter_next(_iter, (_start), (_end))) struct symbol_hole { unsigned long key; const struct symbol *sym; }; /* * Find the last symbol before @offset. */ static int symbol_hole_by_offset(const void *key, const struct rb_node *node) { const struct symbol *s = rb_entry(node, struct symbol, node); struct symbol_hole *sh = (void *)key; if (sh->key < s->offset) return -1; if (sh->key >= s->offset + s->len) { sh->sym = s; return 1; } return 0; } struct section *find_section_by_name(const struct elf *elf, const char *name) { struct section *sec; elf_hash_for_each_possible(section_name, sec, name_hash, str_hash(name)) { if (!strcmp(sec->name, name)) return sec; } return NULL; } static struct section *find_section_by_index(struct elf *elf, unsigned int idx) { struct section *sec; elf_hash_for_each_possible(section, sec, hash, idx) { if (sec->idx == idx) return sec; } return NULL; } static struct symbol *find_symbol_by_index(struct elf *elf, unsigned int idx) { struct symbol *sym; elf_hash_for_each_possible(symbol, sym, hash, idx) { if (sym->idx == idx) return sym; } return NULL; } struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset) { struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *sym; __sym_for_each(sym, tree, offset, offset) { if (sym->offset == offset && !is_sec_sym(sym)) return sym->alias; } return NULL; } struct symbol *find_func_by_offset(struct section *sec, unsigned long offset) { struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *func; __sym_for_each(func, tree, offset, offset) { if (func->offset == offset && is_func_sym(func)) return func->alias; } return NULL; } struct symbol *find_symbol_containing(const struct section *sec, unsigned long offset) { struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *sym = NULL, *tmp; __sym_for_each(tmp, tree, offset, offset) { if (tmp->len) { if (!sym) { sym = tmp; continue; } if (sym->offset != tmp->offset || sym->len != tmp->len) { /* * In the rare case of overlapping symbols, * pick the smaller one. * * TODO: outlaw overlapping symbols */ if (tmp->len < sym->len) sym = tmp; } } } return sym ? sym->alias : NULL; } /* * Returns size of hole starting at @offset. */ int find_symbol_hole_containing(const struct section *sec, unsigned long offset) { struct symbol_hole hole = { .key = offset, .sym = NULL, }; struct rb_node *n; struct symbol *s; /* * Find the rightmost symbol for which @offset is after it. */ n = rb_find(&hole, &sec->symbol_tree.rb_root, symbol_hole_by_offset); /* found a symbol that contains @offset */ if (n) return 0; /* not a hole */ /* * @offset >= sym->offset + sym->len, find symbol after it. * When hole.sym is empty, use the first node to compute the hole. * If there is no symbol in the section, the first node will be NULL, * in which case, -1 is returned to skip the whole section. */ if (hole.sym) n = rb_next(&hole.sym->node); else n = rb_first_cached(&sec->symbol_tree); if (!n) return -1; /* until end of address space */ /* hole until start of next symbol */ s = rb_entry(n, struct symbol, node); return s->offset - offset; } struct symbol *find_func_containing(struct section *sec, unsigned long offset) { struct rb_root_cached *tree = (struct rb_root_cached *)&sec->symbol_tree; struct symbol *func; __sym_for_each(func, tree, offset, offset) { if (is_func_sym(func)) return func->alias; } return NULL; } struct symbol *find_symbol_by_name(const struct elf *elf, const char *name) { struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(name)) { if (!strcmp(sym->name, name)) return sym; } return NULL; } /* Find local symbol with matching STT_FILE */ static struct symbol *find_local_symbol_by_file_and_name(const struct elf *elf, struct symbol *file, const char *name) { struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { if (sym->bind == STB_LOCAL && sym->file == file && !strcmp(sym->name, name)) { return sym; } } return NULL; } struct symbol *find_global_symbol_by_name(const struct elf *elf, const char *name) { struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash_demangled(name)) { if (!strcmp(sym->name, name) && !is_local_sym(sym)) return sym; } return NULL; } void iterate_global_symbol_by_demangled_name(const struct elf *elf, const char *demangled_name, void (*process)(struct symbol *sym, void *data), void *data) { struct symbol *sym; elf_hash_for_each_possible(symbol_name, sym, name_hash, str_hash(demangled_name)) { if (!strcmp(sym->demangled_name, demangled_name) && !is_local_sym(sym)) process(sym, data); } } struct reloc *find_reloc_by_dest_range(const struct elf *elf, struct section *sec, unsigned long offset, unsigned int len) { struct reloc *reloc, *r = NULL; struct section *rsec; unsigned long o; rsec = sec->rsec; if (!rsec) return NULL; for_offset_range(o, offset, offset + len) { elf_hash_for_each_possible(reloc, reloc, hash, sec_offset_hash(rsec, o)) { if (reloc->sec != rsec) continue; if (reloc_offset(reloc) >= offset && reloc_offset(reloc) < offset + len) { if (!r || reloc_offset(reloc) < reloc_offset(r)) r = reloc; } } if (r) return r; } return NULL; } struct reloc *find_reloc_by_dest(const struct elf *elf, struct section *sec, unsigned long offset) { return find_reloc_by_dest_range(elf, sec, offset, 1); } static bool is_dwarf_section(struct section *sec) { return !strncmp(sec->name, ".debug_", 7); } static int read_sections(struct elf *elf) { Elf_Scn *s = NULL; struct section *sec; size_t shstrndx, sections_nr; int i; if (elf_getshdrnum(elf->elf, §ions_nr)) { ERROR_ELF("elf_getshdrnum"); return -1; } if (elf_getshdrstrndx(elf->elf, &shstrndx)) { ERROR_ELF("elf_getshdrstrndx"); return -1; } if (!elf_alloc_hash(section, sections_nr) || !elf_alloc_hash(section_name, sections_nr)) return -1; elf->section_data = calloc(sections_nr, sizeof(*sec)); if (!elf->section_data) { ERROR_GLIBC("calloc"); return -1; } for (i = 0; i < sections_nr; i++) { sec = &elf->section_data[i]; INIT_LIST_HEAD(&sec->symbol_list); s = elf_getscn(elf->elf, i); if (!s) { ERROR_ELF("elf_getscn"); return -1; } sec->idx = elf_ndxscn(s); if (!gelf_getshdr(s, &sec->sh)) { ERROR_ELF("gelf_getshdr"); return -1; } sec->name = elf_strptr(elf->elf, shstrndx, sec->sh.sh_name); if (!sec->name) { ERROR_ELF("elf_strptr"); return -1; } if (sec_size(sec) != 0 && !is_dwarf_section(sec)) { sec->data = elf_getdata(s, NULL); if (!sec->data) { ERROR_ELF("elf_getdata"); return -1; } if (sec->data->d_off != 0 || sec->data->d_size != sec_size(sec)) { ERROR("unexpected data attributes for %s", sec->name); return -1; } } list_add_tail(&sec->list, &elf->sections); elf_hash_add(section, &sec->hash, sec->idx); elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name)); if (is_reloc_sec(sec)) elf->num_relocs += sec_num_entries(sec); } if (opts.stats) { printf("nr_sections: %lu\n", (unsigned long)sections_nr); printf("section_bits: %d\n", elf->section_bits); } /* sanity check, one more call to elf_nextscn() should return NULL */ if (elf_nextscn(elf->elf, s)) { ERROR("section entry mismatch"); return -1; } return 0; } /* * Returns desired length of the demangled name. * If name doesn't need demangling, return strlen(name). */ static ssize_t demangled_name_len(const char *name) { ssize_t idx; const char *p; p = strstr(name, ".llvm."); if (p) return p - name; if (!strstarts(name, "__UNIQUE_ID_") && !strchr(name, '.')) return strlen(name); for (idx = strlen(name) - 1; idx >= 0; idx--) { char c = name[idx]; if (!isdigit(c) && c != '.' && c != '_') break; } if (idx <= 0) return strlen(name); return idx + 1; } /* * Remove number suffix of a symbol. * * Specifically, remove trailing numbers for "__UNIQUE_ID_" symbols and * symbols with '.'. * * With CONFIG_LTO_CLANG_THIN, it is possible to have nested __UNIQUE_ID_, * such as * * __UNIQUE_ID_addressable___UNIQUE_ID_pci_invalid_bar_694_695 *