History

libbpf: Update gen_loader to emit BTF_KIND_FUNC relocations

This change updates the BPF syscall loader to relocate BTF_KIND_FUNC relocations, with support for weak kfunc relocations. The general idea is to move map_fds to loader map, and also use the data for storing kfunc BTF fds. Since both reuse the fd_array parameter, they need to be kept together. For map_fds, we reserve MAX_USED_MAPS slots in a region, and for kfunc, we reserve MAX_KFUNC_DESCS. This is done so that insn->off has more chances of being <= INT16_MAX than treating data map as a sparse array and adding fd as needed. When the MAX_KFUNC_DESCS limit is reached, we fall back to the sparse array model, so that as long as it does remain <= INT16_MAX, we pass an index relative to the start of fd_array. We store all ksyms in an array where we try to avoid calling the bpf_btf_find_by_name_kind helper, and also reuse the BTF fd that was already stored. This also speeds up the loading process compared to emitting calls in all cases, in later tests.

Kumar Kartikeya Dwivedi committed 5 years ago

Commit 18f4fcc1 parent 466b2e13 files · +280 −58Browse files

Unified · Split

3 files
  1. tools/lib/bpf/bpf_gen_internal.h +15 −1
  2. tools/lib/bpf/gen_loader.c +261 −53
  3. tools/lib/bpf/libbpf.c +4 −4
tools/lib/bpf/bpf_gen_internal.h+15−1
@@ -7,6 +7,15 @@ struct ksym_relo_desc { · 6 unmodified lines
7 const char *name;7 const char *name;
8 int kind;8 int kind;
9 int insn_idx;9 int insn_idx;
10 bool is_weak;
11};
12
13struct ksym_desc {
14 const char *name;
15 int ref;
16 int kind;
17 int off;
18 int insn;
10};19};
11 20
12struct bpf_gen {21struct bpf_gen {
@@ -24,6 +33,10 @@ struct bpf_gen { · 11 unmodified lines
24 int relo_cnt;33 int relo_cnt;
25 char attach_target[128];34 char attach_target[128];
26 int attach_kind;35 int attach_kind;
36 struct ksym_desc *ksyms;
37 __u32 nr_ksyms;
38 int fd_array;
39 int nr_fd_array;
27};40};
28 41
29void bpf_gen__init(struct bpf_gen *gen, int log_level);42void bpf_gen__init(struct bpf_gen *gen, int log_level);
@@ -36,6 +49,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, struct bpf_prog_load_params *load_a · 6 unmodified lines
36void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size);49void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size);
37void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx);50void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx);
38void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type);51void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type);
39void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind, int insn_idx);52void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, int kind,
53 int insn_idx);
40 54
41#endif55#endif
tools/lib/bpf/gen_loader.c+261−53
@@ -14,8 +14,10 @@ · 13 unmodified lines
14#include "bpf_gen_internal.h"14#include "bpf_gen_internal.h"
15#include "skel_internal.h"15#include "skel_internal.h"
16 16
17#define MAX_USED_MAPS 6417#define MAX_USED_MAPS 64
18#define MAX_USED_PROGS 3218#define MAX_USED_PROGS 32
19#define MAX_KFUNC_DESCS 256
20#define MAX_FD_ARRAY_SZ (MAX_USED_PROGS + MAX_KFUNC_DESCS)
19 21
20/* The following structure describes the stack layout of the loader program.22/* The following structure describes the stack layout of the loader program.
21 * In addition R6 contains the pointer to context.23 * In addition R6 contains the pointer to context.
@@ -30,7 +32,6 @@ · 8 unmodified lines
30 */32 */
31struct loader_stack {33struct loader_stack {
32 __u32 btf_fd;34 __u32 btf_fd;
33 __u32 map_fd[MAX_USED_MAPS];
34 __u32 prog_fd[MAX_USED_PROGS];35 __u32 prog_fd[MAX_USED_PROGS];
35 __u32 inner_map_fd;36 __u32 inner_map_fd;
36};37};
@@ -143,13 +144,49 @@ static int add_data(struct bpf_gen *gen, const void *data, __u32 size) · 106 unmodified lines
143 if (realloc_data_buf(gen, size8))144 if (realloc_data_buf(gen, size8))
144 return 0;145 return 0;
145 prev = gen->data_cur;146 prev = gen->data_cur;
146 memcpy(gen->data_cur, data, size);147 if (data) {
147 gen->data_cur += size;148 memcpy(gen->data_cur, data, size);
148 memcpy(gen->data_cur, &zero, size8 - size);149 memcpy(gen->data_cur + size, &zero, size8 - size);
149 gen->data_cur += size8 - size;150 } else {
151 memset(gen->data_cur, 0, size8);
152 }
153 gen->data_cur += size8;
150 return prev - gen->data_start;154 return prev - gen->data_start;
151}155}
152 156
157/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
158 * to start of fd_array. Caller can decide if it is usable or not.
159 */
160static int add_map_fd(struct bpf_gen *gen)
161{
162 if (!gen->fd_array)
163 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
164 if (gen->nr_maps == MAX_USED_MAPS) {
165 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
166 gen->error = -E2BIG;
167 return 0;
168 }
169 return gen->nr_maps++;
170}
171
172static int add_kfunc_btf_fd(struct bpf_gen *gen)
173{
174 int cur;
175
176 if (!gen->fd_array)
177 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
178 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
179 cur = add_data(gen, NULL, sizeof(int));
180 return (cur - gen->fd_array) / sizeof(int);
181 }
182 return MAX_USED_MAPS + gen->nr_fd_array++;
183}
184
185static int blob_fd_array_off(struct bpf_gen *gen, int index)
186{
187 return gen->fd_array + index * sizeof(int);
188}
189
153static int insn_bytes_to_bpf_size(__u32 sz)190static int insn_bytes_to_bpf_size(__u32 sz)
154{191{
155 switch (sz) {192 switch (sz) {
@@ -171,14 +208,22 @@ static void emit_rel_store(struct bpf_gen *gen, int off, int data) · 15 unmodified lines
171 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));208 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
172}209}
173 210
174/* *(u64 *)(blob + off) = (u64)(void *)(%sp + stack_off) */211static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
175static void emit_rel_store_sp(struct bpf_gen *gen, int off, int stack_off)
176{212{
177 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_10));213 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
178 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, stack_off));214 0, 0, 0, blob_off));
215 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
179 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,216 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
180 0, 0, 0, off));217 0, 0, 0, off));
181 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));218 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
219}
220
221static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
222{
223 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
224 0, 0, 0, blob_off));
225 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
226 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
182}227}
183 228
184static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,229static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
@@ -326,11 +371,11 @@ int bpf_gen__finish(struct bpf_gen *gen) · 141 unmodified lines
326 offsetof(struct bpf_prog_desc, prog_fd), 4,371 offsetof(struct bpf_prog_desc, prog_fd), 4,
327 stack_off(prog_fd[i]));372 stack_off(prog_fd[i]));
328 for (i = 0; i < gen->nr_maps; i++)373 for (i = 0; i < gen->nr_maps; i++)
329 move_stack2ctx(gen,374 move_blob2ctx(gen,
330 sizeof(struct bpf_loader_ctx) +375 sizeof(struct bpf_loader_ctx) +
331 sizeof(struct bpf_map_desc) * i +376 sizeof(struct bpf_map_desc) * i +
332 offsetof(struct bpf_map_desc, map_fd), 4,377 offsetof(struct bpf_map_desc, map_fd), 4,
333 stack_off(map_fd[i]));378 blob_fd_array_off(gen, i));
334 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));379 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
335 emit(gen, BPF_EXIT_INSN());380 emit(gen, BPF_EXIT_INSN());
336 pr_debug("gen: finish %d\n", gen->error);381 pr_debug("gen: finish %d\n", gen->error);
@@ -390,7 +435,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 53 unmodified lines
390{435{
391 int attr_size = offsetofend(union bpf_attr, btf_vmlinux_value_type_id);436 int attr_size = offsetofend(union bpf_attr, btf_vmlinux_value_type_id);
392 bool close_inner_map_fd = false;437 bool close_inner_map_fd = false;
393 int map_create_attr;438 int map_create_attr, idx;
394 union bpf_attr attr;439 union bpf_attr attr;
395 440
396 memset(&attr, 0, attr_size);441 memset(&attr, 0, attr_size);
@@ -467,9 +512,11 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 70 unmodified lines
467 gen->error = -EDOM; /* internal bug */512 gen->error = -EDOM; /* internal bug */
468 return;513 return;
469 } else {514 } else {
470 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,515 /* add_map_fd does gen->nr_maps++ */
471 stack_off(map_fd[map_idx])));516 idx = add_map_fd(gen);
472 gen->nr_maps++;517 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
518 0, 0, 0, blob_fd_array_off(gen, idx)));
519 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
473 }520 }
474 if (close_inner_map_fd)521 if (close_inner_map_fd)
475 emit_sys_close_stack(gen, stack_off(inner_map_fd));522 emit_sys_close_stack(gen, stack_off(inner_map_fd));
@@ -511,8 +558,8 @@ static void emit_find_attach_target(struct bpf_gen *gen) · 35 unmodified lines
511 */558 */
512}559}
513 560
514void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind,561void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
515 int insn_idx)562 int kind, int insn_idx)
516{563{
517 struct ksym_relo_desc *relo;564 struct ksym_relo_desc *relo;
518 565
@@ -524,38 +571,192 @@ void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind, · 5 unmodified lines
524 gen->relos = relo;571 gen->relos = relo;
525 relo += gen->relo_cnt;572 relo += gen->relo_cnt;
526 relo->name = name;573 relo->name = name;
574 relo->is_weak = is_weak;
527 relo->kind = kind;575 relo->kind = kind;
528 relo->insn_idx = insn_idx;576 relo->insn_idx = insn_idx;
529 gen->relo_cnt++;577 gen->relo_cnt++;
530}578}
531 579
532static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)580/* returns existing ksym_desc with ref incremented, or inserts a new one */
581static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
533{582{
534 int name, insn, len = strlen(relo->name) + 1;583 struct ksym_desc *kdesc;
535 584
536 pr_debug("gen: emit_relo: %s at %d\n", relo->name, relo->insn_idx);585 for (int i = 0; i < gen->nr_ksyms; i++) {
537 name = add_data(gen, relo->name, len);586 if (!strcmp(gen->ksyms[i].name, relo->name)) {
587 gen->ksyms[i].ref++;
588 return &gen->ksyms[i];
589 }
590 }
591 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
592 if (!kdesc) {
593 gen->error = -ENOMEM;
594 return NULL;
595 }
596 gen->ksyms = kdesc;
597 kdesc = &gen->ksyms[gen->nr_ksyms++];
598 kdesc->name = relo->name;
599 kdesc->kind = relo->kind;
600 kdesc->ref = 1;
601 kdesc->off = 0;
602 kdesc->insn = 0;
603 return kdesc;
604}
605
606/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
607 * Returns result in BPF_REG_7
608 */
609static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
610{
611 int name_off, len = strlen(relo->name) + 1;
538 612
613 name_off = add_data(gen, relo->name, len);
539 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,614 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
540 0, 0, 0, name));615 0, 0, 0, name_off));
541 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));616 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
542 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));617 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
543 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));618 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
544 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));619 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
545 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));620 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
546 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);621 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
547 emit_check_err(gen);622}
623
624/* Expects:
625 * BPF_REG_8 - pointer to instruction
626 *
627 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
628 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
629 * this would mean a new BTF fd index for each entry. By pairing symbol name
630 * with index, we get the insn->imm, insn->off pairing that kernel uses for
631 * kfunc_tab, which becomes the effective limit even though all of them may
632 * share same index in fd_array (such that kfunc_btf_tab has 1 element).
633 */
634static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
635{
636 struct ksym_desc *kdesc;
637 int btf_fd_idx;
638
639 kdesc = get_ksym_desc(gen, relo);
640 if (!kdesc)
641 return;
642 /* try to copy from existing bpf_insn */
643 if (kdesc->ref > 1) {
644 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
645 kdesc->insn + offsetof(struct bpf_insn, imm));
646 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
647 kdesc->insn + offsetof(struct bpf_insn, off));
648 goto log;
649 }
650 /* remember insn offset, so we can copy BTF ID and FD later */
651 kdesc->insn = insn;
652 emit_bpf_find_by_name_kind(gen, relo);
653 if (!relo->is_weak)
654 emit_check_err(gen);
655 /* get index in fd_array to store BTF FD at */
656 btf_fd_idx = add_kfunc_btf_fd(gen);
657 if (btf_fd_idx > INT16_MAX) {
658 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
659 btf_fd_idx, relo->name);
660 gen->error = -E2BIG;
661 return;
662 }
663 kdesc->off = btf_fd_idx;
664 /* set a default value for imm */
665 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
666 /* skip success case store if ret < 0 */
667 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 1));
548 /* store btf_id into insn[insn_idx].imm */668 /* store btf_id into insn[insn_idx].imm */
549 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx +669 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
550 offsetof(struct bpf_insn, imm);670 /* load fd_array slot pointer */
551 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,671 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
552 0, 0, 0, insn));672 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
553 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 0));673 /* skip store of BTF fd if ret < 0 */
554 if (relo->kind == BTF_KIND_VAR) {674 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 3));
555 /* store btf_obj_fd into insn[insn_idx + 1].imm */675 /* store BTF fd in slot */
556 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));676 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
557 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,677 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
558 sizeof(struct bpf_insn)));678 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
679 /* set a default value for off */
680 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
681 /* skip insn->off store if ret < 0 */
682 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 2));
683 /* skip if vmlinux BTF */
684 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_9, 0, 1));
685 /* store index into insn[insn_idx].off */
686 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
687log:
688 if (!gen->log_level)
689 return;
690 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
691 offsetof(struct bpf_insn, imm)));
692 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
693 offsetof(struct bpf_insn, off)));
694 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
695 relo->name, kdesc->ref);
696 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
697 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
698 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
699 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
700 relo->name, kdesc->ref);
701}
702
703/* Expects:
704 * BPF_REG_8 - pointer to instruction
705 */
706static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
707{
708 struct ksym_desc *kdesc;
709
710 kdesc = get_ksym_desc(gen, relo);
711 if (!kdesc)
712 return;
713 /* try to copy from existing ldimm64 insn */
714 if (kdesc->ref > 1) {
715 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
716 kdesc->insn + offsetof(struct bpf_insn, imm));
717 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
718 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
719 goto log;
720 }
721 /* remember insn offset, so we can copy BTF ID and FD later */
722 kdesc->insn = insn;
723 emit_bpf_find_by_name_kind(gen, relo);
724 emit_check_err(gen);
725 /* store btf_id into insn[insn_idx].imm */
726 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
727 /* store btf_obj_fd into insn[insn_idx + 1].imm */
728 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
729 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
730 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
731log:
732 if (!gen->log_level)
733 return;
734 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
735 offsetof(struct bpf_insn, imm)));
736 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
737 offsetof(struct bpf_insn, imm)));
738 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var (%s:count=%d): imm: %%d, fd: %%d",
739 relo->name, kdesc->ref);
740}
741
742static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
743{
744 int insn;
745
746 pr_debug("gen: emit_relo (%d): %s at %d\n", relo->kind, relo->name, relo->insn_idx);
747 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
748 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
749 switch (relo->kind) {
750 case BTF_KIND_VAR:
751 emit_relo_ksym_btf(gen, relo, insn);
752 break;
753 case BTF_KIND_FUNC:
754 emit_relo_kfunc_btf(gen, relo, insn);
755 break;
756 default:
757 pr_warn("Unknown relocation kind '%d'\n", relo->kind);
758 gen->error = -EDOM;
759 return;
559 }760 }
560}761}
561 762
@@ -571,14 +772,22 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) · 9 unmodified lines
571{772{
572 int i, insn;773 int i, insn;
573 774
574 for (i = 0; i < gen->relo_cnt; i++) {775 for (i = 0; i < gen->nr_ksyms; i++) {
575 if (gen->relos[i].kind != BTF_KIND_VAR)776 if (gen->ksyms[i].kind == BTF_KIND_VAR) {
576 continue;777 /* close fd recorded in insn[insn_idx + 1].imm */
577 /* close fd recorded in insn[insn_idx + 1].imm */778 insn = gen->ksyms[i].insn;
578 insn = insns +779 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
579 sizeof(struct bpf_insn) * (gen->relos[i].insn_idx + 1) +780 emit_sys_close_blob(gen, insn);
580 offsetof(struct bpf_insn, imm);781 } else { /* BTF_KIND_FUNC */
581 emit_sys_close_blob(gen, insn);782 emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->ksyms[i].off));
783 if (gen->ksyms[i].off < MAX_FD_ARRAY_SZ)
784 gen->nr_fd_array--;
785 }
786 }
787 if (gen->nr_ksyms) {
788 free(gen->ksyms);
789 gen->nr_ksyms = 0;
790 gen->ksyms = NULL;
582 }791 }
583 if (gen->relo_cnt) {792 if (gen->relo_cnt) {
584 free(gen->relos);793 free(gen->relos);
@@ -637,9 +846,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, · 52 unmodified lines
637 /* populate union bpf_attr with a pointer to line_info */846 /* populate union bpf_attr with a pointer to line_info */
638 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);847 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
639 848
640 /* populate union bpf_attr fd_array with a pointer to stack where map_fds are saved */849 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
641 emit_rel_store_sp(gen, attr_field(prog_load_attr, fd_array),850 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
642 stack_off(map_fd[0]));
643 851
644 /* populate union bpf_attr with user provided log details */852 /* populate union bpf_attr with user provided log details */
645 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,853 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
@@ -706,8 +914,8 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, · 60 unmodified lines
706 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));914 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
707 915
708 map_update_attr = add_data(gen, &attr, attr_size);916 map_update_attr = add_data(gen, &attr, attr_size);
709 move_stack2blob(gen, attr_field(map_update_attr, map_fd), 4,917 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
710 stack_off(map_fd[map_idx]));918 blob_fd_array_off(gen, map_idx));
711 emit_rel_store(gen, attr_field(map_update_attr, key), key);919 emit_rel_store(gen, attr_field(map_update_attr, key), key);
712 emit_rel_store(gen, attr_field(map_update_attr, value), value);920 emit_rel_store(gen, attr_field(map_update_attr, value), value);
713 /* emit MAP_UPDATE_ELEM command */921 /* emit MAP_UPDATE_ELEM command */
@@ -725,8 +933,8 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) · 11 unmodified lines
725 memset(&attr, 0, attr_size);933 memset(&attr, 0, attr_size);
726 pr_debug("gen: map_freeze: idx %d\n", map_idx);934 pr_debug("gen: map_freeze: idx %d\n", map_idx);
727 map_freeze_attr = add_data(gen, &attr, attr_size);935 map_freeze_attr = add_data(gen, &attr, attr_size);
728 move_stack2blob(gen, attr_field(map_freeze_attr, map_fd), 4,936 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
729 stack_off(map_fd[map_idx]));937 blob_fd_array_off(gen, map_idx));
730 /* emit MAP_FREEZE command */938 /* emit MAP_FREEZE command */
731 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);939 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
732 debug_ret(gen, "map_freeze");940 debug_ret(gen, "map_freeze");
tools/lib/bpf/libbpf.c+4−4
@@ -6360,12 +6360,12 @@ static int bpf_program__record_externs(struct bpf_program *prog) · 6359 unmodified lines
6360 ext->name);6360 ext->name);
6361 return -ENOTSUP;6361 return -ENOTSUP;
6362 }6362 }
6363 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR,6363 bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
6364 relo->insn_idx);6364 BTF_KIND_VAR, relo->insn_idx);
6365 break;6365 break;
6366 case RELO_EXTERN_FUNC:6366 case RELO_EXTERN_FUNC:
6367 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC,6367 bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
6368 relo->insn_idx);6368 BTF_KIND_FUNC, relo->insn_idx);
6369 break;6369 break;
6370 default:6370 default:
6371 continue;6371 continue;