libbpf: Update gen_loader to emit BTF_KIND_FUNC relocations

This change updates the BPF syscall loader to relocate BTF_KIND_FUNC relocations, with support for weak kfunc relocations. The general idea is to move map_fds to loader map, and also use the data for storing kfunc BTF fds. Since both reuse the fd_array parameter, they need to be kept together. For map_fds, we reserve MAX_USED_MAPS slots in a region, and for kfunc, we reserve MAX_KFUNC_DESCS. This is done so that insn->off has more chances of being <= INT16_MAX than treating data map as a sparse array and adding fd as needed. When the MAX_KFUNC_DESCS limit is reached, we fall back to the sparse array model, so that as long as it does remain <= INT16_MAX, we pass an index relative to the start of fd_array. We store all ksyms in an array where we try to avoid calling the bpf_btf_find_by_name_kind helper, and also reuse the BTF fd that was already stored. This also speeds up the loading process compared to emitting calls in all cases, in later tests.

Kumar Kartikeya Dwivedi committed 5 years ago

Commit 18f4fcc1 parent 466b2e13 files · +280 −58Browse files

Unified · Split

3 files

tools/lib/bpf/bpf_gen_internal.h +15 −1
tools/lib/bpf/gen_loader.c +261 −53
tools/lib/bpf/libbpf.c +4 −4

tools/lib/bpf/bpf_gen_internal.h+15−1

@@ -7,6 +7,15 @@ struct ksym_relo_desc { · 6 unmodified lines
7	7	const char *name;
8	8	int kind;
9	9	int insn_idx;
	10	+ bool is_weak;
	11	+};
	12	+
	13	+struct ksym_desc {
	14	+ const char *name;
	15	+ int ref;
	16	+ int kind;
	17	+ int off;
	18	+ int insn;
10	19	};
11	20
12	21	struct bpf_gen {
@@ -24,6 +33,10 @@ struct bpf_gen { · 11 unmodified lines
24	33	int relo_cnt;
25	34	char attach_target[128];
26	35	int attach_kind;
	36	+ struct ksym_desc *ksyms;
	37	+ __u32 nr_ksyms;
	38	+ int fd_array;
	39	+ int nr_fd_array;
27	40	};
28	41
29	42	void bpf_gen__init(struct bpf_gen *gen, int log_level);
@@ -36,6 +49,7 @@ void bpf_gen__prog_load(struct bpf_gen gen, struct bpf_prog_load_params load_a · 6 unmodified lines
36	49	void bpf_gen__map_update_elem(struct bpf_gen gen, int map_idx, void value, __u32 value_size);
37	50	void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx);
38	51	void bpf_gen__record_attach_target(struct bpf_gen gen, const char name, enum bpf_attach_type type);
39		−void bpf_gen__record_extern(struct bpf_gen gen, const char name, int kind, int insn_idx);
	52	+void bpf_gen__record_extern(struct bpf_gen gen, const char name, bool is_weak, int kind,
	53	+ int insn_idx);
40	54
41	55	#endif

tools/lib/bpf/gen_loader.c+261−53

@@ -14,8 +14,10 @@ · 13 unmodified lines
14	14	#include "bpf_gen_internal.h"
15	15	#include "skel_internal.h"
16	16
17		−#define MAX_USED_MAPS 64
18		−#define MAX_USED_PROGS 32
	17	+#define MAX_USED_MAPS 64
	18	+#define MAX_USED_PROGS 32
	19	+#define MAX_KFUNC_DESCS 256
	20	+#define MAX_FD_ARRAY_SZ (MAX_USED_PROGS + MAX_KFUNC_DESCS)
19	21
20	22	/* The following structure describes the stack layout of the loader program.
21	23	* In addition R6 contains the pointer to context.
@@ -30,7 +32,6 @@ · 8 unmodified lines
30	32	*/
31	33	struct loader_stack {
32	34	__u32 btf_fd;
33		− __u32 map_fd[MAX_USED_MAPS];
34	35	__u32 prog_fd[MAX_USED_PROGS];
35	36	__u32 inner_map_fd;
36	37	};
@@ -143,13 +144,49 @@ static int add_data(struct bpf_gen gen, const void data, __u32 size) · 106 unmodified lines
143	144	if (realloc_data_buf(gen, size8))
144	145	return 0;
145	146	prev = gen->data_cur;
146		− memcpy(gen->data_cur, data, size);
147		− gen->data_cur += size;
148		− memcpy(gen->data_cur, &zero, size8 - size);
149		− gen->data_cur += size8 - size;
	147	+ if (data) {
	148	+ memcpy(gen->data_cur, data, size);
	149	+ memcpy(gen->data_cur + size, &zero, size8 - size);
	150	+ } else {
	151	+ memset(gen->data_cur, 0, size8);
	152	+ }
	153	+ gen->data_cur += size8;
150	154	return prev - gen->data_start;
151	155	}
152	156
	157	+/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
	158	+ * to start of fd_array. Caller can decide if it is usable or not.
	159	+ */
	160	+static int add_map_fd(struct bpf_gen *gen)
	161	+{
	162	+ if (!gen->fd_array)
	163	+ gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
	164	+ if (gen->nr_maps == MAX_USED_MAPS) {
	165	+ pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
	166	+ gen->error = -E2BIG;
	167	+ return 0;
	168	+ }
	169	+ return gen->nr_maps++;
	170	+}
	171	+
	172	+static int add_kfunc_btf_fd(struct bpf_gen *gen)
	173	+{
	174	+ int cur;
	175	+
	176	+ if (!gen->fd_array)
	177	+ gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
	178	+ if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
	179	+ cur = add_data(gen, NULL, sizeof(int));
	180	+ return (cur - gen->fd_array) / sizeof(int);
	181	+ }
	182	+ return MAX_USED_MAPS + gen->nr_fd_array++;
	183	+}
	184	+
	185	+static int blob_fd_array_off(struct bpf_gen *gen, int index)
	186	+{
	187	+ return gen->fd_array + index * sizeof(int);
	188	+}
	189	+
153	190	static int insn_bytes_to_bpf_size(__u32 sz)
154	191	{
155	192	switch (sz) {
@@ -171,14 +208,22 @@ static void emit_rel_store(struct bpf_gen *gen, int off, int data) · 15 unmodified lines
171	208	emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
172	209	}
173	210
174		−/* (u64 )(blob + off) = (u64)(void )(%sp + stack_off) /
175		−static void emit_rel_store_sp(struct bpf_gen *gen, int off, int stack_off)
	211	+static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
176	212	{
177		− emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_10));
178		− emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, stack_off));
	213	+ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
	214	+ 0, 0, 0, blob_off));
	215	+ emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
179	216	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
180	217	0, 0, 0, off));
181		− emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
	218	+ emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
	219	+}
	220	+
	221	+static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
	222	+{
	223	+ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
	224	+ 0, 0, 0, blob_off));
	225	+ emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
	226	+ emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
182	227	}
183	228
184	229	static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
@@ -326,11 +371,11 @@ int bpf_gen__finish(struct bpf_gen *gen) · 141 unmodified lines
326	371	offsetof(struct bpf_prog_desc, prog_fd), 4,
327	372	stack_off(prog_fd[i]));
328	373	for (i = 0; i < gen->nr_maps; i++)
329		− move_stack2ctx(gen,
330		− sizeof(struct bpf_loader_ctx) +
331		− sizeof(struct bpf_map_desc) * i +
332		− offsetof(struct bpf_map_desc, map_fd), 4,
333		− stack_off(map_fd[i]));
	374	+ move_blob2ctx(gen,
	375	+ sizeof(struct bpf_loader_ctx) +
	376	+ sizeof(struct bpf_map_desc) * i +
	377	+ offsetof(struct bpf_map_desc, map_fd), 4,
	378	+ blob_fd_array_off(gen, i));
334	379	emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
335	380	emit(gen, BPF_EXIT_INSN());
336	381	pr_debug("gen: finish %d\n", gen->error);
@@ -390,7 +435,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 53 unmodified lines
390	435	{
391	436	int attr_size = offsetofend(union bpf_attr, btf_vmlinux_value_type_id);
392	437	bool close_inner_map_fd = false;
393		− int map_create_attr;
	438	+ int map_create_attr, idx;
394	439	union bpf_attr attr;
395	440
396	441	memset(&attr, 0, attr_size);
@@ -467,9 +512,11 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 70 unmodified lines
467	512	gen->error = -EDOM; /* internal bug */
468	513	return;
469	514	} else {
470		− emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
471		− stack_off(map_fd[map_idx])));
472		− gen->nr_maps++;
	515	+ /* add_map_fd does gen->nr_maps++ */
	516	+ idx = add_map_fd(gen);
	517	+ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
	518	+ 0, 0, 0, blob_fd_array_off(gen, idx)));
	519	+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
473	520	}
474	521	if (close_inner_map_fd)
475	522	emit_sys_close_stack(gen, stack_off(inner_map_fd));
@@ -511,8 +558,8 @@ static void emit_find_attach_target(struct bpf_gen *gen) · 35 unmodified lines
511	558	*/
512	559	}
513	560
514		−void bpf_gen__record_extern(struct bpf_gen gen, const char name, int kind,
515		− int insn_idx)
	561	+void bpf_gen__record_extern(struct bpf_gen gen, const char name, bool is_weak,
	562	+ int kind, int insn_idx)
516	563	{
517	564	struct ksym_relo_desc *relo;
518	565
@@ -524,38 +571,192 @@ void bpf_gen__record_extern(struct bpf_gen gen, const char name, int kind, · 5 unmodified lines
524	571	gen->relos = relo;
525	572	relo += gen->relo_cnt;
526	573	relo->name = name;
	574	+ relo->is_weak = is_weak;
527	575	relo->kind = kind;
528	576	relo->insn_idx = insn_idx;
529	577	gen->relo_cnt++;
530	578	}
531	579
532		−static void emit_relo(struct bpf_gen gen, struct ksym_relo_desc relo, int insns)
	580	+/* returns existing ksym_desc with ref incremented, or inserts a new one */
	581	+static struct ksym_desc get_ksym_desc(struct bpf_gen gen, struct ksym_relo_desc *relo)
533	582	{
534		− int name, insn, len = strlen(relo->name) + 1;
	583	+ struct ksym_desc *kdesc;
535	584
536		− pr_debug("gen: emit_relo: %s at %d\n", relo->name, relo->insn_idx);
537		− name = add_data(gen, relo->name, len);
	585	+ for (int i = 0; i < gen->nr_ksyms; i++) {
	586	+ if (!strcmp(gen->ksyms[i].name, relo->name)) {
	587	+ gen->ksyms[i].ref++;
	588	+ return &gen->ksyms[i];
	589	+ }
	590	+ }
	591	+ kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
	592	+ if (!kdesc) {
	593	+ gen->error = -ENOMEM;
	594	+ return NULL;
	595	+ }
	596	+ gen->ksyms = kdesc;
	597	+ kdesc = &gen->ksyms[gen->nr_ksyms++];
	598	+ kdesc->name = relo->name;
	599	+ kdesc->kind = relo->kind;
	600	+ kdesc->ref = 1;
	601	+ kdesc->off = 0;
	602	+ kdesc->insn = 0;
	603	+ return kdesc;
	604	+}
	605	+
	606	+/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
	607	+ * Returns result in BPF_REG_7
	608	+ */
	609	+static void emit_bpf_find_by_name_kind(struct bpf_gen gen, struct ksym_relo_desc relo)
	610	+{
	611	+ int name_off, len = strlen(relo->name) + 1;
538	612
	613	+ name_off = add_data(gen, relo->name, len);
539	614	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
540		− 0, 0, 0, name));
	615	+ 0, 0, 0, name_off));
541	616	emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
542	617	emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
543	618	emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
544	619	emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
545	620	emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
546	621	debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
547		− emit_check_err(gen);
	622	+}
	623	+
	624	+/* Expects:
	625	+ * BPF_REG_8 - pointer to instruction
	626	+ *
	627	+ * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
	628	+ * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
	629	+ * this would mean a new BTF fd index for each entry. By pairing symbol name
	630	+ * with index, we get the insn->imm, insn->off pairing that kernel uses for
	631	+ * kfunc_tab, which becomes the effective limit even though all of them may
	632	+ * share same index in fd_array (such that kfunc_btf_tab has 1 element).
	633	+ */
	634	+static void emit_relo_kfunc_btf(struct bpf_gen gen, struct ksym_relo_desc relo, int insn)
	635	+{
	636	+ struct ksym_desc *kdesc;
	637	+ int btf_fd_idx;
	638	+
	639	+ kdesc = get_ksym_desc(gen, relo);
	640	+ if (!kdesc)
	641	+ return;
	642	+ /* try to copy from existing bpf_insn */
	643	+ if (kdesc->ref > 1) {
	644	+ move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
	645	+ kdesc->insn + offsetof(struct bpf_insn, imm));
	646	+ move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
	647	+ kdesc->insn + offsetof(struct bpf_insn, off));
	648	+ goto log;
	649	+ }
	650	+ /* remember insn offset, so we can copy BTF ID and FD later */
	651	+ kdesc->insn = insn;
	652	+ emit_bpf_find_by_name_kind(gen, relo);
	653	+ if (!relo->is_weak)
	654	+ emit_check_err(gen);
	655	+ /* get index in fd_array to store BTF FD at */
	656	+ btf_fd_idx = add_kfunc_btf_fd(gen);
	657	+ if (btf_fd_idx > INT16_MAX) {
	658	+ pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
	659	+ btf_fd_idx, relo->name);
	660	+ gen->error = -E2BIG;
	661	+ return;
	662	+ }
	663	+ kdesc->off = btf_fd_idx;
	664	+ /* set a default value for imm */
	665	+ emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
	666	+ /* skip success case store if ret < 0 */
	667	+ emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 1));
548	668	/* store btf_id into insn[insn_idx].imm */
549		− insn = insns + sizeof(struct bpf_insn) * relo->insn_idx +
550		− offsetof(struct bpf_insn, imm);
	669	+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
	670	+ /* load fd_array slot pointer */
551	671	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
552		− 0, 0, 0, insn));
553		− emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 0));
554		− if (relo->kind == BTF_KIND_VAR) {
555		− /* store btf_obj_fd into insn[insn_idx + 1].imm */
556		− emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
557		− emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
558		− sizeof(struct bpf_insn)));
	672	+ 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
	673	+ /* skip store of BTF fd if ret < 0 */
	674	+ emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 3));
	675	+ /* store BTF fd in slot */
	676	+ emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
	677	+ emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
	678	+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
	679	+ /* set a default value for off */
	680	+ emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
	681	+ /* skip insn->off store if ret < 0 */
	682	+ emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 2));
	683	+ /* skip if vmlinux BTF */
	684	+ emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_9, 0, 1));
	685	+ /* store index into insn[insn_idx].off */
	686	+ emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
	687	+log:
	688	+ if (!gen->log_level)
	689	+ return;
	690	+ emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
	691	+ offsetof(struct bpf_insn, imm)));
	692	+ emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
	693	+ offsetof(struct bpf_insn, off)));
	694	+ debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
	695	+ relo->name, kdesc->ref);
	696	+ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
	697	+ 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
	698	+ emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
	699	+ debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
	700	+ relo->name, kdesc->ref);
	701	+}
	702	+
	703	+/* Expects:
	704	+ * BPF_REG_8 - pointer to instruction
	705	+ */
	706	+static void emit_relo_ksym_btf(struct bpf_gen gen, struct ksym_relo_desc relo, int insn)
	707	+{
	708	+ struct ksym_desc *kdesc;
	709	+
	710	+ kdesc = get_ksym_desc(gen, relo);
	711	+ if (!kdesc)
	712	+ return;
	713	+ /* try to copy from existing ldimm64 insn */
	714	+ if (kdesc->ref > 1) {
	715	+ move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
	716	+ kdesc->insn + offsetof(struct bpf_insn, imm));
	717	+ move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
	718	+ kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
	719	+ goto log;
	720	+ }
	721	+ /* remember insn offset, so we can copy BTF ID and FD later */
	722	+ kdesc->insn = insn;
	723	+ emit_bpf_find_by_name_kind(gen, relo);
	724	+ emit_check_err(gen);
	725	+ /* store btf_id into insn[insn_idx].imm */
	726	+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
	727	+ /* store btf_obj_fd into insn[insn_idx + 1].imm */
	728	+ emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
	729	+ emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
	730	+ sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
	731	+log:
	732	+ if (!gen->log_level)
	733	+ return;
	734	+ emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
	735	+ offsetof(struct bpf_insn, imm)));
	736	+ emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
	737	+ offsetof(struct bpf_insn, imm)));
	738	+ debug_regs(gen, BPF_REG_7, BPF_REG_9, " var (%s:count=%d): imm: %%d, fd: %%d",
	739	+ relo->name, kdesc->ref);
	740	+}
	741	+
	742	+static void emit_relo(struct bpf_gen gen, struct ksym_relo_desc relo, int insns)
	743	+{
	744	+ int insn;
	745	+
	746	+ pr_debug("gen: emit_relo (%d): %s at %d\n", relo->kind, relo->name, relo->insn_idx);
	747	+ insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
	748	+ emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
	749	+ switch (relo->kind) {
	750	+ case BTF_KIND_VAR:
	751	+ emit_relo_ksym_btf(gen, relo, insn);
	752	+ break;
	753	+ case BTF_KIND_FUNC:
	754	+ emit_relo_kfunc_btf(gen, relo, insn);
	755	+ break;
	756	+ default:
	757	+ pr_warn("Unknown relocation kind '%d'\n", relo->kind);
	758	+ gen->error = -EDOM;
	759	+ return;
559	760	}
560	761	}
561	762
@@ -571,14 +772,22 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) · 9 unmodified lines
571	772	{
572	773	int i, insn;
573	774
574		− for (i = 0; i < gen->relo_cnt; i++) {
575		− if (gen->relos[i].kind != BTF_KIND_VAR)
576		− continue;
577		− /* close fd recorded in insn[insn_idx + 1].imm */
578		− insn = insns +
579		− sizeof(struct bpf_insn) * (gen->relos[i].insn_idx + 1) +
580		− offsetof(struct bpf_insn, imm);
581		− emit_sys_close_blob(gen, insn);
	775	+ for (i = 0; i < gen->nr_ksyms; i++) {
	776	+ if (gen->ksyms[i].kind == BTF_KIND_VAR) {
	777	+ /* close fd recorded in insn[insn_idx + 1].imm */
	778	+ insn = gen->ksyms[i].insn;
	779	+ insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
	780	+ emit_sys_close_blob(gen, insn);
	781	+ } else { /* BTF_KIND_FUNC */
	782	+ emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->ksyms[i].off));
	783	+ if (gen->ksyms[i].off < MAX_FD_ARRAY_SZ)
	784	+ gen->nr_fd_array--;
	785	+ }
	786	+ }
	787	+ if (gen->nr_ksyms) {
	788	+ free(gen->ksyms);
	789	+ gen->nr_ksyms = 0;
	790	+ gen->ksyms = NULL;
582	791	}
583	792	if (gen->relo_cnt) {
584	793	free(gen->relos);
@@ -637,9 +846,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, · 52 unmodified lines
637	846	/* populate union bpf_attr with a pointer to line_info */
638	847	emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
639	848
640		− /* populate union bpf_attr fd_array with a pointer to stack where map_fds are saved */
641		− emit_rel_store_sp(gen, attr_field(prog_load_attr, fd_array),
642		− stack_off(map_fd[0]));
	849	+ /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
	850	+ emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
643	851
644	852	/* populate union bpf_attr with user provided log details */
645	853	move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
@@ -706,8 +914,8 @@ void bpf_gen__map_update_elem(struct bpf_gen gen, int map_idx, void pvalue, · 60 unmodified lines
706	914	emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
707	915
708	916	map_update_attr = add_data(gen, &attr, attr_size);
709		− move_stack2blob(gen, attr_field(map_update_attr, map_fd), 4,
710		− stack_off(map_fd[map_idx]));
	917	+ move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
	918	+ blob_fd_array_off(gen, map_idx));
711	919	emit_rel_store(gen, attr_field(map_update_attr, key), key);
712	920	emit_rel_store(gen, attr_field(map_update_attr, value), value);
713	921	/* emit MAP_UPDATE_ELEM command */
@@ -725,8 +933,8 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) · 11 unmodified lines
725	933	memset(&attr, 0, attr_size);
726	934	pr_debug("gen: map_freeze: idx %d\n", map_idx);
727	935	map_freeze_attr = add_data(gen, &attr, attr_size);
728		− move_stack2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
729		− stack_off(map_fd[map_idx]));
	936	+ move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
	937	+ blob_fd_array_off(gen, map_idx));
730	938	/* emit MAP_FREEZE command */
731	939	emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
732	940	debug_ret(gen, "map_freeze");

tools/lib/bpf/libbpf.c+4−4

@@ -6360,12 +6360,12 @@ static int bpf_program__record_externs(struct bpf_program *prog) · 6359 unmodified lines
6360	6360	ext->name);
6361	6361	return -ENOTSUP;
6362	6362	}
6363		− bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR,
6364		− relo->insn_idx);
	6363	+ bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
	6364	+ BTF_KIND_VAR, relo->insn_idx);
6365	6365	break;
6366	6366	case RELO_EXTERN_FUNC:
6367		− bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC,
6368		− relo->insn_idx);
	6367	+ bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
	6368	+ BTF_KIND_FUNC, relo->insn_idx);
6369	6369	break;
6370	6370	default:
6371	6371	continue;