History

libbpf: Reduce bpf_core_apply_relo_insn() stack usage.

Reduce bpf_core_apply_relo_insn() stack usage and bump BPF_CORE_SPEC_MAX_LEN limit back to 64.

Alexei Starovoitov committed 4 years ago

Commit 78c1f8d1 parent 0bf40544 files · +61 −46Browse files

Unified · Split

4 files
  1. kernel/bpf/btf.c +10 −1
  2. tools/lib/bpf/libbpf.c +3 −1
  3. tools/lib/bpf/relo_core.c +19 −43
  4. tools/lib/bpf/relo_core.h +29 −1
kernel/bpf/btf.c+10−1
@@ -6742,8 +6742,16 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, · 6741 unmodified lines
67426742 {
67436743 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL;
67446744 struct bpf_core_cand_list cands = {};
6745+ struct bpf_core_spec *specs;
67456746 int err;
67466747
6748+ /* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
6749+ * into arrays of btf_ids of struct fields and array indices.
6750+ */
6751+ specs = kcalloc(3, sizeof(*specs), GFP_KERNEL);
6752+ if (!specs)
6753+ return -ENOMEM;
6754+
67476755 if (need_cands) {
67486756 struct bpf_cand_cache *cc;
67496757 int i;
@@ -6779,8 +6787,9 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, · 29 unmodified lines
67796787 }
67806788
67816789 err = bpf_core_apply_relo_insn((void *)ctx->log, insn, relo->insn_off / 8,
6782− relo, relo_idx, ctx->btf, &cands);
6790+ relo, relo_idx, ctx->btf, &cands, specs);
67836791 out:
6792+ kfree(specs);
67846793 if (need_cands) {
67856794 kfree(cands.cands);
67866795 mutex_unlock(&cand_cache_mutex);
tools/lib/bpf/libbpf.c+3−1
@@ -5515,6 +5515,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog, · 5514 unmodified lines
55155515 const struct btf *local_btf,
55165516 struct hashmap *cand_cache)
55175517 {
5518+ struct bpf_core_spec specs_scratch[3] = {};
55185519 const void *type_key = u32_as_hash_key(relo->type_id);
55195520 struct bpf_core_cand_list *cands = NULL;
55205521 const char *prog_name = prog->name;
@@ -5569,7 +5570,8 @@ static int bpf_core_apply_relo(struct bpf_program *prog, · 48 unmodified lines
55695570 }
55705571 }
55715572
5572− return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands);
5573+ return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo,
5574+ relo_idx, local_btf, cands, specs_scratch);
55735575 }
55745576
55755577 static int
tools/lib/bpf/relo_core.c+19−43
@@ -68,33 +68,6 @@ enum libbpf_print_level { · 67 unmodified lines
6868 #include "libbpf_internal.h"
6969 #endif
7070
71−#define BPF_CORE_SPEC_MAX_LEN 32
72−
73−/* represents BPF CO-RE field or array element accessor */
74−struct bpf_core_accessor {
75− __u32 type_id; /* struct/union type or array element type */
76− __u32 idx; /* field index or array index */
77− const char *name; /* field name or NULL for array accessor */
78−};
79−
80−struct bpf_core_spec {
81− const struct btf *btf;
82− /* high-level spec: named fields and array indices only */
83− struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
84− /* original unresolved (no skip_mods_or_typedefs) root type ID */
85− __u32 root_type_id;
86− /* CO-RE relocation kind */
87− enum bpf_core_relo_kind relo_kind;
88− /* high-level spec length */
89− int len;
90− /* raw, low-level spec: 1-to-1 with accessor spec string */
91− int raw_spec[BPF_CORE_SPEC_MAX_LEN];
92− /* raw spec length */
93− int raw_len;
94− /* field bit offset represented by spec */
95− __u32 bit_offset;
96−};
97−
9871 static bool is_flex_arr(const struct btf *btf,
9972 const struct bpf_core_accessor *acc,
10073 const struct btf_array *arr)
@@ -1200,9 +1173,12 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 1099 unmodified lines
12001173 const struct bpf_core_relo *relo,
12011174 int relo_idx,
12021175 const struct btf *local_btf,
1203− struct bpf_core_cand_list *cands)
1176+ struct bpf_core_cand_list *cands,
1177+ struct bpf_core_spec *specs_scratch)
12041178 {
1205− struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
1179+ struct bpf_core_spec *local_spec = &specs_scratch[0];
1180+ struct bpf_core_spec *cand_spec = &specs_scratch[1];
1181+ struct bpf_core_spec *targ_spec = &specs_scratch[2];
12061182 struct bpf_core_relo_res cand_res, targ_res;
12071183 const struct btf_type *local_type;
12081184 const char *local_name;
@@ -1221,7 +1197,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 12 unmodified lines
12211197 return -EINVAL;
12221198
12231199 err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str,
1224− relo->kind, &local_spec);
1200+ relo->kind, local_spec);
12251201 if (err) {
12261202 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
12271203 prog_name, relo_idx, local_id, btf_kind_str(local_type),
@@ -1232,15 +1208,15 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 4 unmodified lines
12321208
12331209 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name,
12341210 relo_idx, core_relo_kind_str(relo->kind), relo->kind);
1235− bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &local_spec);
1211+ bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, local_spec);
12361212 libbpf_print(LIBBPF_DEBUG, "\n");
12371213
12381214 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
12391215 if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
12401216 targ_res.validate = true;
12411217 targ_res.poison = false;
1242− targ_res.orig_val = local_spec.root_type_id;
1243− targ_res.new_val = local_spec.root_type_id;
1218+ targ_res.orig_val = local_spec->root_type_id;
1219+ targ_res.new_val = local_spec->root_type_id;
12441220 goto patch_insn;
12451221 }
12461222
@@ -1253,38 +1229,38 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 6 unmodified lines
12531229
12541230
12551231 for (i = 0, j = 0; i < cands->len; i++) {
1256− err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
1257− cands->cands[i].id, &cand_spec);
1232+ err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
1233+ cands->cands[i].id, cand_spec);
12581234 if (err < 0) {
12591235 pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
12601236 prog_name, relo_idx, i);
1261− bpf_core_dump_spec(prog_name, LIBBPF_WARN, &cand_spec);
1237+ bpf_core_dump_spec(prog_name, LIBBPF_WARN, cand_spec);
12621238 libbpf_print(LIBBPF_WARN, ": %d\n", err);
12631239 return err;
12641240 }
12651241
12661242 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name,
12671243 relo_idx, err == 0 ? "non-matching" : "matching", i);
1268− bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &cand_spec);
1244+ bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, cand_spec);
12691245 libbpf_print(LIBBPF_DEBUG, "\n");
12701246
12711247 if (err == 0)
12721248 continue;
12731249
1274− err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
1250+ err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res);
12751251 if (err)
12761252 return err;
12771253
12781254 if (j == 0) {
12791255 targ_res = cand_res;
1280− targ_spec = cand_spec;
1281− } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
1256+ *targ_spec = *cand_spec;
1257+ } else if (cand_spec->bit_offset != targ_spec->bit_offset) {
12821258 /* if there are many field relo candidates, they
12831259 * should all resolve to the same bit offset
12841260 */
12851261 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
1286− prog_name, relo_idx, cand_spec.bit_offset,
1287− targ_spec.bit_offset);
1262+ prog_name, relo_idx, cand_spec->bit_offset,
1263+ targ_spec->bit_offset);
12881264 return -EINVAL;
12891265 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
12901266 /* all candidates should result in the same relocation
@@ -1328,7 +1304,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 37 unmodified lines
13281304 prog_name, relo_idx);
13291305
13301306 /* calculate single target relo result explicitly */
1331− err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, NULL, &targ_res);
1307+ err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, &targ_res);
13321308 if (err)
13331309 return err;
13341310 }
tools/lib/bpf/relo_core.h+29−1
@@ -17,11 +17,39 @@ struct bpf_core_cand_list { · 16 unmodified lines
1717 int len;
1818 };
1919
20+#define BPF_CORE_SPEC_MAX_LEN 64
21+
22+/* represents BPF CO-RE field or array element accessor */
23+struct bpf_core_accessor {
24+ __u32 type_id; /* struct/union type or array element type */
25+ __u32 idx; /* field index or array index */
26+ const char *name; /* field name or NULL for array accessor */
27+};
28+
29+struct bpf_core_spec {
30+ const struct btf *btf;
31+ /* high-level spec: named fields and array indices only */
32+ struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
33+ /* original unresolved (no skip_mods_or_typedefs) root type ID */
34+ __u32 root_type_id;
35+ /* CO-RE relocation kind */
36+ enum bpf_core_relo_kind relo_kind;
37+ /* high-level spec length */
38+ int len;
39+ /* raw, low-level spec: 1-to-1 with accessor spec string */
40+ int raw_spec[BPF_CORE_SPEC_MAX_LEN];
41+ /* raw spec length */
42+ int raw_len;
43+ /* field bit offset represented by spec */
44+ __u32 bit_offset;
45+};
46+
2047 int bpf_core_apply_relo_insn(const char *prog_name,
2148 struct bpf_insn *insn, int insn_idx,
2249 const struct bpf_core_relo *relo, int relo_idx,
2350 const struct btf *local_btf,
24− struct bpf_core_cand_list *cands);
51+ struct bpf_core_cand_list *cands,
52+ struct bpf_core_spec *specs_scratch);
2553 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
2654 const struct btf *targ_btf, __u32 targ_id);
2755