History

libbpf: Reduce bpf_core_apply_relo_insn() stack usage.

Reduce bpf_core_apply_relo_insn() stack usage and bump BPF_CORE_SPEC_MAX_LEN limit back to 64.

Alexei Starovoitov committed 4 years ago

Commit 78c1f8d1 parent 0bf40544 files · +61 −46Browse files

Unified · Split

4 files
  1. kernel/bpf/btf.c +10 −1
  2. tools/lib/bpf/libbpf.c +3 −1
  3. tools/lib/bpf/relo_core.c +19 −43
  4. tools/lib/bpf/relo_core.h +29 −1
kernel/bpf/btf.c+10−1
@@ -6742,8 +6742,16 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, · 6741 unmodified lines
6742{6742{
6743 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL;6743 bool need_cands = relo->kind != BPF_CORE_TYPE_ID_LOCAL;
6744 struct bpf_core_cand_list cands = {};6744 struct bpf_core_cand_list cands = {};
6745 struct bpf_core_spec *specs;
6745 int err;6746 int err;
6746 6747
6748 /* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
6749 * into arrays of btf_ids of struct fields and array indices.
6750 */
6751 specs = kcalloc(3, sizeof(*specs), GFP_KERNEL);
6752 if (!specs)
6753 return -ENOMEM;
6754
6747 if (need_cands) {6755 if (need_cands) {
6748 struct bpf_cand_cache *cc;6756 struct bpf_cand_cache *cc;
6749 int i;6757 int i;
@@ -6779,8 +6787,9 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, · 29 unmodified lines
6779 }6787 }
6780 6788
6781 err = bpf_core_apply_relo_insn((void *)ctx->log, insn, relo->insn_off / 8,6789 err = bpf_core_apply_relo_insn((void *)ctx->log, insn, relo->insn_off / 8,
6782 relo, relo_idx, ctx->btf, &cands);6790 relo, relo_idx, ctx->btf, &cands, specs);
6783out:6791out:
6792 kfree(specs);
6784 if (need_cands) {6793 if (need_cands) {
6785 kfree(cands.cands);6794 kfree(cands.cands);
6786 mutex_unlock(&cand_cache_mutex);6795 mutex_unlock(&cand_cache_mutex);
tools/lib/bpf/libbpf.c+3−1
@@ -5515,6 +5515,7 @@ static int bpf_core_apply_relo(struct bpf_program *prog, · 5514 unmodified lines
5515 const struct btf *local_btf,5515 const struct btf *local_btf,
5516 struct hashmap *cand_cache)5516 struct hashmap *cand_cache)
5517{5517{
5518 struct bpf_core_spec specs_scratch[3] = {};
5518 const void *type_key = u32_as_hash_key(relo->type_id);5519 const void *type_key = u32_as_hash_key(relo->type_id);
5519 struct bpf_core_cand_list *cands = NULL;5520 struct bpf_core_cand_list *cands = NULL;
5520 const char *prog_name = prog->name;5521 const char *prog_name = prog->name;
@@ -5569,7 +5570,8 @@ static int bpf_core_apply_relo(struct bpf_program *prog, · 48 unmodified lines
5569 }5570 }
5570 }5571 }
5571 5572
5572 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands);5573 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo,
5574 relo_idx, local_btf, cands, specs_scratch);
5573}5575}
5574 5576
5575static int5577static int
tools/lib/bpf/relo_core.c+19−43
@@ -68,33 +68,6 @@ enum libbpf_print_level { · 67 unmodified lines
68#include "libbpf_internal.h"68#include "libbpf_internal.h"
69#endif69#endif
70 70
71#define BPF_CORE_SPEC_MAX_LEN 32
72
73/* represents BPF CO-RE field or array element accessor */
74struct bpf_core_accessor {
75 __u32 type_id; /* struct/union type or array element type */
76 __u32 idx; /* field index or array index */
77 const char *name; /* field name or NULL for array accessor */
78};
79
80struct bpf_core_spec {
81 const struct btf *btf;
82 /* high-level spec: named fields and array indices only */
83 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
84 /* original unresolved (no skip_mods_or_typedefs) root type ID */
85 __u32 root_type_id;
86 /* CO-RE relocation kind */
87 enum bpf_core_relo_kind relo_kind;
88 /* high-level spec length */
89 int len;
90 /* raw, low-level spec: 1-to-1 with accessor spec string */
91 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
92 /* raw spec length */
93 int raw_len;
94 /* field bit offset represented by spec */
95 __u32 bit_offset;
96};
97
98static bool is_flex_arr(const struct btf *btf,71static bool is_flex_arr(const struct btf *btf,
99 const struct bpf_core_accessor *acc,72 const struct bpf_core_accessor *acc,
100 const struct btf_array *arr)73 const struct btf_array *arr)
@@ -1200,9 +1173,12 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 1099 unmodified lines
1200 const struct bpf_core_relo *relo,1173 const struct bpf_core_relo *relo,
1201 int relo_idx,1174 int relo_idx,
1202 const struct btf *local_btf,1175 const struct btf *local_btf,
1203 struct bpf_core_cand_list *cands)1176 struct bpf_core_cand_list *cands,
1177 struct bpf_core_spec *specs_scratch)
1204{1178{
1205 struct bpf_core_spec local_spec, cand_spec, targ_spec = {};1179 struct bpf_core_spec *local_spec = &specs_scratch[0];
1180 struct bpf_core_spec *cand_spec = &specs_scratch[1];
1181 struct bpf_core_spec *targ_spec = &specs_scratch[2];
1206 struct bpf_core_relo_res cand_res, targ_res;1182 struct bpf_core_relo_res cand_res, targ_res;
1207 const struct btf_type *local_type;1183 const struct btf_type *local_type;
1208 const char *local_name;1184 const char *local_name;
@@ -1221,7 +1197,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 12 unmodified lines
1221 return -EINVAL;1197 return -EINVAL;
1222 1198
1223 err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str,1199 err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str,
1224 relo->kind, &local_spec);1200 relo->kind, local_spec);
1225 if (err) {1201 if (err) {
1226 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",1202 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
1227 prog_name, relo_idx, local_id, btf_kind_str(local_type),1203 prog_name, relo_idx, local_id, btf_kind_str(local_type),
@@ -1232,15 +1208,15 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 4 unmodified lines
1232 1208
1233 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name,1209 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name,
1234 relo_idx, core_relo_kind_str(relo->kind), relo->kind);1210 relo_idx, core_relo_kind_str(relo->kind), relo->kind);
1235 bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &local_spec);1211 bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, local_spec);
1236 libbpf_print(LIBBPF_DEBUG, "\n");1212 libbpf_print(LIBBPF_DEBUG, "\n");
1237 1213
1238 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */1214 /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
1239 if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {1215 if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
1240 targ_res.validate = true;1216 targ_res.validate = true;
1241 targ_res.poison = false;1217 targ_res.poison = false;
1242 targ_res.orig_val = local_spec.root_type_id;1218 targ_res.orig_val = local_spec->root_type_id;
1243 targ_res.new_val = local_spec.root_type_id;1219 targ_res.new_val = local_spec->root_type_id;
1244 goto patch_insn;1220 goto patch_insn;
1245 }1221 }
1246 1222
@@ -1253,38 +1229,38 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 6 unmodified lines
1253 1229
1254 1230
1255 for (i = 0, j = 0; i < cands->len; i++) {1231 for (i = 0, j = 0; i < cands->len; i++) {
1256 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,1232 err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
1257 cands->cands[i].id, &cand_spec);1233 cands->cands[i].id, cand_spec);
1258 if (err < 0) {1234 if (err < 0) {
1259 pr_warn("prog '%s': relo #%d: error matching candidate #%d ",1235 pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
1260 prog_name, relo_idx, i);1236 prog_name, relo_idx, i);
1261 bpf_core_dump_spec(prog_name, LIBBPF_WARN, &cand_spec);1237 bpf_core_dump_spec(prog_name, LIBBPF_WARN, cand_spec);
1262 libbpf_print(LIBBPF_WARN, ": %d\n", err);1238 libbpf_print(LIBBPF_WARN, ": %d\n", err);
1263 return err;1239 return err;
1264 }1240 }
1265 1241
1266 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name,1242 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name,
1267 relo_idx, err == 0 ? "non-matching" : "matching", i);1243 relo_idx, err == 0 ? "non-matching" : "matching", i);
1268 bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, &cand_spec);1244 bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, cand_spec);
1269 libbpf_print(LIBBPF_DEBUG, "\n");1245 libbpf_print(LIBBPF_DEBUG, "\n");
1270 1246
1271 if (err == 0)1247 if (err == 0)
1272 continue;1248 continue;
1273 1249
1274 err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, &cand_spec, &cand_res);1250 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, cand_spec, &cand_res);
1275 if (err)1251 if (err)
1276 return err;1252 return err;
1277 1253
1278 if (j == 0) {1254 if (j == 0) {
1279 targ_res = cand_res;1255 targ_res = cand_res;
1280 targ_spec = cand_spec;1256 *targ_spec = *cand_spec;
1281 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {1257 } else if (cand_spec->bit_offset != targ_spec->bit_offset) {
1282 /* if there are many field relo candidates, they1258 /* if there are many field relo candidates, they
1283 * should all resolve to the same bit offset1259 * should all resolve to the same bit offset
1284 */1260 */
1285 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",1261 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
1286 prog_name, relo_idx, cand_spec.bit_offset,1262 prog_name, relo_idx, cand_spec->bit_offset,
1287 targ_spec.bit_offset);1263 targ_spec->bit_offset);
1288 return -EINVAL;1264 return -EINVAL;
1289 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {1265 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
1290 /* all candidates should result in the same relocation1266 /* all candidates should result in the same relocation
@@ -1328,7 +1304,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn, · 37 unmodified lines
1328 prog_name, relo_idx);1304 prog_name, relo_idx);
1329 1305
1330 /* calculate single target relo result explicitly */1306 /* calculate single target relo result explicitly */
1331 err = bpf_core_calc_relo(prog_name, relo, relo_idx, &local_spec, NULL, &targ_res);1307 err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, &targ_res);
1332 if (err)1308 if (err)
1333 return err;1309 return err;
1334 }1310 }
tools/lib/bpf/relo_core.h+29−1
@@ -17,11 +17,39 @@ struct bpf_core_cand_list { · 16 unmodified lines
17 int len;17 int len;
18};18};
19 19
20#define BPF_CORE_SPEC_MAX_LEN 64
21
22/* represents BPF CO-RE field or array element accessor */
23struct bpf_core_accessor {
24 __u32 type_id; /* struct/union type or array element type */
25 __u32 idx; /* field index or array index */
26 const char *name; /* field name or NULL for array accessor */
27};
28
29struct bpf_core_spec {
30 const struct btf *btf;
31 /* high-level spec: named fields and array indices only */
32 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
33 /* original unresolved (no skip_mods_or_typedefs) root type ID */
34 __u32 root_type_id;
35 /* CO-RE relocation kind */
36 enum bpf_core_relo_kind relo_kind;
37 /* high-level spec length */
38 int len;
39 /* raw, low-level spec: 1-to-1 with accessor spec string */
40 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
41 /* raw spec length */
42 int raw_len;
43 /* field bit offset represented by spec */
44 __u32 bit_offset;
45};
46
20int bpf_core_apply_relo_insn(const char *prog_name,47int bpf_core_apply_relo_insn(const char *prog_name,
21 struct bpf_insn *insn, int insn_idx,48 struct bpf_insn *insn, int insn_idx,
22 const struct bpf_core_relo *relo, int relo_idx,49 const struct bpf_core_relo *relo, int relo_idx,
23 const struct btf *local_btf,50 const struct btf *local_btf,
24 struct bpf_core_cand_list *cands);51 struct bpf_core_cand_list *cands,
52 struct bpf_core_spec *specs_scratch);
25int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,53int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
26 const struct btf *targ_btf, __u32 targ_id);54 const struct btf *targ_btf, __u32 targ_id);
27 55