History

bpf: Refactor trampoline update code

As we need to introduce a third type of attachment for trampolines, the flattened signature of arch_prepare_bpf_trampoline gets even more complicated. Refactor the prog and count argument to arch_prepare_bpf_trampoline to use bpf_tramp_progs to simplify the addition and accounting for new attachment types.

KP Singh committed 6 years ago

Commit 88fd9e51 parent cc6fa774 files · +71 −45Browse files

Unified · Split

4 files
  1. arch/x86/net/bpf_jit_comp.c +16 −15
  2. include/linux/bpf.h +11 −2
  3. kernel/bpf/bpf_struct_ops.c +9 −1
  4. kernel/bpf/trampoline.c +35 −27
arch/x86/net/bpf_jit_comp.c+16−15
@@ -1362,12 +1362,12 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, · 1361 unmodified lines
13621362 }
13631363
13641364 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
1365− struct bpf_prog **progs, int prog_cnt, int stack_size)
1365+ struct bpf_tramp_progs *tp, int stack_size)
13661366 {
13671367 u8 *prog = *pprog;
13681368 int cnt = 0, i;
13691369
1370− for (i = 0; i < prog_cnt; i++) {
1370+ for (i = 0; i < tp->nr_progs; i++) {
13711371 if (emit_call(&prog, __bpf_prog_enter, prog))
13721372 return -EINVAL;
13731373 /* remember prog start time returned by __bpf_prog_enter */
@@ -1376,17 +1376,17 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 2 unmodified lines
13761376 /* arg1: lea rdi, [rbp - stack_size] */
13771377 EMIT4(0x48, 0x8D, 0x7D, -stack_size);
13781378 /* arg2: progs[i]->insnsi for interpreter */
1379− if (!progs[i]->jited)
1379+ if (!tp->progs[i]->jited)
13801380 emit_mov_imm64(&prog, BPF_REG_2,
1381− (long) progs[i]->insnsi >> 32,
1382− (u32) (long) progs[i]->insnsi);
1381+ (long) tp->progs[i]->insnsi >> 32,
1382+ (u32) (long) tp->progs[i]->insnsi);
13831383 /* call JITed bpf program or interpreter */
1384− if (emit_call(&prog, progs[i]->bpf_func, prog))
1384+ if (emit_call(&prog, tp->progs[i]->bpf_func, prog))
13851385 return -EINVAL;
13861386
13871387 /* arg1: mov rdi, progs[i] */
1388− emit_mov_imm64(&prog, BPF_REG_1, (long) progs[i] >> 32,
1389− (u32) (long) progs[i]);
1388+ emit_mov_imm64(&prog, BPF_REG_1, (long) tp->progs[i] >> 32,
1389+ (u32) (long) tp->progs[i]);
13901390 /* arg2: mov rsi, rbx <- start time in nsec */
13911391 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
13921392 if (emit_call(&prog, __bpf_prog_exit, prog))
@@ -1458,12 +1458,13 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 65 unmodified lines
14581458 */
14591459 int arch_prepare_bpf_trampoline(void *image, void *image_end,
14601460 const struct btf_func_model *m, u32 flags,
1461− struct bpf_prog **fentry_progs, int fentry_cnt,
1462− struct bpf_prog **fexit_progs, int fexit_cnt,
1461+ struct bpf_tramp_progs *tprogs,
14631462 void *orig_call)
14641463 {
14651464 int cnt = 0, nr_args = m->nr_args;
14661465 int stack_size = nr_args * 8;
1466+ struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
1467+ struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
14671468 u8 *prog;
14681469
14691470 /* x86-64 supports up to 6 arguments. 7+ can be added in the future */
@@ -1492,12 +1493,12 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 22 unmodified lines
14921493
14931494 save_regs(m, &prog, nr_args, stack_size);
14941495
1495− if (fentry_cnt)
1496− if (invoke_bpf(m, &prog, fentry_progs, fentry_cnt, stack_size))
1496+ if (fentry->nr_progs)
1497+ if (invoke_bpf(m, &prog, fentry, stack_size))
14971498 return -EINVAL;
14981499
14991500 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1500− if (fentry_cnt)
1501+ if (fentry->nr_progs)
15011502 restore_regs(m, &prog, nr_args, stack_size);
15021503
15031504 /* call original function */
@@ -1507,8 +1508,8 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 3 unmodified lines
15071508 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
15081509 }
15091510
1510− if (fexit_cnt)
1511− if (invoke_bpf(m, &prog, fexit_progs, fexit_cnt, stack_size))
1511+ if (fexit->nr_progs)
1512+ if (invoke_bpf(m, &prog, fexit, stack_size))
15121513 return -EINVAL;
15131514
15141515 if (flags & BPF_TRAMP_F_RESTORE_REGS)
include/linux/bpf.h+11−2
@@ -433,6 +433,16 @@ struct btf_func_model { · 432 unmodified lines
433433 */
434434 #define BPF_TRAMP_F_SKIP_FRAME BIT(2)
435435
436+/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
437+ * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
438+ */
439+#define BPF_MAX_TRAMP_PROGS 40
440+
441+struct bpf_tramp_progs {
442+ struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS];
443+ int nr_progs;
444+};
445+
436446 /* Different use cases for BPF trampoline:
437447 * 1. replace nop at the function entry (kprobe equivalent)
438448 * flags = BPF_TRAMP_F_RESTORE_REGS
@@ -455,8 +465,7 @@ struct btf_func_model { · 16 unmodified lines
455465 */
456466 int arch_prepare_bpf_trampoline(void *image, void *image_end,
457467 const struct btf_func_model *m, u32 flags,
458− struct bpf_prog **fentry_progs, int fentry_cnt,
459− struct bpf_prog **fexit_progs, int fexit_cnt,
468+ struct bpf_tramp_progs *tprogs,
460469 void *orig_call);
461470 /* these two functions are called from generated trampoline */
462471 u64 notrace __bpf_prog_enter(void);
kernel/bpf/bpf_struct_ops.c+9−1
@@ -320,6 +320,7 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 319 unmodified lines
320320 struct bpf_struct_ops_value *uvalue, *kvalue;
321321 const struct btf_member *member;
322322 const struct btf_type *t = st_ops->type;
323+ struct bpf_tramp_progs *tprogs = NULL;
323324 void *udata, *kdata;
324325 int prog_fd, err = 0;
325326 void *image;
@@ -343,6 +344,10 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 17 unmodified lines
343344 if (uvalue->state || refcount_read(&uvalue->refcnt))
344345 return -EINVAL;
345346
347+ tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
348+ if (!tprogs)
349+ return -ENOMEM;
350+
346351 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
347352 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
348353
@@ -425,10 +430,12 @@ static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 76 unmodified lines
425430 goto reset_unlock;
426431 }
427432
433+ tprogs[BPF_TRAMP_FENTRY].progs[0] = prog;
434+ tprogs[BPF_TRAMP_FENTRY].nr_progs = 1;
428435 err = arch_prepare_bpf_trampoline(image,
429436 st_map->image + PAGE_SIZE,
430437 &st_ops->func_models[i], 0,
431− &prog, 1, NULL, 0, NULL);
438+ tprogs, NULL);
432439 if (err < 0)
433440 goto reset_unlock;
434441
@@ -469,6 +476,7 @@ reset_unlock: · 34 unmodified lines
469476 memset(uvalue, 0, map->value_size);
470477 memset(kvalue, 0, map->value_size);
471478 unlock:
479+ kfree(tprogs);
472480 mutex_unlock(&st_map->lock);
473481 return err;
474482 }
kernel/bpf/trampoline.c+35−27
@@ -190,40 +190,49 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) · 189 unmodified lines
190190 return ret;
191191 }
192192
193−/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
194− * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2
195− */
196−#define BPF_MAX_TRAMP_PROGS 40
193+static struct bpf_tramp_progs *
194+bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total)
195+{
196+ const struct bpf_prog_aux *aux;
197+ struct bpf_tramp_progs *tprogs;
198+ struct bpf_prog **progs;
199+ int kind;
200+
201+ *total = 0;
202+ tprogs = kcalloc(BPF_TRAMP_MAX, sizeof(*tprogs), GFP_KERNEL);
203+ if (!tprogs)
204+ return ERR_PTR(-ENOMEM);
205+
206+ for (kind = 0; kind < BPF_TRAMP_MAX; kind++) {
207+ tprogs[kind].nr_progs = tr->progs_cnt[kind];
208+ *total += tr->progs_cnt[kind];
209+ progs = tprogs[kind].progs;
210+
211+ hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist)
212+ *progs++ = aux->prog;
213+ }
214+ return tprogs;
215+}
197216
198217 static int bpf_trampoline_update(struct bpf_trampoline *tr)
199218 {
200219 void *old_image = tr->image + ((tr->selector + 1) & 1) * BPF_IMAGE_SIZE/2;
201220 void *new_image = tr->image + (tr->selector & 1) * BPF_IMAGE_SIZE/2;
202− struct bpf_prog *progs_to_run[BPF_MAX_TRAMP_PROGS];
203− int fentry_cnt = tr->progs_cnt[BPF_TRAMP_FENTRY];
204− int fexit_cnt = tr->progs_cnt[BPF_TRAMP_FEXIT];
205− struct bpf_prog **progs, **fentry, **fexit;
221+ struct bpf_tramp_progs *tprogs;
206222 u32 flags = BPF_TRAMP_F_RESTORE_REGS;
207− struct bpf_prog_aux *aux;
208− int err;
223+ int err, total;
209224
210− if (fentry_cnt + fexit_cnt == 0) {
225+ tprogs = bpf_trampoline_get_progs(tr, &total);
226+ if (IS_ERR(tprogs))
227+ return PTR_ERR(tprogs);
228+
229+ if (total == 0) {
211230 err = unregister_fentry(tr, old_image);
212231 tr->selector = 0;
213232 goto out;
214233 }
215234
216− /* populate fentry progs */
217− fentry = progs = progs_to_run;
218− hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FENTRY], tramp_hlist)
219− *progs++ = aux->prog;
220−
221− /* populate fexit progs */
222− fexit = progs;
223− hlist_for_each_entry(aux, &tr->progs_hlist[BPF_TRAMP_FEXIT], tramp_hlist)
224− *progs++ = aux->prog;
225−
226− if (fexit_cnt)
235+ if (tprogs[BPF_TRAMP_FEXIT].nr_progs)
227236 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
228237
229238 /* Though the second half of trampoline page is unused a task could be
@@ -232,12 +241,11 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 2 unmodified lines
232241 * preempted task. Hence wait for tasks to voluntarily schedule or go
233242 * to userspace.
234243 */
244+
235245 synchronize_rcu_tasks();
236246
237247 err = arch_prepare_bpf_trampoline(new_image, new_image + BPF_IMAGE_SIZE / 2,
238− &tr->func.model, flags,
239− fentry, fentry_cnt,
240− fexit, fexit_cnt,
248+ &tr->func.model, flags, tprogs,
241249 tr->func.addr);
242250 if (err < 0)
243251 goto out;
@@ -252,6 +260,7 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 8 unmodified lines
252260 goto out;
253261 tr->selector++;
254262 out:
263+ kfree(tprogs);
255264 return err;
256265 }
257266
@@ -409,8 +418,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 151 unmodified lines
409418 int __weak
410419 arch_prepare_bpf_trampoline(void *image, void *image_end,
411420 const struct btf_func_model *m, u32 flags,
412− struct bpf_prog **fentry_progs, int fentry_cnt,
413− struct bpf_prog **fexit_progs, int fexit_cnt,
421+ struct bpf_tramp_progs *tprogs,
414422 void *orig_call)
415423 {
416424 return -ENOTSUPP;