History

bpf: Introduce BPF_MODIFY_RETURN

When multiple programs are attached, each program receives the return value from the previous program on the stack and the last program provides the return value to the attached function. The fmod_ret bpf programs are run after the fentry programs and before the fexit programs. The original function is only called if all the fmod_ret programs return 0 to avoid any unintended side-effects. The success value, i.e. 0 is not currently configurable but can be made so where user-space can specify it at load time. For example: int func_to_be_attached(int a, int b) { <--- do_fentry do_fmod_ret: <update ret by calling fmod_ret> if (ret != 0) goto do_fexit; original_function: <side_effects_happen_here> } <--- do_fexit The fmod_ret program attached to this function can be defined as: SEC("fmod_ret/func_to_be_attached") int BPF_PROG(func_name, int a, int b, int ret) { // This will skip the original function logic. return 1; } The first fmod_ret program is passed 0 in its return argument.

KP Singh committed 6 years ago

Commit ae240821 parent 7e639208 files · +130 −13Browse files

Unified · Split

8 files
  1. arch/x86/net/bpf_jit_comp.c +119 −11
  2. include/linux/bpf.h +1 −0
  3. include/uapi/linux/bpf.h +1 −0
  4. kernel/bpf/btf.c +2 −1
  5. kernel/bpf/syscall.c +1 −0
  6. kernel/bpf/trampoline.c +4 −1
  7. kernel/bpf/verifier.c +1 −0
  8. tools/include/uapi/linux/bpf.h +1 −0
arch/x86/net/bpf_jit_comp.c+119−11
@@ -1362,7 +1362,7 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, · 1361 unmodified lines
13621362 }
13631363
13641364 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
1365− struct bpf_prog *p, int stack_size)
1365+ struct bpf_prog *p, int stack_size, bool mod_ret)
13661366 {
13671367 u8 *prog = *pprog;
13681368 int cnt = 0;
@@ -1383,6 +1383,13 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 14 unmodified lines
13831383 if (emit_call(&prog, p->bpf_func, prog))
13841384 return -EINVAL;
13851385
1386+ /* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
1387+ * of the previous call which is then passed on the stack to
1388+ * the next BPF program.
1389+ */
1390+ if (mod_ret)
1391+ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1392+
13861393 /* arg1: mov rdi, progs[i] */
13871394 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,
13881395 (u32) (long) p);
@@ -1442,6 +1449,23 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) · 53 unmodified lines
14421449 return 0;
14431450 }
14441451
1452+static int emit_mod_ret_check_imm8(u8 **pprog, int value)
1453+{
1454+ u8 *prog = *pprog;
1455+ int cnt = 0;
1456+
1457+ if (!is_imm8(value))
1458+ return -EINVAL;
1459+
1460+ if (value == 0)
1461+ EMIT2(0x85, add_2reg(0xC0, BPF_REG_0, BPF_REG_0));
1462+ else
1463+ EMIT3(0x83, add_1reg(0xF8, BPF_REG_0), value);
1464+
1465+ *pprog = prog;
1466+ return 0;
1467+}
1468+
14451469 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
14461470 struct bpf_tramp_progs *tp, int stack_size)
14471471 {
@@ -1449,9 +1473,49 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 1 unmodified line
14491473 u8 *prog = *pprog;
14501474
14511475 for (i = 0; i < tp->nr_progs; i++) {
1452− if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size))
1476+ if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false))
1477+ return -EINVAL;
1478+ }
1479+ *pprog = prog;
1480+ return 0;
1481+}
1482+
1483+static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
1484+ struct bpf_tramp_progs *tp, int stack_size,
1485+ u8 **branches)
1486+{
1487+ u8 *prog = *pprog;
1488+ int i;
1489+
1490+ /* The first fmod_ret program will receive a garbage return value.
1491+ * Set this to 0 to avoid confusing the program.
1492+ */
1493+ emit_mov_imm32(&prog, false, BPF_REG_0, 0);
1494+ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1495+ for (i = 0; i < tp->nr_progs; i++) {
1496+ if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))
14531497 return -EINVAL;
1498+
1499+ /* Generate a branch:
1500+ *
1501+ * if (ret != 0)
1502+ * goto do_fexit;
1503+ *
1504+ * If needed this can be extended to any integer value which can
1505+ * be passed by user-space when the program is loaded.
1506+ */
1507+ if (emit_mod_ret_check_imm8(&prog, 0))
1508+ return -EINVAL;
1509+
1510+ /* Save the location of the branch and Generate 6 nops
1511+ * (4 bytes for an offset and 2 bytes for the jump) These nops
1512+ * are replaced with a conditional jump once do_fexit (i.e. the
1513+ * start of the fexit invocation) is finalized.
1514+ */
1515+ branches[i] = prog;
1516+ emit_nops(&prog, 4 + 2);
14541517 }
1518+
14551519 *pprog = prog;
14561520 return 0;
14571521 }
@@ -1521,10 +1585,12 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 63 unmodified lines
15211585 struct bpf_tramp_progs *tprogs,
15221586 void *orig_call)
15231587 {
1524− int cnt = 0, nr_args = m->nr_args;
1588+ int ret, i, cnt = 0, nr_args = m->nr_args;
15251589 int stack_size = nr_args * 8;
15261590 struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
15271591 struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
1592+ struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
1593+ u8 **branches = NULL;
15281594 u8 *prog;
15291595
15301596 /* x86-64 supports up to 6 arguments. 7+ can be added in the future */
@@ -1557,24 +1623,60 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 26 unmodified lines
15571623 if (invoke_bpf(m, &prog, fentry, stack_size))
15581624 return -EINVAL;
15591625
1626+ if (fmod_ret->nr_progs) {
1627+ branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),
1628+ GFP_KERNEL);
1629+ if (!branches)
1630+ return -ENOMEM;
1631+
1632+ if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size,
1633+ branches)) {
1634+ ret = -EINVAL;
1635+ goto cleanup;
1636+ }
1637+ }
1638+
15601639 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1561− if (fentry->nr_progs)
1640+ if (fentry->nr_progs || fmod_ret->nr_progs)
15621641 restore_regs(m, &prog, nr_args, stack_size);
15631642
15641643 /* call original function */
1565− if (emit_call(&prog, orig_call, prog))
1566− return -EINVAL;
1644+ if (emit_call(&prog, orig_call, prog)) {
1645+ ret = -EINVAL;
1646+ goto cleanup;
1647+ }
15671648 /* remember return value in a stack for bpf prog to access */
15681649 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
15691650 }
15701651
1652+ if (fmod_ret->nr_progs) {
1653+ /* From Intel 64 and IA-32 Architectures Optimization
1654+ * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
1655+ * Coding Rule 11: All branch targets should be 16-byte
1656+ * aligned.
1657+ */
1658+ emit_align(&prog, 16);
1659+ /* Update the branches saved in invoke_bpf_mod_ret with the
1660+ * aligned address of do_fexit.
1661+ */
1662+ for (i = 0; i < fmod_ret->nr_progs; i++)
1663+ emit_cond_near_jump(&branches[i], prog, branches[i],
1664+ X86_JNE);
1665+ }
1666+
15711667 if (fexit->nr_progs)
1572− if (invoke_bpf(m, &prog, fexit, stack_size))
1573− return -EINVAL;
1668+ if (invoke_bpf(m, &prog, fexit, stack_size)) {
1669+ ret = -EINVAL;
1670+ goto cleanup;
1671+ }
15741672
15751673 if (flags & BPF_TRAMP_F_RESTORE_REGS)
15761674 restore_regs(m, &prog, nr_args, stack_size);
15771675
1676+ /* This needs to be done regardless. If there were fmod_ret programs,
1677+ * the return value is only updated on the stack and still needs to be
1678+ * restored to R0.
1679+ */
15781680 if (flags & BPF_TRAMP_F_CALL_ORIG)
15791681 /* restore original return value back into RAX */
15801682 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
@@ -1586,9 +1688,15 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 5 unmodified lines
15861688 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
15871689 EMIT1(0xC3); /* ret */
15881690 /* Make sure the trampoline generation logic doesn't overflow */
1589− if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY))
1590− return -EFAULT;
1591− return prog - (u8 *)image;
1691+ if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) {
1692+ ret = -EFAULT;
1693+ goto cleanup;
1694+ }
1695+ ret = prog - (u8 *)image;
1696+
1697+cleanup:
1698+ kfree(branches);
1699+ return ret;
15921700 }
15931701
15941702 static int emit_fallback_jump(u8 **pprog)
include/linux/bpf.h+1−0
@@ -474,6 +474,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); · 473 unmodified lines
474474 enum bpf_tramp_prog_type {
475475 BPF_TRAMP_FENTRY,
476476 BPF_TRAMP_FEXIT,
477+ BPF_TRAMP_MODIFY_RETURN,
477478 BPF_TRAMP_MAX,
478479 BPF_TRAMP_REPLACE, /* more than MAX */
479480 };
include/uapi/linux/bpf.h+1−0
@@ -210,6 +210,7 @@ enum bpf_attach_type { · 209 unmodified lines
210210 BPF_TRACE_RAW_TP,
211211 BPF_TRACE_FENTRY,
212212 BPF_TRACE_FEXIT,
213+ BPF_MODIFY_RETURN,
213214 __MAX_BPF_ATTACH_TYPE
214215 };
215216
kernel/bpf/btf.c+2−1
@@ -3710,7 +3710,8 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 3709 unmodified lines
37103710 nr_args--;
37113711 }
37123712
3713− if (prog->expected_attach_type == BPF_TRACE_FEXIT &&
3713+ if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
3714+ prog->expected_attach_type == BPF_MODIFY_RETURN) &&
37143715 arg == nr_args) {
37153716 if (!t)
37163717 /* Default prog with 5 args. 6th arg is retval. */
kernel/bpf/syscall.c+1−0
@@ -2324,6 +2324,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) · 2323 unmodified lines
23242324
23252325 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
23262326 prog->expected_attach_type != BPF_TRACE_FEXIT &&
2327+ prog->expected_attach_type != BPF_MODIFY_RETURN &&
23272328 prog->type != BPF_PROG_TYPE_EXT) {
23282329 err = -EINVAL;
23292330 goto out_put_prog;
kernel/bpf/trampoline.c+4−1
@@ -232,7 +232,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 231 unmodified lines
232232 goto out;
233233 }
234234
235− if (tprogs[BPF_TRAMP_FEXIT].nr_progs)
235+ if (tprogs[BPF_TRAMP_FEXIT].nr_progs ||
236+ tprogs[BPF_TRAMP_MODIFY_RETURN].nr_progs)
236237 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
237238
238239 /* Though the second half of trampoline page is unused a task could be
@@ -269,6 +270,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t) · 30 unmodified lines
269270 switch (t) {
270271 case BPF_TRACE_FENTRY:
271272 return BPF_TRAMP_FENTRY;
273+ case BPF_MODIFY_RETURN:
274+ return BPF_TRAMP_MODIFY_RETURN;
272275 case BPF_TRACE_FEXIT:
273276 return BPF_TRAMP_FEXIT;
274277 default:
kernel/bpf/verifier.c+1−0
@@ -9950,6 +9950,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 9949 unmodified lines
99509950 if (!prog_extension)
99519951 return -EINVAL;
99529952 /* fallthrough */
9953+ case BPF_MODIFY_RETURN:
99539954 case BPF_TRACE_FENTRY:
99549955 case BPF_TRACE_FEXIT:
99559956 if (!btf_type_is_func(t)) {
tools/include/uapi/linux/bpf.h+1−0
@@ -210,6 +210,7 @@ enum bpf_attach_type { · 209 unmodified lines
210210 BPF_TRACE_RAW_TP,
211211 BPF_TRACE_FENTRY,
212212 BPF_TRACE_FEXIT,
213+ BPF_MODIFY_RETURN,
213214 __MAX_BPF_ATTACH_TYPE
214215 };
215216

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t