History

bpf: Introduce BPF_MODIFY_RETURN

When multiple programs are attached, each program receives the return value from the previous program on the stack and the last program provides the return value to the attached function. The fmod_ret bpf programs are run after the fentry programs and before the fexit programs. The original function is only called if all the fmod_ret programs return 0 to avoid any unintended side-effects. The success value, i.e. 0 is not currently configurable but can be made so where user-space can specify it at load time. For example: int func_to_be_attached(int a, int b) { <--- do_fentry do_fmod_ret: <update ret by calling fmod_ret> if (ret != 0) goto do_fexit; original_function: <side_effects_happen_here> } <--- do_fexit The fmod_ret program attached to this function can be defined as: SEC("fmod_ret/func_to_be_attached") int BPF_PROG(func_name, int a, int b, int ret) { // This will skip the original function logic. return 1; } The first fmod_ret program is passed 0 in its return argument.

KP Singh committed 6 years ago

Commit ae240821 parent 7e639208 files · +130 −13Browse files

Unified · Split

8 files
  1. arch/x86/net/bpf_jit_comp.c +119 −11
  2. include/linux/bpf.h +1 −0
  3. include/uapi/linux/bpf.h +1 −0
  4. kernel/bpf/btf.c +2 −1
  5. kernel/bpf/syscall.c +1 −0
  6. kernel/bpf/trampoline.c +4 −1
  7. kernel/bpf/verifier.c +1 −0
  8. tools/include/uapi/linux/bpf.h +1 −0
arch/x86/net/bpf_jit_comp.c+119−11
@@ -1362,7 +1362,7 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, · 1361 unmodified lines
1362}1362}
1363 1363
1364static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,1364static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
1365 struct bpf_prog *p, int stack_size)1365 struct bpf_prog *p, int stack_size, bool mod_ret)
1366{1366{
1367 u8 *prog = *pprog;1367 u8 *prog = *pprog;
1368 int cnt = 0;1368 int cnt = 0;
@@ -1383,6 +1383,13 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 14 unmodified lines
1383 if (emit_call(&prog, p->bpf_func, prog))1383 if (emit_call(&prog, p->bpf_func, prog))
1384 return -EINVAL;1384 return -EINVAL;
1385 1385
1386 /* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
1387 * of the previous call which is then passed on the stack to
1388 * the next BPF program.
1389 */
1390 if (mod_ret)
1391 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1392
1386 /* arg1: mov rdi, progs[i] */1393 /* arg1: mov rdi, progs[i] */
1387 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,1394 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,
1388 (u32) (long) p);1395 (u32) (long) p);
@@ -1442,6 +1449,23 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) · 53 unmodified lines
1442 return 0;1449 return 0;
1443}1450}
1444 1451
1452static int emit_mod_ret_check_imm8(u8 **pprog, int value)
1453{
1454 u8 *prog = *pprog;
1455 int cnt = 0;
1456
1457 if (!is_imm8(value))
1458 return -EINVAL;
1459
1460 if (value == 0)
1461 EMIT2(0x85, add_2reg(0xC0, BPF_REG_0, BPF_REG_0));
1462 else
1463 EMIT3(0x83, add_1reg(0xF8, BPF_REG_0), value);
1464
1465 *pprog = prog;
1466 return 0;
1467}
1468
1445static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,1469static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
1446 struct bpf_tramp_progs *tp, int stack_size)1470 struct bpf_tramp_progs *tp, int stack_size)
1447{1471{
@@ -1449,9 +1473,49 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 1 unmodified line
1449 u8 *prog = *pprog;1473 u8 *prog = *pprog;
1450 1474
1451 for (i = 0; i < tp->nr_progs; i++) {1475 for (i = 0; i < tp->nr_progs; i++) {
1452 if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size))1476 if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false))
1477 return -EINVAL;
1478 }
1479 *pprog = prog;
1480 return 0;
1481}
1482
1483static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
1484 struct bpf_tramp_progs *tp, int stack_size,
1485 u8 **branches)
1486{
1487 u8 *prog = *pprog;
1488 int i;
1489
1490 /* The first fmod_ret program will receive a garbage return value.
1491 * Set this to 0 to avoid confusing the program.
1492 */
1493 emit_mov_imm32(&prog, false, BPF_REG_0, 0);
1494 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1495 for (i = 0; i < tp->nr_progs; i++) {
1496 if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true))
1453 return -EINVAL;1497 return -EINVAL;
1498
1499 /* Generate a branch:
1500 *
1501 * if (ret != 0)
1502 * goto do_fexit;
1503 *
1504 * If needed this can be extended to any integer value which can
1505 * be passed by user-space when the program is loaded.
1506 */
1507 if (emit_mod_ret_check_imm8(&prog, 0))
1508 return -EINVAL;
1509
1510 /* Save the location of the branch and Generate 6 nops
1511 * (4 bytes for an offset and 2 bytes for the jump) These nops
1512 * are replaced with a conditional jump once do_fexit (i.e. the
1513 * start of the fexit invocation) is finalized.
1514 */
1515 branches[i] = prog;
1516 emit_nops(&prog, 4 + 2);
1454 }1517 }
1518
1455 *pprog = prog;1519 *pprog = prog;
1456 return 0;1520 return 0;
1457}1521}
@@ -1521,10 +1585,12 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 63 unmodified lines
1521 struct bpf_tramp_progs *tprogs,1585 struct bpf_tramp_progs *tprogs,
1522 void *orig_call)1586 void *orig_call)
1523{1587{
1524 int cnt = 0, nr_args = m->nr_args;1588 int ret, i, cnt = 0, nr_args = m->nr_args;
1525 int stack_size = nr_args * 8;1589 int stack_size = nr_args * 8;
1526 struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];1590 struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY];
1527 struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];1591 struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT];
1592 struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN];
1593 u8 **branches = NULL;
1528 u8 *prog;1594 u8 *prog;
1529 1595
1530 /* x86-64 supports up to 6 arguments. 7+ can be added in the future */1596 /* x86-64 supports up to 6 arguments. 7+ can be added in the future */
@@ -1557,24 +1623,60 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 26 unmodified lines
1557 if (invoke_bpf(m, &prog, fentry, stack_size))1623 if (invoke_bpf(m, &prog, fentry, stack_size))
1558 return -EINVAL;1624 return -EINVAL;
1559 1625
1626 if (fmod_ret->nr_progs) {
1627 branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *),
1628 GFP_KERNEL);
1629 if (!branches)
1630 return -ENOMEM;
1631
1632 if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size,
1633 branches)) {
1634 ret = -EINVAL;
1635 goto cleanup;
1636 }
1637 }
1638
1560 if (flags & BPF_TRAMP_F_CALL_ORIG) {1639 if (flags & BPF_TRAMP_F_CALL_ORIG) {
1561 if (fentry->nr_progs)1640 if (fentry->nr_progs || fmod_ret->nr_progs)
1562 restore_regs(m, &prog, nr_args, stack_size);1641 restore_regs(m, &prog, nr_args, stack_size);
1563 1642
1564 /* call original function */1643 /* call original function */
1565 if (emit_call(&prog, orig_call, prog))1644 if (emit_call(&prog, orig_call, prog)) {
1566 return -EINVAL;1645 ret = -EINVAL;
1646 goto cleanup;
1647 }
1567 /* remember return value in a stack for bpf prog to access */1648 /* remember return value in a stack for bpf prog to access */
1568 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);1649 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
1569 }1650 }
1570 1651
1652 if (fmod_ret->nr_progs) {
1653 /* From Intel 64 and IA-32 Architectures Optimization
1654 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
1655 * Coding Rule 11: All branch targets should be 16-byte
1656 * aligned.
1657 */
1658 emit_align(&prog, 16);
1659 /* Update the branches saved in invoke_bpf_mod_ret with the
1660 * aligned address of do_fexit.
1661 */
1662 for (i = 0; i < fmod_ret->nr_progs; i++)
1663 emit_cond_near_jump(&branches[i], prog, branches[i],
1664 X86_JNE);
1665 }
1666
1571 if (fexit->nr_progs)1667 if (fexit->nr_progs)
1572 if (invoke_bpf(m, &prog, fexit, stack_size))1668 if (invoke_bpf(m, &prog, fexit, stack_size)) {
1573 return -EINVAL;1669 ret = -EINVAL;
1670 goto cleanup;
1671 }
1574 1672
1575 if (flags & BPF_TRAMP_F_RESTORE_REGS)1673 if (flags & BPF_TRAMP_F_RESTORE_REGS)
1576 restore_regs(m, &prog, nr_args, stack_size);1674 restore_regs(m, &prog, nr_args, stack_size);
1577 1675
1676 /* This needs to be done regardless. If there were fmod_ret programs,
1677 * the return value is only updated on the stack and still needs to be
1678 * restored to R0.
1679 */
1578 if (flags & BPF_TRAMP_F_CALL_ORIG)1680 if (flags & BPF_TRAMP_F_CALL_ORIG)
1579 /* restore original return value back into RAX */1681 /* restore original return value back into RAX */
1580 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);1682 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
@@ -1586,9 +1688,15 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 5 unmodified lines
1586 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */1688 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
1587 EMIT1(0xC3); /* ret */1689 EMIT1(0xC3); /* ret */
1588 /* Make sure the trampoline generation logic doesn't overflow */1690 /* Make sure the trampoline generation logic doesn't overflow */
1589 if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY))1691 if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) {
1590 return -EFAULT;1692 ret = -EFAULT;
1591 return prog - (u8 *)image;1693 goto cleanup;
1694 }
1695 ret = prog - (u8 *)image;
1696
1697cleanup:
1698 kfree(branches);
1699 return ret;
1592}1700}
1593 1701
1594static int emit_fallback_jump(u8 **pprog)1702static int emit_fallback_jump(u8 **pprog)
include/linux/bpf.h+1−0
@@ -474,6 +474,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); · 473 unmodified lines
474enum bpf_tramp_prog_type {474enum bpf_tramp_prog_type {
475 BPF_TRAMP_FENTRY,475 BPF_TRAMP_FENTRY,
476 BPF_TRAMP_FEXIT,476 BPF_TRAMP_FEXIT,
477 BPF_TRAMP_MODIFY_RETURN,
477 BPF_TRAMP_MAX,478 BPF_TRAMP_MAX,
478 BPF_TRAMP_REPLACE, /* more than MAX */479 BPF_TRAMP_REPLACE, /* more than MAX */
479};480};
include/uapi/linux/bpf.h+1−0
@@ -210,6 +210,7 @@ enum bpf_attach_type { · 209 unmodified lines
210 BPF_TRACE_RAW_TP,210 BPF_TRACE_RAW_TP,
211 BPF_TRACE_FENTRY,211 BPF_TRACE_FENTRY,
212 BPF_TRACE_FEXIT,212 BPF_TRACE_FEXIT,
213 BPF_MODIFY_RETURN,
213 __MAX_BPF_ATTACH_TYPE214 __MAX_BPF_ATTACH_TYPE
214};215};
215 216
kernel/bpf/btf.c+2−1
@@ -3710,7 +3710,8 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 3709 unmodified lines
3710 nr_args--;3710 nr_args--;
3711 }3711 }
3712 3712
3713 if (prog->expected_attach_type == BPF_TRACE_FEXIT &&3713 if ((prog->expected_attach_type == BPF_TRACE_FEXIT ||
3714 prog->expected_attach_type == BPF_MODIFY_RETURN) &&
3714 arg == nr_args) {3715 arg == nr_args) {
3715 if (!t)3716 if (!t)
3716 /* Default prog with 5 args. 6th arg is retval. */3717 /* Default prog with 5 args. 6th arg is retval. */
kernel/bpf/syscall.c+1−0
@@ -2324,6 +2324,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) · 2323 unmodified lines
2324 2324
2325 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&2325 if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2326 prog->expected_attach_type != BPF_TRACE_FEXIT &&2326 prog->expected_attach_type != BPF_TRACE_FEXIT &&
2327 prog->expected_attach_type != BPF_MODIFY_RETURN &&
2327 prog->type != BPF_PROG_TYPE_EXT) {2328 prog->type != BPF_PROG_TYPE_EXT) {
2328 err = -EINVAL;2329 err = -EINVAL;
2329 goto out_put_prog;2330 goto out_put_prog;
kernel/bpf/trampoline.c+4−1
@@ -232,7 +232,8 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 231 unmodified lines
232 goto out;232 goto out;
233 }233 }
234 234
235 if (tprogs[BPF_TRAMP_FEXIT].nr_progs)235 if (tprogs[BPF_TRAMP_FEXIT].nr_progs ||
236 tprogs[BPF_TRAMP_MODIFY_RETURN].nr_progs)
236 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;237 flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
237 238
238 /* Though the second half of trampoline page is unused a task could be239 /* Though the second half of trampoline page is unused a task could be
@@ -269,6 +270,8 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(enum bpf_attach_type t) · 30 unmodified lines
269 switch (t) {270 switch (t) {
270 case BPF_TRACE_FENTRY:271 case BPF_TRACE_FENTRY:
271 return BPF_TRAMP_FENTRY;272 return BPF_TRAMP_FENTRY;
273 case BPF_MODIFY_RETURN:
274 return BPF_TRAMP_MODIFY_RETURN;
272 case BPF_TRACE_FEXIT:275 case BPF_TRACE_FEXIT:
273 return BPF_TRAMP_FEXIT;276 return BPF_TRAMP_FEXIT;
274 default:277 default:
kernel/bpf/verifier.c+1−0
@@ -9950,6 +9950,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 9949 unmodified lines
9950 if (!prog_extension)9950 if (!prog_extension)
9951 return -EINVAL;9951 return -EINVAL;
9952 /* fallthrough */9952 /* fallthrough */
9953 case BPF_MODIFY_RETURN:
9953 case BPF_TRACE_FENTRY:9954 case BPF_TRACE_FENTRY:
9954 case BPF_TRACE_FEXIT:9955 case BPF_TRACE_FEXIT:
9955 if (!btf_type_is_func(t)) {9956 if (!btf_type_is_func(t)) {
tools/include/uapi/linux/bpf.h+1−0
@@ -210,6 +210,7 @@ enum bpf_attach_type { · 209 unmodified lines
210 BPF_TRACE_RAW_TP,210 BPF_TRACE_RAW_TP,
211 BPF_TRACE_FENTRY,211 BPF_TRACE_FENTRY,
212 BPF_TRACE_FEXIT,212 BPF_TRACE_FEXIT,
213 BPF_MODIFY_RETURN,
213 __MAX_BPF_ATTACH_TYPE214 __MAX_BPF_ATTACH_TYPE
214};215};
215 216

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t