| @@ -1362,7 +1362,7 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, · 1361 unmodified lines |
| 1362 | } | 1362 | } |
| 1363 | | 1363 | |
| 1364 | static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, | 1364 | static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, |
| 1365 | struct bpf_prog *p, int stack_size) | 1365 | struct bpf_prog *p, int stack_size, bool mod_ret) |
| 1366 | { | 1366 | { |
| 1367 | u8 *prog = *pprog; | 1367 | u8 *prog = *pprog; |
| 1368 | int cnt = 0; | 1368 | int cnt = 0; |
| @@ -1383,6 +1383,13 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 14 unmodified lines |
| 1383 | if (emit_call(&prog, p->bpf_func, prog)) | 1383 | if (emit_call(&prog, p->bpf_func, prog)) |
| 1384 | return -EINVAL; | 1384 | return -EINVAL; |
| 1385 | | 1385 | |
| | 1386 | /* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return |
| | 1387 | * of the previous call which is then passed on the stack to |
| | 1388 | * the next BPF program. |
| | 1389 | */ |
| | 1390 | if (mod_ret) |
| | 1391 | emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| | 1392 | |
| 1386 | /* arg1: mov rdi, progs[i] */ | 1393 | /* arg1: mov rdi, progs[i] */ |
| 1387 | emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, | 1394 | emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, |
| 1388 | (u32) (long) p); | 1395 | (u32) (long) p); |
| @@ -1442,6 +1449,23 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) · 53 unmodified lines |
| 1442 | return 0; | 1449 | return 0; |
| 1443 | } | 1450 | } |
| 1444 | | 1451 | |
| | 1452 | static int emit_mod_ret_check_imm8(u8 **pprog, int value) |
| | 1453 | { |
| | 1454 | u8 *prog = *pprog; |
| | 1455 | int cnt = 0; |
| | 1456 | |
| | 1457 | if (!is_imm8(value)) |
| | 1458 | return -EINVAL; |
| | 1459 | |
| | 1460 | if (value == 0) |
| | 1461 | EMIT2(0x85, add_2reg(0xC0, BPF_REG_0, BPF_REG_0)); |
| | 1462 | else |
| | 1463 | EMIT3(0x83, add_1reg(0xF8, BPF_REG_0), value); |
| | 1464 | |
| | 1465 | *pprog = prog; |
| | 1466 | return 0; |
| | 1467 | } |
| | 1468 | |
| 1445 | static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, | 1469 | static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, |
| 1446 | struct bpf_tramp_progs *tp, int stack_size) | 1470 | struct bpf_tramp_progs *tp, int stack_size) |
| 1447 | { | 1471 | { |
| @@ -1449,9 +1473,49 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 1 unmodified line |
| 1449 | u8 *prog = *pprog; | 1473 | u8 *prog = *pprog; |
| 1450 | | 1474 | |
| 1451 | for (i = 0; i < tp->nr_progs; i++) { | 1475 | for (i = 0; i < tp->nr_progs; i++) { |
| 1452 | if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size)) | 1476 | if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false)) |
| | 1477 | return -EINVAL; |
| | 1478 | } |
| | 1479 | *pprog = prog; |
| | 1480 | return 0; |
| | 1481 | } |
| | 1482 | |
| | 1483 | static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, |
| | 1484 | struct bpf_tramp_progs *tp, int stack_size, |
| | 1485 | u8 **branches) |
| | 1486 | { |
| | 1487 | u8 *prog = *pprog; |
| | 1488 | int i; |
| | 1489 | |
| | 1490 | /* The first fmod_ret program will receive a garbage return value. |
| | 1491 | * Set this to 0 to avoid confusing the program. |
| | 1492 | */ |
| | 1493 | emit_mov_imm32(&prog, false, BPF_REG_0, 0); |
| | 1494 | emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| | 1495 | for (i = 0; i < tp->nr_progs; i++) { |
| | 1496 | if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true)) |
| 1453 | return -EINVAL; | 1497 | return -EINVAL; |
| | 1498 | |
| | 1499 | /* Generate a branch: |
| | 1500 | * |
| | 1501 | * if (ret != 0) |
| | 1502 | * goto do_fexit; |
| | 1503 | * |
| | 1504 | * If needed this can be extended to any integer value which can |
| | 1505 | * be passed by user-space when the program is loaded. |
| | 1506 | */ |
| | 1507 | if (emit_mod_ret_check_imm8(&prog, 0)) |
| | 1508 | return -EINVAL; |
| | 1509 | |
| | 1510 | /* Save the location of the branch and Generate 6 nops |
| | 1511 | * (4 bytes for an offset and 2 bytes for the jump) These nops |
| | 1512 | * are replaced with a conditional jump once do_fexit (i.e. the |
| | 1513 | * start of the fexit invocation) is finalized. |
| | 1514 | */ |
| | 1515 | branches[i] = prog; |
| | 1516 | emit_nops(&prog, 4 + 2); |
| 1454 | } | 1517 | } |
| | 1518 | |
| 1455 | *pprog = prog; | 1519 | *pprog = prog; |
| 1456 | return 0; | 1520 | return 0; |
| 1457 | } | 1521 | } |
| @@ -1521,10 +1585,12 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 63 unmodified lines |
| 1521 | struct bpf_tramp_progs *tprogs, | 1585 | struct bpf_tramp_progs *tprogs, |
| 1522 | void *orig_call) | 1586 | void *orig_call) |
| 1523 | { | 1587 | { |
| 1524 | int cnt = 0, nr_args = m->nr_args; | 1588 | int ret, i, cnt = 0, nr_args = m->nr_args; |
| 1525 | int stack_size = nr_args * 8; | 1589 | int stack_size = nr_args * 8; |
| 1526 | struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; | 1590 | struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; |
| 1527 | struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; | 1591 | struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; |
| | 1592 | struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN]; |
| | 1593 | u8 **branches = NULL; |
| 1528 | u8 *prog; | 1594 | u8 *prog; |
| 1529 | | 1595 | |
| 1530 | /* x86-64 supports up to 6 arguments. 7+ can be added in the future */ | 1596 | /* x86-64 supports up to 6 arguments. 7+ can be added in the future */ |
| @@ -1557,24 +1623,60 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 26 unmodified lines |
| 1557 | if (invoke_bpf(m, &prog, fentry, stack_size)) | 1623 | if (invoke_bpf(m, &prog, fentry, stack_size)) |
| 1558 | return -EINVAL; | 1624 | return -EINVAL; |
| 1559 | | 1625 | |
| | 1626 | if (fmod_ret->nr_progs) { |
| | 1627 | branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *), |
| | 1628 | GFP_KERNEL); |
| | 1629 | if (!branches) |
| | 1630 | return -ENOMEM; |
| | 1631 | |
| | 1632 | if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size, |
| | 1633 | branches)) { |
| | 1634 | ret = -EINVAL; |
| | 1635 | goto cleanup; |
| | 1636 | } |
| | 1637 | } |
| | 1638 | |
| 1560 | if (flags & BPF_TRAMP_F_CALL_ORIG) { | 1639 | if (flags & BPF_TRAMP_F_CALL_ORIG) { |
| 1561 | if (fentry->nr_progs) | 1640 | if (fentry->nr_progs || fmod_ret->nr_progs) |
| 1562 | restore_regs(m, &prog, nr_args, stack_size); | 1641 | restore_regs(m, &prog, nr_args, stack_size); |
| 1563 | | 1642 | |
| 1564 | /* call original function */ | 1643 | /* call original function */ |
| 1565 | if (emit_call(&prog, orig_call, prog)) | 1644 | if (emit_call(&prog, orig_call, prog)) { |
| 1566 | return -EINVAL; | 1645 | ret = -EINVAL; |
| | 1646 | goto cleanup; |
| | 1647 | } |
| 1567 | /* remember return value in a stack for bpf prog to access */ | 1648 | /* remember return value in a stack for bpf prog to access */ |
| 1568 | emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); | 1649 | emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| 1569 | } | 1650 | } |
| 1570 | | 1651 | |
| | 1652 | if (fmod_ret->nr_progs) { |
| | 1653 | /* From Intel 64 and IA-32 Architectures Optimization |
| | 1654 | * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler |
| | 1655 | * Coding Rule 11: All branch targets should be 16-byte |
| | 1656 | * aligned. |
| | 1657 | */ |
| | 1658 | emit_align(&prog, 16); |
| | 1659 | /* Update the branches saved in invoke_bpf_mod_ret with the |
| | 1660 | * aligned address of do_fexit. |
| | 1661 | */ |
| | 1662 | for (i = 0; i < fmod_ret->nr_progs; i++) |
| | 1663 | emit_cond_near_jump(&branches[i], prog, branches[i], |
| | 1664 | X86_JNE); |
| | 1665 | } |
| | 1666 | |
| 1571 | if (fexit->nr_progs) | 1667 | if (fexit->nr_progs) |
| 1572 | if (invoke_bpf(m, &prog, fexit, stack_size)) | 1668 | if (invoke_bpf(m, &prog, fexit, stack_size)) { |
| 1573 | return -EINVAL; | 1669 | ret = -EINVAL; |
| | 1670 | goto cleanup; |
| | 1671 | } |
| 1574 | | 1672 | |
| 1575 | if (flags & BPF_TRAMP_F_RESTORE_REGS) | 1673 | if (flags & BPF_TRAMP_F_RESTORE_REGS) |
| 1576 | restore_regs(m, &prog, nr_args, stack_size); | 1674 | restore_regs(m, &prog, nr_args, stack_size); |
| 1577 | | 1675 | |
| | 1676 | /* This needs to be done regardless. If there were fmod_ret programs, |
| | 1677 | * the return value is only updated on the stack and still needs to be |
| | 1678 | * restored to R0. |
| | 1679 | */ |
| 1578 | if (flags & BPF_TRAMP_F_CALL_ORIG) | 1680 | if (flags & BPF_TRAMP_F_CALL_ORIG) |
| 1579 | /* restore original return value back into RAX */ | 1681 | /* restore original return value back into RAX */ |
| 1580 | emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); | 1682 | emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); |
| @@ -1586,9 +1688,15 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 5 unmodified lines |
| 1586 | EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ | 1688 | EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ |
| 1587 | EMIT1(0xC3); /* ret */ | 1689 | EMIT1(0xC3); /* ret */ |
| 1588 | /* Make sure the trampoline generation logic doesn't overflow */ | 1690 | /* Make sure the trampoline generation logic doesn't overflow */ |
| 1589 | if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) | 1691 | if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { |
| 1590 | return -EFAULT; | 1692 | ret = -EFAULT; |
| 1591 | return prog - (u8 *)image; | 1693 | goto cleanup; |
| | 1694 | } |
| | 1695 | ret = prog - (u8 *)image; |
| | 1696 | |
| | 1697 | cleanup: |
| | 1698 | kfree(branches); |
| | 1699 | return ret; |
| 1592 | } | 1700 | } |
| 1593 | | 1701 | |
| 1594 | static int emit_fallback_jump(u8 **pprog) | 1702 | static int emit_fallback_jump(u8 **pprog) |