| @@ -1362,7 +1362,7 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, · 1361 unmodified lines |
| 1362 | 1362 | } |
| 1363 | 1363 | |
| 1364 | 1364 | static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, |
| 1365 | | − struct bpf_prog *p, int stack_size) |
| 1365 | + struct bpf_prog *p, int stack_size, bool mod_ret) |
| 1366 | 1366 | { |
| 1367 | 1367 | u8 *prog = *pprog; |
| 1368 | 1368 | int cnt = 0; |
| @@ -1383,6 +1383,13 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, · 14 unmodified lines |
| 1383 | 1383 | if (emit_call(&prog, p->bpf_func, prog)) |
| 1384 | 1384 | return -EINVAL; |
| 1385 | 1385 | |
| 1386 | + /* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return |
| 1387 | + * of the previous call which is then passed on the stack to |
| 1388 | + * the next BPF program. |
| 1389 | + */ |
| 1390 | + if (mod_ret) |
| 1391 | + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| 1392 | + |
| 1386 | 1393 | /* arg1: mov rdi, progs[i] */ |
| 1387 | 1394 | emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, |
| 1388 | 1395 | (u32) (long) p); |
| @@ -1442,6 +1449,23 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) · 53 unmodified lines |
| 1442 | 1449 | return 0; |
| 1443 | 1450 | } |
| 1444 | 1451 | |
| 1452 | +static int emit_mod_ret_check_imm8(u8 **pprog, int value) |
| 1453 | +{ |
| 1454 | + u8 *prog = *pprog; |
| 1455 | + int cnt = 0; |
| 1456 | + |
| 1457 | + if (!is_imm8(value)) |
| 1458 | + return -EINVAL; |
| 1459 | + |
| 1460 | + if (value == 0) |
| 1461 | + EMIT2(0x85, add_2reg(0xC0, BPF_REG_0, BPF_REG_0)); |
| 1462 | + else |
| 1463 | + EMIT3(0x83, add_1reg(0xF8, BPF_REG_0), value); |
| 1464 | + |
| 1465 | + *pprog = prog; |
| 1466 | + return 0; |
| 1467 | +} |
| 1468 | + |
| 1445 | 1469 | static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, |
| 1446 | 1470 | struct bpf_tramp_progs *tp, int stack_size) |
| 1447 | 1471 | { |
| @@ -1449,9 +1473,49 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, · 1 unmodified line |
| 1449 | 1473 | u8 *prog = *pprog; |
| 1450 | 1474 | |
| 1451 | 1475 | for (i = 0; i < tp->nr_progs; i++) { |
| 1452 | | − if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size)) |
| 1476 | + if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false)) |
| 1477 | + return -EINVAL; |
| 1478 | + } |
| 1479 | + *pprog = prog; |
| 1480 | + return 0; |
| 1481 | +} |
| 1482 | + |
| 1483 | +static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, |
| 1484 | + struct bpf_tramp_progs *tp, int stack_size, |
| 1485 | + u8 **branches) |
| 1486 | +{ |
| 1487 | + u8 *prog = *pprog; |
| 1488 | + int i; |
| 1489 | + |
| 1490 | + /* The first fmod_ret program will receive a garbage return value. |
| 1491 | + * Set this to 0 to avoid confusing the program. |
| 1492 | + */ |
| 1493 | + emit_mov_imm32(&prog, false, BPF_REG_0, 0); |
| 1494 | + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| 1495 | + for (i = 0; i < tp->nr_progs; i++) { |
| 1496 | + if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, true)) |
| 1453 | 1497 | return -EINVAL; |
| 1498 | + |
| 1499 | + /* Generate a branch: |
| 1500 | + * |
| 1501 | + * if (ret != 0) |
| 1502 | + * goto do_fexit; |
| 1503 | + * |
| 1504 | + * If needed this can be extended to any integer value which can |
| 1505 | + * be passed by user-space when the program is loaded. |
| 1506 | + */ |
| 1507 | + if (emit_mod_ret_check_imm8(&prog, 0)) |
| 1508 | + return -EINVAL; |
| 1509 | + |
| 1510 | + /* Save the location of the branch and Generate 6 nops |
| 1511 | + * (4 bytes for an offset and 2 bytes for the jump) These nops |
| 1512 | + * are replaced with a conditional jump once do_fexit (i.e. the |
| 1513 | + * start of the fexit invocation) is finalized. |
| 1514 | + */ |
| 1515 | + branches[i] = prog; |
| 1516 | + emit_nops(&prog, 4 + 2); |
| 1454 | 1517 | } |
| 1518 | + |
| 1455 | 1519 | *pprog = prog; |
| 1456 | 1520 | return 0; |
| 1457 | 1521 | } |
| @@ -1521,10 +1585,12 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 63 unmodified lines |
| 1521 | 1585 | struct bpf_tramp_progs *tprogs, |
| 1522 | 1586 | void *orig_call) |
| 1523 | 1587 | { |
| 1524 | | − int cnt = 0, nr_args = m->nr_args; |
| 1588 | + int ret, i, cnt = 0, nr_args = m->nr_args; |
| 1525 | 1589 | int stack_size = nr_args * 8; |
| 1526 | 1590 | struct bpf_tramp_progs *fentry = &tprogs[BPF_TRAMP_FENTRY]; |
| 1527 | 1591 | struct bpf_tramp_progs *fexit = &tprogs[BPF_TRAMP_FEXIT]; |
| 1592 | + struct bpf_tramp_progs *fmod_ret = &tprogs[BPF_TRAMP_MODIFY_RETURN]; |
| 1593 | + u8 **branches = NULL; |
| 1528 | 1594 | u8 *prog; |
| 1529 | 1595 | |
| 1530 | 1596 | /* x86-64 supports up to 6 arguments. 7+ can be added in the future */ |
| @@ -1557,24 +1623,60 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 26 unmodified lines |
| 1557 | 1623 | if (invoke_bpf(m, &prog, fentry, stack_size)) |
| 1558 | 1624 | return -EINVAL; |
| 1559 | 1625 | |
| 1626 | + if (fmod_ret->nr_progs) { |
| 1627 | + branches = kcalloc(fmod_ret->nr_progs, sizeof(u8 *), |
| 1628 | + GFP_KERNEL); |
| 1629 | + if (!branches) |
| 1630 | + return -ENOMEM; |
| 1631 | + |
| 1632 | + if (invoke_bpf_mod_ret(m, &prog, fmod_ret, stack_size, |
| 1633 | + branches)) { |
| 1634 | + ret = -EINVAL; |
| 1635 | + goto cleanup; |
| 1636 | + } |
| 1637 | + } |
| 1638 | + |
| 1560 | 1639 | if (flags & BPF_TRAMP_F_CALL_ORIG) { |
| 1561 | | − if (fentry->nr_progs) |
| 1640 | + if (fentry->nr_progs || fmod_ret->nr_progs) |
| 1562 | 1641 | restore_regs(m, &prog, nr_args, stack_size); |
| 1563 | 1642 | |
| 1564 | 1643 | /* call original function */ |
| 1565 | | − if (emit_call(&prog, orig_call, prog)) |
| 1566 | | − return -EINVAL; |
| 1644 | + if (emit_call(&prog, orig_call, prog)) { |
| 1645 | + ret = -EINVAL; |
| 1646 | + goto cleanup; |
| 1647 | + } |
| 1567 | 1648 | /* remember return value in a stack for bpf prog to access */ |
| 1568 | 1649 | emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); |
| 1569 | 1650 | } |
| 1570 | 1651 | |
| 1652 | + if (fmod_ret->nr_progs) { |
| 1653 | + /* From Intel 64 and IA-32 Architectures Optimization |
| 1654 | + * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler |
| 1655 | + * Coding Rule 11: All branch targets should be 16-byte |
| 1656 | + * aligned. |
| 1657 | + */ |
| 1658 | + emit_align(&prog, 16); |
| 1659 | + /* Update the branches saved in invoke_bpf_mod_ret with the |
| 1660 | + * aligned address of do_fexit. |
| 1661 | + */ |
| 1662 | + for (i = 0; i < fmod_ret->nr_progs; i++) |
| 1663 | + emit_cond_near_jump(&branches[i], prog, branches[i], |
| 1664 | + X86_JNE); |
| 1665 | + } |
| 1666 | + |
| 1571 | 1667 | if (fexit->nr_progs) |
| 1572 | | − if (invoke_bpf(m, &prog, fexit, stack_size)) |
| 1573 | | − return -EINVAL; |
| 1668 | + if (invoke_bpf(m, &prog, fexit, stack_size)) { |
| 1669 | + ret = -EINVAL; |
| 1670 | + goto cleanup; |
| 1671 | + } |
| 1574 | 1672 | |
| 1575 | 1673 | if (flags & BPF_TRAMP_F_RESTORE_REGS) |
| 1576 | 1674 | restore_regs(m, &prog, nr_args, stack_size); |
| 1577 | 1675 | |
| 1676 | + /* This needs to be done regardless. If there were fmod_ret programs, |
| 1677 | + * the return value is only updated on the stack and still needs to be |
| 1678 | + * restored to R0. |
| 1679 | + */ |
| 1578 | 1680 | if (flags & BPF_TRAMP_F_CALL_ORIG) |
| 1579 | 1681 | /* restore original return value back into RAX */ |
| 1580 | 1682 | emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); |
| @@ -1586,9 +1688,15 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, · 5 unmodified lines |
| 1586 | 1688 | EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ |
| 1587 | 1689 | EMIT1(0xC3); /* ret */ |
| 1588 | 1690 | /* Make sure the trampoline generation logic doesn't overflow */ |
| 1589 | | − if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) |
| 1590 | | − return -EFAULT; |
| 1591 | | − return prog - (u8 *)image; |
| 1691 | + if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { |
| 1692 | + ret = -EFAULT; |
| 1693 | + goto cleanup; |
| 1694 | + } |
| 1695 | + ret = prog - (u8 *)image; |
| 1696 | + |
| 1697 | +cleanup: |
| 1698 | + kfree(branches); |
| 1699 | + return ret; |
| 1592 | 1700 | } |
| 1593 | 1701 | |
| 1594 | 1702 | static int emit_fallback_jump(u8 **pprog) |