History

bpf: Add bpf_trampoline_multi_attach/detach functions

Adding bpf_trampoline_multi_attach/detach functions that allows to attach/detach tracing program to multiple functions/trampolines. The attachment is defined with bpf_program and array of BTF ids of functions to attach the bpf program to. Adding bpf_tracing_multi_link object that holds all the attached trampolines and is initialized in attach and used in detach. The attachment allocates or uses currently existing trampoline for each function to attach and links it with the bpf program. The attach works as follows: - we get all the needed trampolines - lock them and add the bpf program to each (__bpf_trampoline_link_prog) - the trampoline_multi_ops passed in __bpf_trampoline_link_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_add/mod to update needed locations - we unlock all the trampolines The detach works as follows: - we lock all the needed trampolines - remove the program from each (__bpf_trampoline_unlink_prog) - the trampoline_multi_ops passed in __bpf_trampoline_unlink_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_del/mod to update needed locations - we unlock and put all the trampolines We store the old image/flags in the trampoline before the update and use it in case we need to rollback the attachment. We keep the ftrace_hash objects allocated during attach in the link so they can be used for detach as well. Adding trampoline_(un)lock_all functions to (un)lock all trampolines to gate the tracing_multi attachment. Note this is supported only for archs (x86_64) with ftrace direct and have single ops support. CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS It also needs CONFIG_BPF_SYSCALL enabled.

Jiri Olsa committed 5 months ago

Commit aef4dfa1 parent bd066594 files · +373 −0Browse files

Unified · Split

4 files
  1. include/linux/bpf.h +43 −0
  2. include/linux/bpf_verifier.h +4 −0
  3. kernel/bpf/trampoline.c +271 −0
  4. kernel/bpf/verifier.c +55 −0
include/linux/bpf.h+43−0
@@ -33,6 +33,7 @@ · 32 unmodified lines
33#include <linux/memcontrol.h>33#include <linux/memcontrol.h>
34#include <linux/cfi.h>34#include <linux/cfi.h>
35#include <linux/key.h>35#include <linux/key.h>
36#include <linux/ftrace.h>
36#include <asm/rqspinlock.h>37#include <asm/rqspinlock.h>
37 38
38struct bpf_verifier_env;39struct bpf_verifier_env;
@@ -1373,6 +1374,11 @@ struct bpf_trampoline { · 1334 unmodified lines
1373 int progs_cnt[BPF_TRAMP_MAX];1374 int progs_cnt[BPF_TRAMP_MAX];
1374 /* Executable image of trampoline */1375 /* Executable image of trampoline */
1375 struct bpf_tramp_image *cur_image;1376 struct bpf_tramp_image *cur_image;
1377 /* Used as temporary old image storage for multi_attach */
1378 struct {
1379 struct bpf_tramp_image *old_image;
1380 u32 old_flags;
1381 } multi_attach;
1376};1382};
1377 1383
1378struct bpf_attach_target_info {1384struct bpf_attach_target_info {
@@ -1470,6 +1476,8 @@ static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u6 · 91 unmodified lines
1470 return 0;1476 return 0;
1471}1477}
1472 1478
1479struct bpf_tracing_multi_link;
1480
1473#ifdef CONFIG_BPF_JIT1481#ifdef CONFIG_BPF_JIT
1474int bpf_trampoline_link_prog(struct bpf_tramp_node *node,1482int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
1475 struct bpf_trampoline *tr,1483 struct bpf_trampoline *tr,
@@ -1482,6 +1490,11 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, · 6 unmodified lines
1482void bpf_trampoline_put(struct bpf_trampoline *tr);1490void bpf_trampoline_put(struct bpf_trampoline *tr);
1483int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);1491int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1484 1492
1493int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1494 struct bpf_tracing_multi_link *link);
1495int bpf_trampoline_multi_detach(struct bpf_prog *prog,
1496 struct bpf_tracing_multi_link *link);
1497
1485/*1498/*
1486 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn1499 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1487 * indirection with a direct call to the bpf program. If the architecture does1500 * indirection with a direct call to the bpf program. If the architecture does
@@ -1594,6 +1607,16 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) · 106 unmodified lines
1594{1607{
1595 return false;1608 return false;
1596}1609}
1610static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1611 struct bpf_tracing_multi_link *link)
1612{
1613 return -ENOTSUPP;
1614}
1615static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog,
1616 struct bpf_tracing_multi_link *link)
1617{
1618 return -ENOTSUPP;
1619}
1597#endif1620#endif
1598 1621
1599struct bpf_func_info_aux {1622struct bpf_func_info_aux {
@@ -1932,6 +1955,26 @@ struct bpf_tracing_link { · 332 unmodified lines
1932 struct bpf_prog *tgt_prog;1955 struct bpf_prog *tgt_prog;
1933};1956};
1934 1957
1958struct bpf_tracing_multi_node {
1959 struct bpf_tramp_node node;
1960 struct bpf_trampoline *trampoline;
1961 struct ftrace_func_entry entry;
1962};
1963
1964struct bpf_tracing_multi_data {
1965 struct ftrace_hash *unreg;
1966 struct ftrace_hash *modify;
1967 struct ftrace_hash *reg;
1968 struct ftrace_func_entry *entry;
1969};
1970
1971struct bpf_tracing_multi_link {
1972 struct bpf_link link;
1973 struct bpf_tracing_multi_data data;
1974 int nodes_cnt;
1975 struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt);
1976};
1977
1935struct bpf_raw_tp_link {1978struct bpf_raw_tp_link {
1936 struct bpf_link link;1979 struct bpf_link link;
1937 struct bpf_raw_event_map *btp;1980 struct bpf_raw_event_map *btp;
include/linux/bpf_verifier.h+4−0
@@ -1591,6 +1591,10 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset); · 1590 unmodified lines
1591int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,1591int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
1592 struct bpf_insn *insn_buf, int insn_idx, int *cnt);1592 struct bpf_insn *insn_buf, int insn_idx, int *cnt);
1593 1593
1594/* Functions exported from verifier.c, used by trampoline.c */
1595int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id,
1596 struct bpf_attach_target_info *tgt_info);
1597
1594/* Functions in fixups.c, called from bpf_check() */1598/* Functions in fixups.c, called from bpf_check() */
1595int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);1599int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
1596int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);1600int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
kernel/bpf/trampoline.c+271−0
@@ -1447,6 +1447,277 @@ int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1446 unmodified lines
1447 return -ENOTSUPP;1447 return -ENOTSUPP;
1448}1448}
1449 1449
1450#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \
1451 defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) && \
1452 defined(CONFIG_BPF_SYSCALL)
1453
1454static void trampoline_lock_all(void)
1455{
1456 int i;
1457
1458 for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1459 mutex_lock(&trampoline_locks[i].mutex);
1460}
1461
1462static void trampoline_unlock_all(void)
1463{
1464 int i;
1465
1466 for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1467 mutex_unlock(&trampoline_locks[i].mutex);
1468}
1469
1470static void remove_tracing_multi_data(struct bpf_tracing_multi_data *data)
1471{
1472 ftrace_hash_remove(data->reg);
1473 ftrace_hash_remove(data->unreg);
1474 ftrace_hash_remove(data->modify);
1475}
1476
1477static void clear_tracing_multi_data(struct bpf_tracing_multi_data *data)
1478{
1479 remove_tracing_multi_data(data);
1480
1481 free_ftrace_hash(data->reg);
1482 free_ftrace_hash(data->unreg);
1483 free_ftrace_hash(data->modify);
1484}
1485
1486static int init_tracing_multi_data(struct bpf_tracing_multi_data *data)
1487{
1488 data->reg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1489 data->unreg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1490 data->modify = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1491
1492 if (!data->reg || !data->unreg || !data->modify) {
1493 clear_tracing_multi_data(data);
1494 return -ENOMEM;
1495 }
1496 return 0;
1497}
1498
1499static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry *entry,
1500 unsigned long ip, unsigned long direct)
1501{
1502 entry->ip = ip;
1503 entry->direct = direct;
1504 add_ftrace_hash_entry(hash, entry);
1505}
1506
1507static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr)
1508{
1509 unsigned long addr = (unsigned long) im->image;
1510 unsigned long ip = ftrace_location(tr->ip);
1511 struct bpf_tracing_multi_data *data = ptr;
1512
1513 if (bpf_trampoline_use_jmp(tr->flags))
1514 addr = ftrace_jmp_set(addr);
1515
1516 ftrace_hash_add(data->reg, data->entry, ip, addr);
1517 tr->cur_image = im;
1518 return 0;
1519}
1520
1521static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr)
1522{
1523 unsigned long addr = (unsigned long) tr->cur_image->image;
1524 unsigned long ip = ftrace_location(tr->ip);
1525 struct bpf_tracing_multi_data *data = ptr;
1526
1527 if (bpf_trampoline_use_jmp(tr->flags))
1528 addr = ftrace_jmp_set(addr);
1529
1530 ftrace_hash_add(data->unreg, data->entry, ip, addr);
1531 tr->cur_image = NULL;
1532 return 0;
1533}
1534
1535static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
1536 bool lock_direct_mutex, void *ptr)
1537{
1538 unsigned long addr = (unsigned long) im->image;
1539 unsigned long ip = ftrace_location(tr->ip);
1540 struct bpf_tracing_multi_data *data = ptr;
1541
1542 if (bpf_trampoline_use_jmp(tr->flags))
1543 addr = ftrace_jmp_set(addr);
1544
1545 ftrace_hash_add(data->modify, data->entry, ip, addr);
1546 tr->cur_image = im;
1547 return 0;
1548}
1549
1550static const struct bpf_trampoline_ops trampoline_multi_ops = {
1551 .register_fentry = register_fentry_multi,
1552 .unregister_fentry = unregister_fentry_multi,
1553 .modify_fentry = modify_fentry_multi,
1554};
1555
1556static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr)
1557{
1558 tr->multi_attach.old_image = tr->cur_image;
1559 tr->multi_attach.old_flags = tr->flags;
1560}
1561
1562static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr)
1563{
1564 if (tr->multi_attach.old_image)
1565 bpf_tramp_image_put(tr->multi_attach.old_image);
1566
1567 tr->multi_attach.old_image = NULL;
1568 tr->multi_attach.old_flags = 0;
1569}
1570
1571static void bpf_trampoline_multi_attach_rollback(struct bpf_trampoline *tr)
1572{
1573 if (tr->cur_image)
1574 bpf_tramp_image_put(tr->cur_image);
1575 tr->cur_image = tr->multi_attach.old_image;
1576 tr->flags = tr->multi_attach.old_flags;
1577
1578 tr->multi_attach.old_image = NULL;
1579 tr->multi_attach.old_flags = 0;
1580}
1581
1582#define for_each_mnode_cnt(mnode, link, cnt) \
1583 for (i = 0, mnode = &link->nodes[i]; i < cnt; i++, mnode = &link->nodes[i])
1584
1585#define for_each_mnode(mnode, link) \
1586 for_each_mnode_cnt(mnode, link, link->nodes_cnt)
1587
1588int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1589 struct bpf_tracing_multi_link *link)
1590{
1591 struct bpf_tracing_multi_data *data = &link->data;
1592 struct bpf_attach_target_info tgt_info = {};
1593 struct btf *btf = prog->aux->attach_btf;
1594 struct bpf_tracing_multi_node *mnode;
1595 struct bpf_trampoline *tr;
1596 int i, err, rollback_cnt;
1597 u64 key;
1598
1599 for_each_mnode(mnode, link) {
1600 rollback_cnt = i;
1601
1602 err = bpf_check_attach_btf_id_multi(btf, prog, ids[i], &tgt_info);
1603 if (err)
1604 goto rollback_put;
1605
1606 key = bpf_trampoline_compute_key(NULL, btf, ids[i]);
1607
1608 tr = bpf_trampoline_get(key, &tgt_info);
1609 if (!tr) {
1610 err = -ENOMEM;
1611 goto rollback_put;
1612 }
1613
1614 mnode->trampoline = tr;
1615 mnode->node.link = &link->link;
1616
1617 cond_resched();
1618 }
1619
1620 err = init_tracing_multi_data(data);
1621 if (err) {
1622 rollback_cnt = link->nodes_cnt;
1623 goto rollback_put;
1624 }
1625
1626 trampoline_lock_all();
1627
1628 for_each_mnode(mnode, link) {
1629 bpf_trampoline_multi_attach_init(mnode->trampoline);
1630
1631 data->entry = &mnode->entry;
1632 err = __bpf_trampoline_link_prog(&mnode->node, mnode->trampoline, NULL,
1633 &trampoline_multi_ops, data);
1634 if (err) {
1635 rollback_cnt = i;
1636 goto rollback_unlink;
1637 }
1638 }
1639
1640 rollback_cnt = link->nodes_cnt;
1641 if (ftrace_hash_count(data->reg)) {
1642 err = update_ftrace_direct_add(&direct_ops, data->reg);
1643 if (err)
1644 goto rollback_unlink;
1645 }
1646
1647 if (ftrace_hash_count(data->modify)) {
1648 err = update_ftrace_direct_mod(&direct_ops, data->modify, true);
1649 if (err) {
1650 if (ftrace_hash_count(data->reg))
1651 WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->reg));
1652 goto rollback_unlink;
1653 }
1654 }
1655
1656 for_each_mnode(mnode, link)
1657 bpf_trampoline_multi_attach_free(mnode->trampoline);
1658
1659 trampoline_unlock_all();
1660
1661 remove_tracing_multi_data(data);
1662 return 0;
1663
1664rollback_unlink:
1665 for_each_mnode_cnt(mnode, link, rollback_cnt) {
1666 bpf_trampoline_remove_prog(mnode->trampoline, &mnode->node);
1667 bpf_trampoline_multi_attach_rollback(mnode->trampoline);
1668 }
1669
1670 trampoline_unlock_all();
1671
1672 clear_tracing_multi_data(data);
1673 rollback_cnt = link->nodes_cnt;
1674
1675rollback_put:
1676 for_each_mnode_cnt(mnode, link, rollback_cnt)
1677 bpf_trampoline_put(mnode->trampoline);
1678
1679 return err;
1680}
1681
1682int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link)
1683{
1684 struct bpf_tracing_multi_data *data = &link->data;
1685 struct bpf_tracing_multi_node *mnode;
1686 int i;
1687
1688 trampoline_lock_all();
1689
1690 for_each_mnode(mnode, link) {
1691 data->entry = &mnode->entry;
1692 bpf_trampoline_multi_attach_init(mnode->trampoline);
1693 WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline,
1694 NULL, &trampoline_multi_ops, data));
1695 }
1696
1697 if (ftrace_hash_count(data->unreg))
1698 WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->unreg));
1699 if (ftrace_hash_count(data->modify))
1700 WARN_ON_ONCE(update_ftrace_direct_mod(&direct_ops, data->modify, true));
1701
1702 for_each_mnode(mnode, link)
1703 bpf_trampoline_multi_attach_free(mnode->trampoline);
1704
1705 trampoline_unlock_all();
1706
1707 for_each_mnode(mnode, link)
1708 bpf_trampoline_put(mnode->trampoline);
1709
1710 clear_tracing_multi_data(data);
1711 return 0;
1712}
1713
1714#undef for_each_mnode_cnt
1715#undef for_each_mnode
1716
1717#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS &&
1718 CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS &&
1719 CONFIG_BPF_SYSCALL */
1720
1450static int __init init_trampolines(void)1721static int __init init_trampolines(void)
1451{1722{
1452 int i;1723 int i;
kernel/bpf/verifier.c+55−0
@@ -19328,6 +19328,61 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 19327 unmodified lines
19328 return 0;19328 return 0;
19329}19329}
19330 19330
19331int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id,
19332 struct bpf_attach_target_info *tgt_info)
19333{
19334 const struct btf_type *t;
19335 unsigned long addr;
19336 const char *tname;
19337 int err;
19338
19339 if (!btf_id || !btf)
19340 return -EINVAL;
19341
19342 /* Check noreturn attachment. */
19343 if (prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI &&
19344 btf_id_set_contains(&noreturn_deny, btf_id))
19345 return -EINVAL;
19346 /* Check denied attachment. */
19347 if (btf_id_set_contains(&btf_id_deny, btf_id))
19348 return -EINVAL;
19349
19350 /* Check and get function target data. */
19351 t = btf_type_by_id(btf, btf_id);
19352 if (!t)
19353 return -EINVAL;
19354 tname = btf_name_by_offset(btf, t->name_off);
19355 if (!tname)
19356 return -EINVAL;
19357 if (!btf_type_is_func(t))
19358 return -EINVAL;
19359 t = btf_type_by_id(btf, t->type);
19360 if (!btf_type_is_func_proto(t))
19361 return -EINVAL;
19362 err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
19363 if (err < 0)
19364 return err;
19365 if (btf_is_module(btf)) {
19366 /* The bpf program already holds reference to module. */
19367 if (WARN_ON_ONCE(!prog->aux->mod))
19368 return -EINVAL;
19369 addr = find_kallsyms_symbol_value(prog->aux->mod, tname);
19370 } else {
19371 addr = kallsyms_lookup_name(tname);
19372 }
19373 if (!addr || !ftrace_location(addr))
19374 return -ENOENT;
19375
19376 /* Check sleepable program attachment. */
19377 if (prog->sleepable) {
19378 err = btf_id_allow_sleepable(btf_id, addr, prog, btf);
19379 if (err)
19380 return err;
19381 }
19382 tgt_info->tgt_addr = addr;
19383 return 0;
19384}
19385
19331struct btf *bpf_get_btf_vmlinux(void)19386struct btf *bpf_get_btf_vmlinux(void)
19332{19387{
19333 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {19388 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {