History

bpf: Add bpf_trampoline_multi_attach/detach functions

Adding bpf_trampoline_multi_attach/detach functions that allows to attach/detach tracing program to multiple functions/trampolines. The attachment is defined with bpf_program and array of BTF ids of functions to attach the bpf program to. Adding bpf_tracing_multi_link object that holds all the attached trampolines and is initialized in attach and used in detach. The attachment allocates or uses currently existing trampoline for each function to attach and links it with the bpf program. The attach works as follows: - we get all the needed trampolines - lock them and add the bpf program to each (__bpf_trampoline_link_prog) - the trampoline_multi_ops passed in __bpf_trampoline_link_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_add/mod to update needed locations - we unlock all the trampolines The detach works as follows: - we lock all the needed trampolines - remove the program from each (__bpf_trampoline_unlink_prog) - the trampoline_multi_ops passed in __bpf_trampoline_unlink_prog gathers ftrace_hash (ip -> trampoline) objects - we call update_ftrace_direct_del/mod to update needed locations - we unlock and put all the trampolines We store the old image/flags in the trampoline before the update and use it in case we need to rollback the attachment. We keep the ftrace_hash objects allocated during attach in the link so they can be used for detach as well. Adding trampoline_(un)lock_all functions to (un)lock all trampolines to gate the tracing_multi attachment. Note this is supported only for archs (x86_64) with ftrace direct and have single ops support. CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS && CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS It also needs CONFIG_BPF_SYSCALL enabled.

Jiri Olsa committed 5 months ago

Commit aef4dfa1 parent bd066594 files · +373 −0Browse files

Unified · Split

4 files
  1. include/linux/bpf.h +43 −0
  2. include/linux/bpf_verifier.h +4 −0
  3. kernel/bpf/trampoline.c +271 −0
  4. kernel/bpf/verifier.c +55 −0
include/linux/bpf.h+43−0
@@ -33,6 +33,7 @@ · 32 unmodified lines
3333 #include <linux/memcontrol.h>
3434 #include <linux/cfi.h>
3535 #include <linux/key.h>
36+#include <linux/ftrace.h>
3637 #include <asm/rqspinlock.h>
3738
3839 struct bpf_verifier_env;
@@ -1373,6 +1374,11 @@ struct bpf_trampoline { · 1334 unmodified lines
13731374 int progs_cnt[BPF_TRAMP_MAX];
13741375 /* Executable image of trampoline */
13751376 struct bpf_tramp_image *cur_image;
1377+ /* Used as temporary old image storage for multi_attach */
1378+ struct {
1379+ struct bpf_tramp_image *old_image;
1380+ u32 old_flags;
1381+ } multi_attach;
13761382 };
13771383
13781384 struct bpf_attach_target_info {
@@ -1470,6 +1476,8 @@ static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u6 · 91 unmodified lines
14701476 return 0;
14711477 }
14721478
1479+struct bpf_tracing_multi_link;
1480+
14731481 #ifdef CONFIG_BPF_JIT
14741482 int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
14751483 struct bpf_trampoline *tr,
@@ -1482,6 +1490,11 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, · 6 unmodified lines
14821490 void bpf_trampoline_put(struct bpf_trampoline *tr);
14831491 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
14841492
1493+int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1494+ struct bpf_tracing_multi_link *link);
1495+int bpf_trampoline_multi_detach(struct bpf_prog *prog,
1496+ struct bpf_tracing_multi_link *link);
1497+
14851498 /*
14861499 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
14871500 * indirection with a direct call to the bpf program. If the architecture does
@@ -1594,6 +1607,16 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) · 106 unmodified lines
15941607 {
15951608 return false;
15961609 }
1610+static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1611+ struct bpf_tracing_multi_link *link)
1612+{
1613+ return -ENOTSUPP;
1614+}
1615+static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog,
1616+ struct bpf_tracing_multi_link *link)
1617+{
1618+ return -ENOTSUPP;
1619+}
15971620 #endif
15981621
15991622 struct bpf_func_info_aux {
@@ -1932,6 +1955,26 @@ struct bpf_tracing_link { · 332 unmodified lines
19321955 struct bpf_prog *tgt_prog;
19331956 };
19341957
1958+struct bpf_tracing_multi_node {
1959+ struct bpf_tramp_node node;
1960+ struct bpf_trampoline *trampoline;
1961+ struct ftrace_func_entry entry;
1962+};
1963+
1964+struct bpf_tracing_multi_data {
1965+ struct ftrace_hash *unreg;
1966+ struct ftrace_hash *modify;
1967+ struct ftrace_hash *reg;
1968+ struct ftrace_func_entry *entry;
1969+};
1970+
1971+struct bpf_tracing_multi_link {
1972+ struct bpf_link link;
1973+ struct bpf_tracing_multi_data data;
1974+ int nodes_cnt;
1975+ struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt);
1976+};
1977+
19351978 struct bpf_raw_tp_link {
19361979 struct bpf_link link;
19371980 struct bpf_raw_event_map *btp;
include/linux/bpf_verifier.h+4−0
@@ -1591,6 +1591,10 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset); · 1590 unmodified lines
15911591 int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
15921592 struct bpf_insn *insn_buf, int insn_idx, int *cnt);
15931593
1594+/* Functions exported from verifier.c, used by trampoline.c */
1595+int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id,
1596+ struct bpf_attach_target_info *tgt_info);
1597+
15941598 /* Functions in fixups.c, called from bpf_check() */
15951599 int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
15961600 int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
kernel/bpf/trampoline.c+271−0
@@ -1447,6 +1447,277 @@ int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1446 unmodified lines
14471447 return -ENOTSUPP;
14481448 }
14491449
1450+#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) && \
1451+ defined(CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS) && \
1452+ defined(CONFIG_BPF_SYSCALL)
1453+
1454+static void trampoline_lock_all(void)
1455+{
1456+ int i;
1457+
1458+ for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1459+ mutex_lock(&trampoline_locks[i].mutex);
1460+}
1461+
1462+static void trampoline_unlock_all(void)
1463+{
1464+ int i;
1465+
1466+ for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
1467+ mutex_unlock(&trampoline_locks[i].mutex);
1468+}
1469+
1470+static void remove_tracing_multi_data(struct bpf_tracing_multi_data *data)
1471+{
1472+ ftrace_hash_remove(data->reg);
1473+ ftrace_hash_remove(data->unreg);
1474+ ftrace_hash_remove(data->modify);
1475+}
1476+
1477+static void clear_tracing_multi_data(struct bpf_tracing_multi_data *data)
1478+{
1479+ remove_tracing_multi_data(data);
1480+
1481+ free_ftrace_hash(data->reg);
1482+ free_ftrace_hash(data->unreg);
1483+ free_ftrace_hash(data->modify);
1484+}
1485+
1486+static int init_tracing_multi_data(struct bpf_tracing_multi_data *data)
1487+{
1488+ data->reg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1489+ data->unreg = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1490+ data->modify = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
1491+
1492+ if (!data->reg || !data->unreg || !data->modify) {
1493+ clear_tracing_multi_data(data);
1494+ return -ENOMEM;
1495+ }
1496+ return 0;
1497+}
1498+
1499+static void ftrace_hash_add(struct ftrace_hash *hash, struct ftrace_func_entry *entry,
1500+ unsigned long ip, unsigned long direct)
1501+{
1502+ entry->ip = ip;
1503+ entry->direct = direct;
1504+ add_ftrace_hash_entry(hash, entry);
1505+}
1506+
1507+static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_image *im, void *ptr)
1508+{
1509+ unsigned long addr = (unsigned long) im->image;
1510+ unsigned long ip = ftrace_location(tr->ip);
1511+ struct bpf_tracing_multi_data *data = ptr;
1512+
1513+ if (bpf_trampoline_use_jmp(tr->flags))
1514+ addr = ftrace_jmp_set(addr);
1515+
1516+ ftrace_hash_add(data->reg, data->entry, ip, addr);
1517+ tr->cur_image = im;
1518+ return 0;
1519+}
1520+
1521+static int unregister_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, void *ptr)
1522+{
1523+ unsigned long addr = (unsigned long) tr->cur_image->image;
1524+ unsigned long ip = ftrace_location(tr->ip);
1525+ struct bpf_tracing_multi_data *data = ptr;
1526+
1527+ if (bpf_trampoline_use_jmp(tr->flags))
1528+ addr = ftrace_jmp_set(addr);
1529+
1530+ ftrace_hash_add(data->unreg, data->entry, ip, addr);
1531+ tr->cur_image = NULL;
1532+ return 0;
1533+}
1534+
1535+static int modify_fentry_multi(struct bpf_trampoline *tr, u32 orig_flags, struct bpf_tramp_image *im,
1536+ bool lock_direct_mutex, void *ptr)
1537+{
1538+ unsigned long addr = (unsigned long) im->image;
1539+ unsigned long ip = ftrace_location(tr->ip);
1540+ struct bpf_tracing_multi_data *data = ptr;
1541+
1542+ if (bpf_trampoline_use_jmp(tr->flags))
1543+ addr = ftrace_jmp_set(addr);
1544+
1545+ ftrace_hash_add(data->modify, data->entry, ip, addr);
1546+ tr->cur_image = im;
1547+ return 0;
1548+}
1549+
1550+static const struct bpf_trampoline_ops trampoline_multi_ops = {
1551+ .register_fentry = register_fentry_multi,
1552+ .unregister_fentry = unregister_fentry_multi,
1553+ .modify_fentry = modify_fentry_multi,
1554+};
1555+
1556+static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr)
1557+{
1558+ tr->multi_attach.old_image = tr->cur_image;
1559+ tr->multi_attach.old_flags = tr->flags;
1560+}
1561+
1562+static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr)
1563+{
1564+ if (tr->multi_attach.old_image)
1565+ bpf_tramp_image_put(tr->multi_attach.old_image);
1566+
1567+ tr->multi_attach.old_image = NULL;
1568+ tr->multi_attach.old_flags = 0;
1569+}
1570+
1571+static void bpf_trampoline_multi_attach_rollback(struct bpf_trampoline *tr)
1572+{
1573+ if (tr->cur_image)
1574+ bpf_tramp_image_put(tr->cur_image);
1575+ tr->cur_image = tr->multi_attach.old_image;
1576+ tr->flags = tr->multi_attach.old_flags;
1577+
1578+ tr->multi_attach.old_image = NULL;
1579+ tr->multi_attach.old_flags = 0;
1580+}
1581+
1582+#define for_each_mnode_cnt(mnode, link, cnt) \
1583+ for (i = 0, mnode = &link->nodes[i]; i < cnt; i++, mnode = &link->nodes[i])
1584+
1585+#define for_each_mnode(mnode, link) \
1586+ for_each_mnode_cnt(mnode, link, link->nodes_cnt)
1587+
1588+int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1589+ struct bpf_tracing_multi_link *link)
1590+{
1591+ struct bpf_tracing_multi_data *data = &link->data;
1592+ struct bpf_attach_target_info tgt_info = {};
1593+ struct btf *btf = prog->aux->attach_btf;
1594+ struct bpf_tracing_multi_node *mnode;
1595+ struct bpf_trampoline *tr;
1596+ int i, err, rollback_cnt;
1597+ u64 key;
1598+
1599+ for_each_mnode(mnode, link) {
1600+ rollback_cnt = i;
1601+
1602+ err = bpf_check_attach_btf_id_multi(btf, prog, ids[i], &tgt_info);
1603+ if (err)
1604+ goto rollback_put;
1605+
1606+ key = bpf_trampoline_compute_key(NULL, btf, ids[i]);
1607+
1608+ tr = bpf_trampoline_get(key, &tgt_info);
1609+ if (!tr) {
1610+ err = -ENOMEM;
1611+ goto rollback_put;
1612+ }
1613+
1614+ mnode->trampoline = tr;
1615+ mnode->node.link = &link->link;
1616+
1617+ cond_resched();
1618+ }
1619+
1620+ err = init_tracing_multi_data(data);
1621+ if (err) {
1622+ rollback_cnt = link->nodes_cnt;
1623+ goto rollback_put;
1624+ }
1625+
1626+ trampoline_lock_all();
1627+
1628+ for_each_mnode(mnode, link) {
1629+ bpf_trampoline_multi_attach_init(mnode->trampoline);
1630+
1631+ data->entry = &mnode->entry;
1632+ err = __bpf_trampoline_link_prog(&mnode->node, mnode->trampoline, NULL,
1633+ &trampoline_multi_ops, data);
1634+ if (err) {
1635+ rollback_cnt = i;
1636+ goto rollback_unlink;
1637+ }
1638+ }
1639+
1640+ rollback_cnt = link->nodes_cnt;
1641+ if (ftrace_hash_count(data->reg)) {
1642+ err = update_ftrace_direct_add(&direct_ops, data->reg);
1643+ if (err)
1644+ goto rollback_unlink;
1645+ }
1646+
1647+ if (ftrace_hash_count(data->modify)) {
1648+ err = update_ftrace_direct_mod(&direct_ops, data->modify, true);
1649+ if (err) {
1650+ if (ftrace_hash_count(data->reg))
1651+ WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->reg));
1652+ goto rollback_unlink;
1653+ }
1654+ }
1655+
1656+ for_each_mnode(mnode, link)
1657+ bpf_trampoline_multi_attach_free(mnode->trampoline);
1658+
1659+ trampoline_unlock_all();
1660+
1661+ remove_tracing_multi_data(data);
1662+ return 0;
1663+
1664+rollback_unlink:
1665+ for_each_mnode_cnt(mnode, link, rollback_cnt) {
1666+ bpf_trampoline_remove_prog(mnode->trampoline, &mnode->node);
1667+ bpf_trampoline_multi_attach_rollback(mnode->trampoline);
1668+ }
1669+
1670+ trampoline_unlock_all();
1671+
1672+ clear_tracing_multi_data(data);
1673+ rollback_cnt = link->nodes_cnt;
1674+
1675+rollback_put:
1676+ for_each_mnode_cnt(mnode, link, rollback_cnt)
1677+ bpf_trampoline_put(mnode->trampoline);
1678+
1679+ return err;
1680+}
1681+
1682+int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link)
1683+{
1684+ struct bpf_tracing_multi_data *data = &link->data;
1685+ struct bpf_tracing_multi_node *mnode;
1686+ int i;
1687+
1688+ trampoline_lock_all();
1689+
1690+ for_each_mnode(mnode, link) {
1691+ data->entry = &mnode->entry;
1692+ bpf_trampoline_multi_attach_init(mnode->trampoline);
1693+ WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline,
1694+ NULL, &trampoline_multi_ops, data));
1695+ }
1696+
1697+ if (ftrace_hash_count(data->unreg))
1698+ WARN_ON_ONCE(update_ftrace_direct_del(&direct_ops, data->unreg));
1699+ if (ftrace_hash_count(data->modify))
1700+ WARN_ON_ONCE(update_ftrace_direct_mod(&direct_ops, data->modify, true));
1701+
1702+ for_each_mnode(mnode, link)
1703+ bpf_trampoline_multi_attach_free(mnode->trampoline);
1704+
1705+ trampoline_unlock_all();
1706+
1707+ for_each_mnode(mnode, link)
1708+ bpf_trampoline_put(mnode->trampoline);
1709+
1710+ clear_tracing_multi_data(data);
1711+ return 0;
1712+}
1713+
1714+#undef for_each_mnode_cnt
1715+#undef for_each_mnode
1716+
1717+#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS &&
1718+ CONFIG_HAVE_SINGLE_FTRACE_DIRECT_OPS &&
1719+ CONFIG_BPF_SYSCALL */
1720+
14501721 static int __init init_trampolines(void)
14511722 {
14521723 int i;
kernel/bpf/verifier.c+55−0
@@ -19328,6 +19328,61 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 19327 unmodified lines
1932819328 return 0;
1932919329 }
1933019330
19331+int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id,
19332+ struct bpf_attach_target_info *tgt_info)
19333+{
19334+ const struct btf_type *t;
19335+ unsigned long addr;
19336+ const char *tname;
19337+ int err;
19338+
19339+ if (!btf_id || !btf)
19340+ return -EINVAL;
19341+
19342+ /* Check noreturn attachment. */
19343+ if (prog->expected_attach_type == BPF_TRACE_FEXIT_MULTI &&
19344+ btf_id_set_contains(&noreturn_deny, btf_id))
19345+ return -EINVAL;
19346+ /* Check denied attachment. */
19347+ if (btf_id_set_contains(&btf_id_deny, btf_id))
19348+ return -EINVAL;
19349+
19350+ /* Check and get function target data. */
19351+ t = btf_type_by_id(btf, btf_id);
19352+ if (!t)
19353+ return -EINVAL;
19354+ tname = btf_name_by_offset(btf, t->name_off);
19355+ if (!tname)
19356+ return -EINVAL;
19357+ if (!btf_type_is_func(t))
19358+ return -EINVAL;
19359+ t = btf_type_by_id(btf, t->type);
19360+ if (!btf_type_is_func_proto(t))
19361+ return -EINVAL;
19362+ err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
19363+ if (err < 0)
19364+ return err;
19365+ if (btf_is_module(btf)) {
19366+ /* The bpf program already holds reference to module. */
19367+ if (WARN_ON_ONCE(!prog->aux->mod))
19368+ return -EINVAL;
19369+ addr = find_kallsyms_symbol_value(prog->aux->mod, tname);
19370+ } else {
19371+ addr = kallsyms_lookup_name(tname);
19372+ }
19373+ if (!addr || !ftrace_location(addr))
19374+ return -ENOENT;
19375+
19376+ /* Check sleepable program attachment. */
19377+ if (prog->sleepable) {
19378+ err = btf_id_allow_sleepable(btf_id, addr, prog, btf);
19379+ if (err)
19380+ return err;
19381+ }
19382+ tgt_info->tgt_addr = addr;
19383+ return 0;
19384+}
19385+
1933119386 struct btf *bpf_get_btf_vmlinux(void)
1933219387 {
1933319388 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {