History

bpf: Add cookie support to programs attached with kprobe multi link

Adding support to call bpf_get_attach_cookie helper from kprobe programs attached with kprobe multi link. The cookie is provided by array of u64 values, where each value is paired with provided function address or symbol with the same array index. When cookie array is provided it's sorted together with addresses (check bpf_kprobe_multi_cookie_swap). This way we can find cookie based on the address in bpf_get_attach_cookie helper.

Jiri Olsa committed 4 years ago

Commit ca748231 parent 97ee4d24 files · +116 −2Browse files

Unified · Split

4 files
  1. include/uapi/linux/bpf.h +1 −0
  2. kernel/bpf/syscall.c +1 −1
  3. kernel/trace/bpf_trace.c +113 −1
  4. tools/include/uapi/linux/bpf.h +1 −0
include/uapi/linux/bpf.h+1−0
@@ -1487,6 +1487,7 @@ union bpf_attr { · 1486 unmodified lines
14871487 __u32 cnt;
14881488 __aligned_u64 syms;
14891489 __aligned_u64 addrs;
1490+ __aligned_u64 cookies;
14901491 } kprobe_multi;
14911492 };
14921493 } link_create;
kernel/bpf/syscall.c+1−1
@@ -4261,7 +4261,7 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr, · 4260 unmodified lines
42614261 return -EINVAL;
42624262 }
42634263
4264−#define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.addrs
4264+#define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.cookies
42654265 static int link_create(union bpf_attr *attr, bpfptr_t uattr)
42664266 {
42674267 enum bpf_prog_type ptype;
kernel/trace/bpf_trace.c+113−1
@@ -18,6 +18,8 @@ · 17 unmodified lines
1818 #include <linux/btf_ids.h>
1919 #include <linux/bpf_lsm.h>
2020 #include <linux/fprobe.h>
21+#include <linux/bsearch.h>
22+#include <linux/sort.h>
2123
2224 #include <net/bpf_sk_storage.h>
2325
@@ -78,6 +80,7 @@ u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 54 unmodified lines
7880 static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
7981 u64 flags, const struct btf **btf,
8082 s32 *btf_id);
83+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip);
8184
8285 /**
8386 * trace_call_bpf - invoke BPF program
@@ -1050,6 +1053,18 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = { · 966 unmodified lines
10501053 .arg1_type = ARG_PTR_TO_CTX,
10511054 };
10521055
1056+BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs)
1057+{
1058+ return bpf_kprobe_multi_cookie(current->bpf_ctx, instruction_pointer(regs));
1059+}
1060+
1061+static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
1062+ .func = bpf_get_attach_cookie_kprobe_multi,
1063+ .gpl_only = false,
1064+ .ret_type = RET_INTEGER,
1065+ .arg1_type = ARG_PTR_TO_CTX,
1066+};
1067+
10531068 BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)
10541069 {
10551070 struct bpf_trace_run_ctx *run_ctx;
@@ -1297,7 +1312,9 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 241 unmodified lines
12971312 &bpf_get_func_ip_proto_kprobe_multi :
12981313 &bpf_get_func_ip_proto_kprobe;
12991314 case BPF_FUNC_get_attach_cookie:
1300− return &bpf_get_attach_cookie_proto_trace;
1315+ return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ?
1316+ &bpf_get_attach_cookie_proto_kmulti :
1317+ &bpf_get_attach_cookie_proto_trace;
13011318 default:
13021319 return bpf_tracing_func_proto(func_id, prog);
13031320 }
@@ -2203,6 +2220,13 @@ struct bpf_kprobe_multi_link { · 899 unmodified lines
22032220 struct bpf_link link;
22042221 struct fprobe fp;
22052222 unsigned long *addrs;
2223+ /*
2224+ * The run_ctx here is used to get struct bpf_kprobe_multi_link in
2225+ * get_attach_cookie helper, so it can't be used to store data.
2226+ */
2227+ struct bpf_run_ctx run_ctx;
2228+ u64 *cookies;
2229+ u32 cnt;
22062230 };
22072231
22082232 static void bpf_kprobe_multi_link_release(struct bpf_link *link)
@@ -2219,6 +2243,7 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) · 10 unmodified lines
22192243
22202244 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
22212245 kvfree(kmulti_link->addrs);
2246+ kvfree(kmulti_link->cookies);
22222247 kfree(kmulti_link);
22232248 }
22242249
@@ -2227,10 +2252,60 @@ static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { · 2 unmodified lines
22272252 .dealloc = bpf_kprobe_multi_link_dealloc,
22282253 };
22292254
2255+static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
2256+{
2257+ const struct bpf_kprobe_multi_link *link = priv;
2258+ unsigned long *addr_a = a, *addr_b = b;
2259+ u64 *cookie_a, *cookie_b;
2260+ unsigned long tmp1;
2261+ u64 tmp2;
2262+
2263+ cookie_a = link->cookies + (addr_a - link->addrs);
2264+ cookie_b = link->cookies + (addr_b - link->addrs);
2265+
2266+ /* swap addr_a/addr_b and cookie_a/cookie_b values */
2267+ tmp1 = *addr_a; *addr_a = *addr_b; *addr_b = tmp1;
2268+ tmp2 = *cookie_a; *cookie_a = *cookie_b; *cookie_b = tmp2;
2269+}
2270+
2271+static int __bpf_kprobe_multi_cookie_cmp(const void *a, const void *b)
2272+{
2273+ const unsigned long *addr_a = a, *addr_b = b;
2274+
2275+ if (*addr_a == *addr_b)
2276+ return 0;
2277+ return *addr_a < *addr_b ? -1 : 1;
2278+}
2279+
2280+static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv)
2281+{
2282+ return __bpf_kprobe_multi_cookie_cmp(a, b);
2283+}
2284+
2285+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
2286+{
2287+ struct bpf_kprobe_multi_link *link;
2288+ unsigned long *addr;
2289+ u64 *cookie;
2290+
2291+ if (WARN_ON_ONCE(!ctx))
2292+ return 0;
2293+ link = container_of(ctx, struct bpf_kprobe_multi_link, run_ctx);
2294+ if (!link->cookies)
2295+ return 0;
2296+ addr = bsearch(&ip, link->addrs, link->cnt, sizeof(ip),
2297+ __bpf_kprobe_multi_cookie_cmp);
2298+ if (!addr)
2299+ return 0;
2300+ cookie = link->cookies + (addr - link->addrs);
2301+ return *cookie;
2302+}
2303+
22302304 static int
22312305 kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
22322306 struct pt_regs *regs)
22332307 {
2308+ struct bpf_run_ctx *old_run_ctx;
22342309 int err;
22352310
22362311 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
@@ -2240,7 +2315,9 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · 3 unmodified lines
22402315
22412316 migrate_disable();
22422317 rcu_read_lock();
2318+ old_run_ctx = bpf_set_run_ctx(&link->run_ctx);
22432319 err = bpf_prog_run(link->link.prog, regs);
2320+ bpf_reset_run_ctx(old_run_ctx);
22442321 rcu_read_unlock();
22452322 migrate_enable();
22462323
@@ -2326,9 +2403,11 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 79 unmodified lines
23262403 {
23272404 struct bpf_kprobe_multi_link *link = NULL;
23282405 struct bpf_link_primer link_primer;
2406+ void __user *ucookies;
23292407 unsigned long *addrs;
23302408 u32 flags, cnt, size;
23312409 void __user *uaddrs;
2410+ u64 *cookies = NULL;
23322411 void __user *usyms;
23332412 int err;
23342413
@@ -2368,6 +2447,19 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 33 unmodified lines
23682447 goto error;
23692448 }
23702449
2450+ ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies);
2451+ if (ucookies) {
2452+ cookies = kvmalloc(size, GFP_KERNEL);
2453+ if (!cookies) {
2454+ err = -ENOMEM;
2455+ goto error;
2456+ }
2457+ if (copy_from_user(cookies, ucookies, size)) {
2458+ err = -EFAULT;
2459+ goto error;
2460+ }
2461+ }
2462+
23712463 link = kzalloc(sizeof(*link), GFP_KERNEL);
23722464 if (!link) {
23732465 err = -ENOMEM;
@@ -2387,6 +2479,21 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 13 unmodified lines
23872479 link->fp.entry_handler = kprobe_multi_link_handler;
23882480
23892481 link->addrs = addrs;
2482+ link->cookies = cookies;
2483+ link->cnt = cnt;
2484+
2485+ if (cookies) {
2486+ /*
2487+ * Sorting addresses will trigger sorting cookies as well
2488+ * (check bpf_kprobe_multi_cookie_swap). This way we can
2489+ * find cookie based on the address in bpf_get_attach_cookie
2490+ * helper.
2491+ */
2492+ sort_r(addrs, cnt, sizeof(*addrs),
2493+ bpf_kprobe_multi_cookie_cmp,
2494+ bpf_kprobe_multi_cookie_swap,
2495+ link);
2496+ }
23902497
23912498 err = register_fprobe_ips(&link->fp, addrs, cnt);
23922499 if (err) {
@@ -2399,6 +2506,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 6 unmodified lines
23992506 error:
24002507 kfree(link);
24012508 kvfree(addrs);
2509+ kvfree(cookies);
24022510 return err;
24032511 }
24042512 #else /* !CONFIG_FPROBE */
@@ -2406,4 +2514,8 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 1 unmodified line
24062514 {
24072515 return -EOPNOTSUPP;
24082516 }
2517+static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
2518+{
2519+ return 0;
2520+}
24092521 #endif
tools/include/uapi/linux/bpf.h+1−0
@@ -1487,6 +1487,7 @@ union bpf_attr { · 1486 unmodified lines
14871487 __u32 cnt;
14881488 __aligned_u64 syms;
14891489 __aligned_u64 addrs;
1490+ __aligned_u64 cookies;
14901491 } kprobe_multi;
14911492 };
14921493 } link_create;