History

bpf: Add cookie support to programs attached with kprobe multi link

Adding support to call bpf_get_attach_cookie helper from kprobe programs attached with kprobe multi link. The cookie is provided by array of u64 values, where each value is paired with provided function address or symbol with the same array index. When cookie array is provided it's sorted together with addresses (check bpf_kprobe_multi_cookie_swap). This way we can find cookie based on the address in bpf_get_attach_cookie helper.

Jiri Olsa committed 4 years ago

Commit ca748231 parent 97ee4d24 files · +116 −2Browse files

Unified · Split

4 files
  1. include/uapi/linux/bpf.h +1 −0
  2. kernel/bpf/syscall.c +1 −1
  3. kernel/trace/bpf_trace.c +113 −1
  4. tools/include/uapi/linux/bpf.h +1 −0
include/uapi/linux/bpf.h+1−0
@@ -1487,6 +1487,7 @@ union bpf_attr { · 1486 unmodified lines
1487 __u32 cnt;1487 __u32 cnt;
1488 __aligned_u64 syms;1488 __aligned_u64 syms;
1489 __aligned_u64 addrs;1489 __aligned_u64 addrs;
1490 __aligned_u64 cookies;
1490 } kprobe_multi;1491 } kprobe_multi;
1491 };1492 };
1492 } link_create;1493 } link_create;
kernel/bpf/syscall.c+1−1
@@ -4261,7 +4261,7 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, bpfptr_t uattr, · 4260 unmodified lines
4261 return -EINVAL;4261 return -EINVAL;
4262}4262}
4263 4263
4264#define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.addrs4264#define BPF_LINK_CREATE_LAST_FIELD link_create.kprobe_multi.cookies
4265static int link_create(union bpf_attr *attr, bpfptr_t uattr)4265static int link_create(union bpf_attr *attr, bpfptr_t uattr)
4266{4266{
4267 enum bpf_prog_type ptype;4267 enum bpf_prog_type ptype;
kernel/trace/bpf_trace.c+113−1
@@ -18,6 +18,8 @@ · 17 unmodified lines
18#include <linux/btf_ids.h>18#include <linux/btf_ids.h>
19#include <linux/bpf_lsm.h>19#include <linux/bpf_lsm.h>
20#include <linux/fprobe.h>20#include <linux/fprobe.h>
21#include <linux/bsearch.h>
22#include <linux/sort.h>
21 23
22#include <net/bpf_sk_storage.h>24#include <net/bpf_sk_storage.h>
23 25
@@ -78,6 +80,7 @@ u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 54 unmodified lines
78static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,80static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
79 u64 flags, const struct btf **btf,81 u64 flags, const struct btf **btf,
80 s32 *btf_id);82 s32 *btf_id);
83static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip);
81 84
82/**85/**
83 * trace_call_bpf - invoke BPF program86 * trace_call_bpf - invoke BPF program
@@ -1050,6 +1053,18 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = { · 966 unmodified lines
1050 .arg1_type = ARG_PTR_TO_CTX,1053 .arg1_type = ARG_PTR_TO_CTX,
1051};1054};
1052 1055
1056BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs)
1057{
1058 return bpf_kprobe_multi_cookie(current->bpf_ctx, instruction_pointer(regs));
1059}
1060
1061static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = {
1062 .func = bpf_get_attach_cookie_kprobe_multi,
1063 .gpl_only = false,
1064 .ret_type = RET_INTEGER,
1065 .arg1_type = ARG_PTR_TO_CTX,
1066};
1067
1053BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)1068BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx)
1054{1069{
1055 struct bpf_trace_run_ctx *run_ctx;1070 struct bpf_trace_run_ctx *run_ctx;
@@ -1297,7 +1312,9 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 241 unmodified lines
1297 &bpf_get_func_ip_proto_kprobe_multi :1312 &bpf_get_func_ip_proto_kprobe_multi :
1298 &bpf_get_func_ip_proto_kprobe;1313 &bpf_get_func_ip_proto_kprobe;
1299 case BPF_FUNC_get_attach_cookie:1314 case BPF_FUNC_get_attach_cookie:
1300 return &bpf_get_attach_cookie_proto_trace;1315 return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI ?
1316 &bpf_get_attach_cookie_proto_kmulti :
1317 &bpf_get_attach_cookie_proto_trace;
1301 default:1318 default:
1302 return bpf_tracing_func_proto(func_id, prog);1319 return bpf_tracing_func_proto(func_id, prog);
1303 }1320 }
@@ -2203,6 +2220,13 @@ struct bpf_kprobe_multi_link { · 899 unmodified lines
2203 struct bpf_link link;2220 struct bpf_link link;
2204 struct fprobe fp;2221 struct fprobe fp;
2205 unsigned long *addrs;2222 unsigned long *addrs;
2223 /*
2224 * The run_ctx here is used to get struct bpf_kprobe_multi_link in
2225 * get_attach_cookie helper, so it can't be used to store data.
2226 */
2227 struct bpf_run_ctx run_ctx;
2228 u64 *cookies;
2229 u32 cnt;
2206};2230};
2207 2231
2208static void bpf_kprobe_multi_link_release(struct bpf_link *link)2232static void bpf_kprobe_multi_link_release(struct bpf_link *link)
@@ -2219,6 +2243,7 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) · 10 unmodified lines
2219 2243
2220 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);2244 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
2221 kvfree(kmulti_link->addrs);2245 kvfree(kmulti_link->addrs);
2246 kvfree(kmulti_link->cookies);
2222 kfree(kmulti_link);2247 kfree(kmulti_link);
2223}2248}
2224 2249
@@ -2227,10 +2252,60 @@ static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { · 2 unmodified lines
2227 .dealloc = bpf_kprobe_multi_link_dealloc,2252 .dealloc = bpf_kprobe_multi_link_dealloc,
2228};2253};
2229 2254
2255static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
2256{
2257 const struct bpf_kprobe_multi_link *link = priv;
2258 unsigned long *addr_a = a, *addr_b = b;
2259 u64 *cookie_a, *cookie_b;
2260 unsigned long tmp1;
2261 u64 tmp2;
2262
2263 cookie_a = link->cookies + (addr_a - link->addrs);
2264 cookie_b = link->cookies + (addr_b - link->addrs);
2265
2266 /* swap addr_a/addr_b and cookie_a/cookie_b values */
2267 tmp1 = *addr_a; *addr_a = *addr_b; *addr_b = tmp1;
2268 tmp2 = *cookie_a; *cookie_a = *cookie_b; *cookie_b = tmp2;
2269}
2270
2271static int __bpf_kprobe_multi_cookie_cmp(const void *a, const void *b)
2272{
2273 const unsigned long *addr_a = a, *addr_b = b;
2274
2275 if (*addr_a == *addr_b)
2276 return 0;
2277 return *addr_a < *addr_b ? -1 : 1;
2278}
2279
2280static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv)
2281{
2282 return __bpf_kprobe_multi_cookie_cmp(a, b);
2283}
2284
2285static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
2286{
2287 struct bpf_kprobe_multi_link *link;
2288 unsigned long *addr;
2289 u64 *cookie;
2290
2291 if (WARN_ON_ONCE(!ctx))
2292 return 0;
2293 link = container_of(ctx, struct bpf_kprobe_multi_link, run_ctx);
2294 if (!link->cookies)
2295 return 0;
2296 addr = bsearch(&ip, link->addrs, link->cnt, sizeof(ip),
2297 __bpf_kprobe_multi_cookie_cmp);
2298 if (!addr)
2299 return 0;
2300 cookie = link->cookies + (addr - link->addrs);
2301 return *cookie;
2302}
2303
2230static int2304static int
2231kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,2305kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
2232 struct pt_regs *regs)2306 struct pt_regs *regs)
2233{2307{
2308 struct bpf_run_ctx *old_run_ctx;
2234 int err;2309 int err;
2235 2310
2236 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {2311 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
@@ -2240,7 +2315,9 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, · 3 unmodified lines
2240 2315
2241 migrate_disable();2316 migrate_disable();
2242 rcu_read_lock();2317 rcu_read_lock();
2318 old_run_ctx = bpf_set_run_ctx(&link->run_ctx);
2243 err = bpf_prog_run(link->link.prog, regs);2319 err = bpf_prog_run(link->link.prog, regs);
2320 bpf_reset_run_ctx(old_run_ctx);
2244 rcu_read_unlock();2321 rcu_read_unlock();
2245 migrate_enable();2322 migrate_enable();
2246 2323
@@ -2326,9 +2403,11 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 79 unmodified lines
2326{2403{
2327 struct bpf_kprobe_multi_link *link = NULL;2404 struct bpf_kprobe_multi_link *link = NULL;
2328 struct bpf_link_primer link_primer;2405 struct bpf_link_primer link_primer;
2406 void __user *ucookies;
2329 unsigned long *addrs;2407 unsigned long *addrs;
2330 u32 flags, cnt, size;2408 u32 flags, cnt, size;
2331 void __user *uaddrs;2409 void __user *uaddrs;
2410 u64 *cookies = NULL;
2332 void __user *usyms;2411 void __user *usyms;
2333 int err;2412 int err;
2334 2413
@@ -2368,6 +2447,19 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 33 unmodified lines
2368 goto error;2447 goto error;
2369 }2448 }
2370 2449
2450 ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies);
2451 if (ucookies) {
2452 cookies = kvmalloc(size, GFP_KERNEL);
2453 if (!cookies) {
2454 err = -ENOMEM;
2455 goto error;
2456 }
2457 if (copy_from_user(cookies, ucookies, size)) {
2458 err = -EFAULT;
2459 goto error;
2460 }
2461 }
2462
2371 link = kzalloc(sizeof(*link), GFP_KERNEL);2463 link = kzalloc(sizeof(*link), GFP_KERNEL);
2372 if (!link) {2464 if (!link) {
2373 err = -ENOMEM;2465 err = -ENOMEM;
@@ -2387,6 +2479,21 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 13 unmodified lines
2387 link->fp.entry_handler = kprobe_multi_link_handler;2479 link->fp.entry_handler = kprobe_multi_link_handler;
2388 2480
2389 link->addrs = addrs;2481 link->addrs = addrs;
2482 link->cookies = cookies;
2483 link->cnt = cnt;
2484
2485 if (cookies) {
2486 /*
2487 * Sorting addresses will trigger sorting cookies as well
2488 * (check bpf_kprobe_multi_cookie_swap). This way we can
2489 * find cookie based on the address in bpf_get_attach_cookie
2490 * helper.
2491 */
2492 sort_r(addrs, cnt, sizeof(*addrs),
2493 bpf_kprobe_multi_cookie_cmp,
2494 bpf_kprobe_multi_cookie_swap,
2495 link);
2496 }
2390 2497
2391 err = register_fprobe_ips(&link->fp, addrs, cnt);2498 err = register_fprobe_ips(&link->fp, addrs, cnt);
2392 if (err) {2499 if (err) {
@@ -2399,6 +2506,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 6 unmodified lines
2399error:2506error:
2400 kfree(link);2507 kfree(link);
2401 kvfree(addrs);2508 kvfree(addrs);
2509 kvfree(cookies);
2402 return err;2510 return err;
2403}2511}
2404#else /* !CONFIG_FPROBE */2512#else /* !CONFIG_FPROBE */
@@ -2406,4 +2514,8 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 1 unmodified line
2406{2514{
2407 return -EOPNOTSUPP;2515 return -EOPNOTSUPP;
2408}2516}
2517static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx, u64 ip)
2518{
2519 return 0;
2520}
2409#endif2521#endif
tools/include/uapi/linux/bpf.h+1−0
@@ -1487,6 +1487,7 @@ union bpf_attr { · 1486 unmodified lines
1487 __u32 cnt;1487 __u32 cnt;
1488 __aligned_u64 syms;1488 __aligned_u64 syms;
1489 __aligned_u64 addrs;1489 __aligned_u64 addrs;
1490 __aligned_u64 cookies;
1490 } kprobe_multi;1491 } kprobe_multi;
1491 };1492 };
1492 } link_create;1493 } link_create;