History

bpf: Separate bpf_get_[stack|stackid] for perf events BPF

Calling get_perf_callchain() on perf_events from PEBS entries may cause unwinder errors. To fix this issue, the callchain is fetched early. Such perf_events are marked with __PERF_SAMPLE_CALLCHAIN_EARLY. Similarly, calling bpf_get_[stack|stackid] on perf_events from PEBS may also cause unwinder errors. To fix this, add separate version of these two helpers, bpf_get_[stack|stackid]_pe. These two hepers use callchain in bpf_perf_event_data_kern->data->callchain.

Song Liu committed 6 years ago

Commit 7b04d6d1 parent 909e4463 files · +170 −20Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +2 −0
  2. kernel/bpf/stackmap.c +166 −18
  3. kernel/trace/bpf_trace.c +2 −2
include/linux/bpf.h+2−0
@@ -1675,6 +1675,8 @@ extern const struct bpf_func_proto bpf_get_current_comm_proto; · 1674 unmodified lines
1675extern const struct bpf_func_proto bpf_get_stackid_proto;1675extern const struct bpf_func_proto bpf_get_stackid_proto;
1676extern const struct bpf_func_proto bpf_get_stack_proto;1676extern const struct bpf_func_proto bpf_get_stack_proto;
1677extern const struct bpf_func_proto bpf_get_task_stack_proto;1677extern const struct bpf_func_proto bpf_get_task_stack_proto;
1678extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
1679extern const struct bpf_func_proto bpf_get_stack_proto_pe;
1678extern const struct bpf_func_proto bpf_sock_map_update_proto;1680extern const struct bpf_func_proto bpf_sock_map_update_proto;
1679extern const struct bpf_func_proto bpf_sock_hash_update_proto;1681extern const struct bpf_func_proto bpf_sock_hash_update_proto;
1680extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;1682extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
kernel/bpf/stackmap.c+166−18
@@ -4,6 +4,7 @@ · 3 unmodified lines
4#include <linux/bpf.h>4#include <linux/bpf.h>
5#include <linux/jhash.h>5#include <linux/jhash.h>
6#include <linux/filter.h>6#include <linux/filter.h>
7#include <linux/kernel.h>
7#include <linux/stacktrace.h>8#include <linux/stacktrace.h>
8#include <linux/perf_event.h>9#include <linux/perf_event.h>
9#include <linux/elf.h>10#include <linux/elf.h>
@@ -387,11 +388,10 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr) · 377 unmodified lines
387#endif388#endif
388}389}
389 390
390BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,391static long __bpf_get_stackid(struct bpf_map *map,
391 u64, flags)392 struct perf_callchain_entry *trace, u64 flags)
392{393{
393 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);394 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
394 struct perf_callchain_entry *trace;
395 struct stack_map_bucket *bucket, *new_bucket, *old_bucket;395 struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
396 u32 max_depth = map->value_size / stack_map_data_size(map);396 u32 max_depth = map->value_size / stack_map_data_size(map);
397 /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */397 /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
@@ -399,21 +399,9 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 1 unmodified line
399 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;399 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
400 u32 hash, id, trace_nr, trace_len;400 u32 hash, id, trace_nr, trace_len;
401 bool user = flags & BPF_F_USER_STACK;401 bool user = flags & BPF_F_USER_STACK;
402 bool kernel = !user;
403 u64 *ips;402 u64 *ips;
404 bool hash_matches;403 bool hash_matches;
405 404
406 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
407 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
408 return -EINVAL;
409
410 trace = get_perf_callchain(regs, init_nr, kernel, user,
411 sysctl_perf_event_max_stack, false, false);
412
413 if (unlikely(!trace))
414 /* couldn't fetch the stack trace */
415 return -EFAULT;
416
417 /* get_perf_callchain() guarantees that trace->nr >= init_nr405 /* get_perf_callchain() guarantees that trace->nr >= init_nr
418 * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth406 * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
419 */407 */
@@ -478,6 +466,30 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 58 unmodified lines
478 return id;466 return id;
479}467}
480 468
469BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
470 u64, flags)
471{
472 u32 max_depth = map->value_size / stack_map_data_size(map);
473 /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
474 u32 init_nr = sysctl_perf_event_max_stack - max_depth;
475 bool user = flags & BPF_F_USER_STACK;
476 struct perf_callchain_entry *trace;
477 bool kernel = !user;
478
479 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
480 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
481 return -EINVAL;
482
483 trace = get_perf_callchain(regs, init_nr, kernel, user,
484 sysctl_perf_event_max_stack, false, false);
485
486 if (unlikely(!trace))
487 /* couldn't fetch the stack trace */
488 return -EFAULT;
489
490 return __bpf_get_stackid(map, trace, flags);
491}
492
481const struct bpf_func_proto bpf_get_stackid_proto = {493const struct bpf_func_proto bpf_get_stackid_proto = {
482 .func = bpf_get_stackid,494 .func = bpf_get_stackid,
483 .gpl_only = true,495 .gpl_only = true,
@@ -487,7 +499,77 @@ const struct bpf_func_proto bpf_get_stackid_proto = { · 3 unmodified lines
487 .arg3_type = ARG_ANYTHING,499 .arg3_type = ARG_ANYTHING,
488};500};
489 501
502static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
503{
504 __u64 nr_kernel = 0;
505
506 while (nr_kernel < trace->nr) {
507 if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
508 break;
509 nr_kernel++;
510 }
511 return nr_kernel;
512}
513
514BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
515 struct bpf_map *, map, u64, flags)
516{
517 struct perf_event *event = ctx->event;
518 struct perf_callchain_entry *trace;
519 bool kernel, user;
520 __u64 nr_kernel;
521 int ret;
522
523 /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
524 if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
525 return bpf_get_stackid((unsigned long)(ctx->regs),
526 (unsigned long) map, flags, 0, 0);
527
528 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
529 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
530 return -EINVAL;
531
532 user = flags & BPF_F_USER_STACK;
533 kernel = !user;
534
535 trace = ctx->data->callchain;
536 if (unlikely(!trace))
537 return -EFAULT;
538
539 nr_kernel = count_kernel_ip(trace);
540
541 if (kernel) {
542 __u64 nr = trace->nr;
543
544 trace->nr = nr_kernel;
545 ret = __bpf_get_stackid(map, trace, flags);
546
547 /* restore nr */
548 trace->nr = nr;
549 } else { /* user */
550 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
551
552 skip += nr_kernel;
553 if (skip > BPF_F_SKIP_FIELD_MASK)
554 return -EFAULT;
555
556 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
557 ret = __bpf_get_stackid(map, trace, flags);
558 }
559 return ret;
560}
561
562const struct bpf_func_proto bpf_get_stackid_proto_pe = {
563 .func = bpf_get_stackid_pe,
564 .gpl_only = false,
565 .ret_type = RET_INTEGER,
566 .arg1_type = ARG_PTR_TO_CTX,
567 .arg2_type = ARG_CONST_MAP_PTR,
568 .arg3_type = ARG_ANYTHING,
569};
570
490static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,571static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
572 struct perf_callchain_entry *trace_in,
491 void *buf, u32 size, u64 flags)573 void *buf, u32 size, u64 flags)
492{574{
493 u32 init_nr, trace_nr, copy_len, elem_size, num_elem;575 u32 init_nr, trace_nr, copy_len, elem_size, num_elem;
@@ -520,7 +602,9 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · 26 unmodified lines
520 else602 else
521 init_nr = sysctl_perf_event_max_stack - num_elem;603 init_nr = sysctl_perf_event_max_stack - num_elem;
522 604
523 if (kernel && task)605 if (trace_in)
606 trace = trace_in;
607 else if (kernel && task)
524 trace = get_callchain_entry_for_task(task, init_nr);608 trace = get_callchain_entry_for_task(task, init_nr);
525 else609 else
526 trace = get_perf_callchain(regs, init_nr, kernel, user,610 trace = get_perf_callchain(regs, init_nr, kernel, user,
@@ -556,7 +640,7 @@ clear: · 29 unmodified lines
556BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,640BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
557 u64, flags)641 u64, flags)
558{642{
559 return __bpf_get_stack(regs, NULL, buf, size, flags);643 return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
560}644}
561 645
562const struct bpf_func_proto bpf_get_stack_proto = {646const struct bpf_func_proto bpf_get_stack_proto = {
@@ -574,7 +658,7 @@ BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf, · 11 unmodified lines
574{658{
575 struct pt_regs *regs = task_pt_regs(task);659 struct pt_regs *regs = task_pt_regs(task);
576 660
577 return __bpf_get_stack(regs, task, buf, size, flags);661 return __bpf_get_stack(regs, task, NULL, buf, size, flags);
578}662}
579 663
580BTF_ID_LIST(bpf_get_task_stack_btf_ids)664BTF_ID_LIST(bpf_get_task_stack_btf_ids)
@@ -591,6 +675,70 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { · 10 unmodified lines
591 .btf_id = bpf_get_task_stack_btf_ids,675 .btf_id = bpf_get_task_stack_btf_ids,
592};676};
593 677
678BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
679 void *, buf, u32, size, u64, flags)
680{
681 struct perf_event *event = ctx->event;
682 struct perf_callchain_entry *trace;
683 bool kernel, user;
684 int err = -EINVAL;
685 __u64 nr_kernel;
686
687 if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
688 return __bpf_get_stack(ctx->regs, NULL, NULL, buf, size, flags);
689
690 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
691 BPF_F_USER_BUILD_ID)))
692 goto clear;
693
694 user = flags & BPF_F_USER_STACK;
695 kernel = !user;
696
697 err = -EFAULT;
698 trace = ctx->data->callchain;
699 if (unlikely(!trace))
700 goto clear;
701
702 nr_kernel = count_kernel_ip(trace);
703
704 if (kernel) {
705 __u64 nr = trace->nr;
706
707 trace->nr = nr_kernel;
708 err = __bpf_get_stack(ctx->regs, NULL, trace, buf,
709 size, flags);
710
711 /* restore nr */
712 trace->nr = nr;
713 } else { /* user */
714 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
715
716 skip += nr_kernel;
717 if (skip > BPF_F_SKIP_FIELD_MASK)
718 goto clear;
719
720 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
721 err = __bpf_get_stack(ctx->regs, NULL, trace, buf,
722 size, flags);
723 }
724 return err;
725
726clear:
727 memset(buf, 0, size);
728 return err;
729
730}
731
732const struct bpf_func_proto bpf_get_stack_proto_pe = {
733 .func = bpf_get_stack_pe,
734 .gpl_only = true,
735 .ret_type = RET_INTEGER,
736 .arg1_type = ARG_PTR_TO_CTX,
737 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
738 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
739 .arg4_type = ARG_ANYTHING,
740};
741
594/* Called from eBPF program */742/* Called from eBPF program */
595static void *stack_map_lookup_elem(struct bpf_map *map, void *key)743static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
596{744{
kernel/trace/bpf_trace.c+2−2
@@ -1411,9 +1411,9 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1410 unmodified lines
1411 case BPF_FUNC_perf_event_output:1411 case BPF_FUNC_perf_event_output:
1412 return &bpf_perf_event_output_proto_tp;1412 return &bpf_perf_event_output_proto_tp;
1413 case BPF_FUNC_get_stackid:1413 case BPF_FUNC_get_stackid:
1414 return &bpf_get_stackid_proto_tp;1414 return &bpf_get_stackid_proto_pe;
1415 case BPF_FUNC_get_stack:1415 case BPF_FUNC_get_stack:
1416 return &bpf_get_stack_proto_tp;1416 return &bpf_get_stack_proto_pe;
1417 case BPF_FUNC_perf_prog_read_value:1417 case BPF_FUNC_perf_prog_read_value:
1418 return &bpf_perf_prog_read_value_proto;1418 return &bpf_perf_prog_read_value_proto;
1419 case BPF_FUNC_read_branch_records:1419 case BPF_FUNC_read_branch_records: