History

bpf: Separate bpf_get_[stack|stackid] for perf events BPF

Calling get_perf_callchain() on perf_events from PEBS entries may cause unwinder errors. To fix this issue, the callchain is fetched early. Such perf_events are marked with __PERF_SAMPLE_CALLCHAIN_EARLY. Similarly, calling bpf_get_[stack|stackid] on perf_events from PEBS may also cause unwinder errors. To fix this, add separate version of these two helpers, bpf_get_[stack|stackid]_pe. These two hepers use callchain in bpf_perf_event_data_kern->data->callchain.

Song Liu committed 6 years ago

Commit 7b04d6d1 parent 909e4463 files · +170 −20Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +2 −0
  2. kernel/bpf/stackmap.c +166 −18
  3. kernel/trace/bpf_trace.c +2 −2
include/linux/bpf.h+2−0
@@ -1675,6 +1675,8 @@ extern const struct bpf_func_proto bpf_get_current_comm_proto; · 1674 unmodified lines
16751675 extern const struct bpf_func_proto bpf_get_stackid_proto;
16761676 extern const struct bpf_func_proto bpf_get_stack_proto;
16771677 extern const struct bpf_func_proto bpf_get_task_stack_proto;
1678+extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
1679+extern const struct bpf_func_proto bpf_get_stack_proto_pe;
16781680 extern const struct bpf_func_proto bpf_sock_map_update_proto;
16791681 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
16801682 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
kernel/bpf/stackmap.c+166−18
@@ -4,6 +4,7 @@ · 3 unmodified lines
44 #include <linux/bpf.h>
55 #include <linux/jhash.h>
66 #include <linux/filter.h>
7+#include <linux/kernel.h>
78 #include <linux/stacktrace.h>
89 #include <linux/perf_event.h>
910 #include <linux/elf.h>
@@ -387,11 +388,10 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr) · 377 unmodified lines
387388 #endif
388389 }
389390
390−BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
391− u64, flags)
391+static long __bpf_get_stackid(struct bpf_map *map,
392+ struct perf_callchain_entry *trace, u64 flags)
392393 {
393394 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
394− struct perf_callchain_entry *trace;
395395 struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
396396 u32 max_depth = map->value_size / stack_map_data_size(map);
397397 /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
@@ -399,21 +399,9 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 1 unmodified line
399399 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
400400 u32 hash, id, trace_nr, trace_len;
401401 bool user = flags & BPF_F_USER_STACK;
402− bool kernel = !user;
403402 u64 *ips;
404403 bool hash_matches;
405404
406− if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
407− BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
408− return -EINVAL;
409−
410− trace = get_perf_callchain(regs, init_nr, kernel, user,
411− sysctl_perf_event_max_stack, false, false);
412−
413− if (unlikely(!trace))
414− /* couldn't fetch the stack trace */
415− return -EFAULT;
416−
417405 /* get_perf_callchain() guarantees that trace->nr >= init_nr
418406 * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
419407 */
@@ -478,6 +466,30 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 58 unmodified lines
478466 return id;
479467 }
480468
469+BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
470+ u64, flags)
471+{
472+ u32 max_depth = map->value_size / stack_map_data_size(map);
473+ /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
474+ u32 init_nr = sysctl_perf_event_max_stack - max_depth;
475+ bool user = flags & BPF_F_USER_STACK;
476+ struct perf_callchain_entry *trace;
477+ bool kernel = !user;
478+
479+ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
480+ BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
481+ return -EINVAL;
482+
483+ trace = get_perf_callchain(regs, init_nr, kernel, user,
484+ sysctl_perf_event_max_stack, false, false);
485+
486+ if (unlikely(!trace))
487+ /* couldn't fetch the stack trace */
488+ return -EFAULT;
489+
490+ return __bpf_get_stackid(map, trace, flags);
491+}
492+
481493 const struct bpf_func_proto bpf_get_stackid_proto = {
482494 .func = bpf_get_stackid,
483495 .gpl_only = true,
@@ -487,7 +499,77 @@ const struct bpf_func_proto bpf_get_stackid_proto = { · 3 unmodified lines
487499 .arg3_type = ARG_ANYTHING,
488500 };
489501
502+static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
503+{
504+ __u64 nr_kernel = 0;
505+
506+ while (nr_kernel < trace->nr) {
507+ if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
508+ break;
509+ nr_kernel++;
510+ }
511+ return nr_kernel;
512+}
513+
514+BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
515+ struct bpf_map *, map, u64, flags)
516+{
517+ struct perf_event *event = ctx->event;
518+ struct perf_callchain_entry *trace;
519+ bool kernel, user;
520+ __u64 nr_kernel;
521+ int ret;
522+
523+ /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
524+ if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
525+ return bpf_get_stackid((unsigned long)(ctx->regs),
526+ (unsigned long) map, flags, 0, 0);
527+
528+ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
529+ BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
530+ return -EINVAL;
531+
532+ user = flags & BPF_F_USER_STACK;
533+ kernel = !user;
534+
535+ trace = ctx->data->callchain;
536+ if (unlikely(!trace))
537+ return -EFAULT;
538+
539+ nr_kernel = count_kernel_ip(trace);
540+
541+ if (kernel) {
542+ __u64 nr = trace->nr;
543+
544+ trace->nr = nr_kernel;
545+ ret = __bpf_get_stackid(map, trace, flags);
546+
547+ /* restore nr */
548+ trace->nr = nr;
549+ } else { /* user */
550+ u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
551+
552+ skip += nr_kernel;
553+ if (skip > BPF_F_SKIP_FIELD_MASK)
554+ return -EFAULT;
555+
556+ flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
557+ ret = __bpf_get_stackid(map, trace, flags);
558+ }
559+ return ret;
560+}
561+
562+const struct bpf_func_proto bpf_get_stackid_proto_pe = {
563+ .func = bpf_get_stackid_pe,
564+ .gpl_only = false,
565+ .ret_type = RET_INTEGER,
566+ .arg1_type = ARG_PTR_TO_CTX,
567+ .arg2_type = ARG_CONST_MAP_PTR,
568+ .arg3_type = ARG_ANYTHING,
569+};
570+
490571 static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
572+ struct perf_callchain_entry *trace_in,
491573 void *buf, u32 size, u64 flags)
492574 {
493575 u32 init_nr, trace_nr, copy_len, elem_size, num_elem;
@@ -520,7 +602,9 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · 26 unmodified lines
520602 else
521603 init_nr = sysctl_perf_event_max_stack - num_elem;
522604
523− if (kernel && task)
605+ if (trace_in)
606+ trace = trace_in;
607+ else if (kernel && task)
524608 trace = get_callchain_entry_for_task(task, init_nr);
525609 else
526610 trace = get_perf_callchain(regs, init_nr, kernel, user,
@@ -556,7 +640,7 @@ clear: · 29 unmodified lines
556640 BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
557641 u64, flags)
558642 {
559− return __bpf_get_stack(regs, NULL, buf, size, flags);
643+ return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
560644 }
561645
562646 const struct bpf_func_proto bpf_get_stack_proto = {
@@ -574,7 +658,7 @@ BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf, · 11 unmodified lines
574658 {
575659 struct pt_regs *regs = task_pt_regs(task);
576660
577− return __bpf_get_stack(regs, task, buf, size, flags);
661+ return __bpf_get_stack(regs, task, NULL, buf, size, flags);
578662 }
579663
580664 BTF_ID_LIST(bpf_get_task_stack_btf_ids)
@@ -591,6 +675,70 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { · 10 unmodified lines
591675 .btf_id = bpf_get_task_stack_btf_ids,
592676 };
593677
678+BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
679+ void *, buf, u32, size, u64, flags)
680+{
681+ struct perf_event *event = ctx->event;
682+ struct perf_callchain_entry *trace;
683+ bool kernel, user;
684+ int err = -EINVAL;
685+ __u64 nr_kernel;
686+
687+ if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY))
688+ return __bpf_get_stack(ctx->regs, NULL, NULL, buf, size, flags);
689+
690+ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
691+ BPF_F_USER_BUILD_ID)))
692+ goto clear;
693+
694+ user = flags & BPF_F_USER_STACK;
695+ kernel = !user;
696+
697+ err = -EFAULT;
698+ trace = ctx->data->callchain;
699+ if (unlikely(!trace))
700+ goto clear;
701+
702+ nr_kernel = count_kernel_ip(trace);
703+
704+ if (kernel) {
705+ __u64 nr = trace->nr;
706+
707+ trace->nr = nr_kernel;
708+ err = __bpf_get_stack(ctx->regs, NULL, trace, buf,
709+ size, flags);
710+
711+ /* restore nr */
712+ trace->nr = nr;
713+ } else { /* user */
714+ u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
715+
716+ skip += nr_kernel;
717+ if (skip > BPF_F_SKIP_FIELD_MASK)
718+ goto clear;
719+
720+ flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
721+ err = __bpf_get_stack(ctx->regs, NULL, trace, buf,
722+ size, flags);
723+ }
724+ return err;
725+
726+clear:
727+ memset(buf, 0, size);
728+ return err;
729+
730+}
731+
732+const struct bpf_func_proto bpf_get_stack_proto_pe = {
733+ .func = bpf_get_stack_pe,
734+ .gpl_only = true,
735+ .ret_type = RET_INTEGER,
736+ .arg1_type = ARG_PTR_TO_CTX,
737+ .arg2_type = ARG_PTR_TO_UNINIT_MEM,
738+ .arg3_type = ARG_CONST_SIZE_OR_ZERO,
739+ .arg4_type = ARG_ANYTHING,
740+};
741+
594742 /* Called from eBPF program */
595743 static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
596744 {
kernel/trace/bpf_trace.c+2−2
@@ -1411,9 +1411,9 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1410 unmodified lines
14111411 case BPF_FUNC_perf_event_output:
14121412 return &bpf_perf_event_output_proto_tp;
14131413 case BPF_FUNC_get_stackid:
1414− return &bpf_get_stackid_proto_tp;
1414+ return &bpf_get_stackid_proto_pe;
14151415 case BPF_FUNC_get_stack:
1416− return &bpf_get_stack_proto_tp;
1416+ return &bpf_get_stack_proto_pe;
14171417 case BPF_FUNC_perf_prog_read_value:
14181418 return &bpf_perf_prog_read_value_proto;
14191419 case BPF_FUNC_read_branch_records: