Calling get_perf_callchain() on perf_events from PEBS entries may cause unwinder errors. To fix this issue, the callchain is fetched early. Such perf_events are marked with __PERF_SAMPLE_CALLCHAIN_EARLY. Similarly, calling bpf_get_[stack|stackid] on perf_events from PEBS may also cause unwinder errors. To fix this, add separate version of these two helpers, bpf_get_[stack|stackid]_pe. These two hepers use callchain in bpf_perf_event_data_kern->data->callchain.
Showing the changes to kernel (2 of 3 files). Show all
Unified · Split
| @@ -4,6 +4,7 @@ · 3 unmodified lines | |||
| 4 | #include <linux/bpf.h> | 4 | #include <linux/bpf.h> |
| 5 | #include <linux/jhash.h> | 5 | #include <linux/jhash.h> |
| 6 | #include <linux/filter.h> | 6 | #include <linux/filter.h> |
| 7 | #include <linux/kernel.h> | ||
| 7 | #include <linux/stacktrace.h> | 8 | #include <linux/stacktrace.h> |
| 8 | #include <linux/perf_event.h> | 9 | #include <linux/perf_event.h> |
| 9 | #include <linux/elf.h> | 10 | #include <linux/elf.h> |
| @@ -387,11 +388,10 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr) · 377 unmodified lines | |||
| 387 | #endif | 388 | #endif |
| 388 | } | 389 | } |
| 389 | 390 | ||
| 390 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, | 391 | static long __bpf_get_stackid(struct bpf_map *map, |
| 391 | u64, flags) | 392 | struct perf_callchain_entry *trace, u64 flags) |
| 392 | { | 393 | { |
| 393 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); | 394 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); |
| 394 | struct perf_callchain_entry *trace; | ||
| 395 | struct stack_map_bucket *bucket, *new_bucket, *old_bucket; | 395 | struct stack_map_bucket *bucket, *new_bucket, *old_bucket; |
| 396 | u32 max_depth = map->value_size / stack_map_data_size(map); | 396 | u32 max_depth = map->value_size / stack_map_data_size(map); |
| 397 | /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */ | 397 | /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */ |
| @@ -399,21 +399,9 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 1 unmodified line | |||
| 399 | u32 skip = flags & BPF_F_SKIP_FIELD_MASK; | 399 | u32 skip = flags & BPF_F_SKIP_FIELD_MASK; |
| 400 | u32 hash, id, trace_nr, trace_len; | 400 | u32 hash, id, trace_nr, trace_len; |
| 401 | bool user = flags & BPF_F_USER_STACK; | 401 | bool user = flags & BPF_F_USER_STACK; |
| 402 | bool kernel = !user; | ||
| 403 | u64 *ips; | 402 | u64 *ips; |
| 404 | bool hash_matches; | 403 | bool hash_matches; |
| 405 | 404 | ||
| 406 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | | ||
| 407 | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) | ||
| 408 | return -EINVAL; | ||
| 409 | |||
| 410 | trace = get_perf_callchain(regs, init_nr, kernel, user, | ||
| 411 | sysctl_perf_event_max_stack, false, false); | ||
| 412 | |||
| 413 | if (unlikely(!trace)) | ||
| 414 | /* couldn't fetch the stack trace */ | ||
| 415 | return -EFAULT; | ||
| 416 | |||
| 417 | /* get_perf_callchain() guarantees that trace->nr >= init_nr | 405 | /* get_perf_callchain() guarantees that trace->nr >= init_nr |
| 418 | * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth | 406 | * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth |
| 419 | */ | 407 | */ |
| @@ -478,6 +466,30 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, · 58 unmodified lines | |||
| 478 | return id; | 466 | return id; |
| 479 | } | 467 | } |
| 480 | 468 | ||
| 469 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, | ||
| 470 | u64, flags) | ||
| 471 | { | ||
| 472 | u32 max_depth = map->value_size / stack_map_data_size(map); | ||
| 473 | /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */ | ||
| 474 | u32 init_nr = sysctl_perf_event_max_stack - max_depth; | ||
| 475 | bool user = flags & BPF_F_USER_STACK; | ||
| 476 | struct perf_callchain_entry *trace; | ||
| 477 | bool kernel = !user; | ||
| 478 | |||
| 479 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | | ||
| 480 | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) | ||
| 481 | return -EINVAL; | ||
| 482 | |||
| 483 | trace = get_perf_callchain(regs, init_nr, kernel, user, | ||
| 484 | sysctl_perf_event_max_stack, false, false); | ||
| 485 | |||
| 486 | if (unlikely(!trace)) | ||
| 487 | /* couldn't fetch the stack trace */ | ||
| 488 | return -EFAULT; | ||
| 489 | |||
| 490 | return __bpf_get_stackid(map, trace, flags); | ||
| 491 | } | ||
| 492 | |||
| 481 | const struct bpf_func_proto bpf_get_stackid_proto = { | 493 | const struct bpf_func_proto bpf_get_stackid_proto = { |
| 482 | .func = bpf_get_stackid, | 494 | .func = bpf_get_stackid, |
| 483 | .gpl_only = true, | 495 | .gpl_only = true, |
| @@ -487,7 +499,77 @@ const struct bpf_func_proto bpf_get_stackid_proto = { · 3 unmodified lines | |||
| 487 | .arg3_type = ARG_ANYTHING, | 499 | .arg3_type = ARG_ANYTHING, |
| 488 | }; | 500 | }; |
| 489 | 501 | ||
| 502 | static __u64 count_kernel_ip(struct perf_callchain_entry *trace) | ||
| 503 | { | ||
| 504 | __u64 nr_kernel = 0; | ||
| 505 | |||
| 506 | while (nr_kernel < trace->nr) { | ||
| 507 | if (trace->ip[nr_kernel] == PERF_CONTEXT_USER) | ||
| 508 | break; | ||
| 509 | nr_kernel++; | ||
| 510 | } | ||
| 511 | return nr_kernel; | ||
| 512 | } | ||
| 513 | |||
| 514 | BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, | ||
| 515 | struct bpf_map *, map, u64, flags) | ||
| 516 | { | ||
| 517 | struct perf_event *event = ctx->event; | ||
| 518 | struct perf_callchain_entry *trace; | ||
| 519 | bool kernel, user; | ||
| 520 | __u64 nr_kernel; | ||
| 521 | int ret; | ||
| 522 | |||
| 523 | /* perf_sample_data doesn't have callchain, use bpf_get_stackid */ | ||
| 524 | if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) | ||
| 525 | return bpf_get_stackid((unsigned long)(ctx->regs), | ||
| 526 | (unsigned long) map, flags, 0, 0); | ||
| 527 | |||
| 528 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | | ||
| 529 | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) | ||
| 530 | return -EINVAL; | ||
| 531 | |||
| 532 | user = flags & BPF_F_USER_STACK; | ||
| 533 | kernel = !user; | ||
| 534 | |||
| 535 | trace = ctx->data->callchain; | ||
| 536 | if (unlikely(!trace)) | ||
| 537 | return -EFAULT; | ||
| 538 | |||
| 539 | nr_kernel = count_kernel_ip(trace); | ||
| 540 | |||
| 541 | if (kernel) { | ||
| 542 | __u64 nr = trace->nr; | ||
| 543 | |||
| 544 | trace->nr = nr_kernel; | ||
| 545 | ret = __bpf_get_stackid(map, trace, flags); | ||
| 546 | |||
| 547 | /* restore nr */ | ||
| 548 | trace->nr = nr; | ||
| 549 | } else { /* user */ | ||
| 550 | u64 skip = flags & BPF_F_SKIP_FIELD_MASK; | ||
| 551 | |||
| 552 | skip += nr_kernel; | ||
| 553 | if (skip > BPF_F_SKIP_FIELD_MASK) | ||
| 554 | return -EFAULT; | ||
| 555 | |||
| 556 | flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; | ||
| 557 | ret = __bpf_get_stackid(map, trace, flags); | ||
| 558 | } | ||
| 559 | return ret; | ||
| 560 | } | ||
| 561 | |||
| 562 | const struct bpf_func_proto bpf_get_stackid_proto_pe = { | ||
| 563 | .func = bpf_get_stackid_pe, | ||
| 564 | .gpl_only = false, | ||
| 565 | .ret_type = RET_INTEGER, | ||
| 566 | .arg1_type = ARG_PTR_TO_CTX, | ||
| 567 | .arg2_type = ARG_CONST_MAP_PTR, | ||
| 568 | .arg3_type = ARG_ANYTHING, | ||
| 569 | }; | ||
| 570 | |||
| 490 | static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, | 571 | static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, |
| 572 | struct perf_callchain_entry *trace_in, | ||
| 491 | void *buf, u32 size, u64 flags) | 573 | void *buf, u32 size, u64 flags) |
| 492 | { | 574 | { |
| 493 | u32 init_nr, trace_nr, copy_len, elem_size, num_elem; | 575 | u32 init_nr, trace_nr, copy_len, elem_size, num_elem; |
| @@ -520,7 +602,9 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · 26 unmodified lines | |||
| 520 | else | 602 | else |
| 521 | init_nr = sysctl_perf_event_max_stack - num_elem; | 603 | init_nr = sysctl_perf_event_max_stack - num_elem; |
| 522 | 604 | ||
| 523 | if (kernel && task) | 605 | if (trace_in) |
| 606 | trace = trace_in; | ||
| 607 | else if (kernel && task) | ||
| 524 | trace = get_callchain_entry_for_task(task, init_nr); | 608 | trace = get_callchain_entry_for_task(task, init_nr); |
| 525 | else | 609 | else |
| 526 | trace = get_perf_callchain(regs, init_nr, kernel, user, | 610 | trace = get_perf_callchain(regs, init_nr, kernel, user, |
| @@ -556,7 +640,7 @@ clear: · 29 unmodified lines | |||
| 556 | BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, | 640 | BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, |
| 557 | u64, flags) | 641 | u64, flags) |
| 558 | { | 642 | { |
| 559 | return __bpf_get_stack(regs, NULL, buf, size, flags); | 643 | return __bpf_get_stack(regs, NULL, NULL, buf, size, flags); |
| 560 | } | 644 | } |
| 561 | 645 | ||
| 562 | const struct bpf_func_proto bpf_get_stack_proto = { | 646 | const struct bpf_func_proto bpf_get_stack_proto = { |
| @@ -574,7 +658,7 @@ BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf, · 11 unmodified lines | |||
| 574 | { | 658 | { |
| 575 | struct pt_regs *regs = task_pt_regs(task); | 659 | struct pt_regs *regs = task_pt_regs(task); |
| 576 | 660 | ||
| 577 | return __bpf_get_stack(regs, task, buf, size, flags); | 661 | return __bpf_get_stack(regs, task, NULL, buf, size, flags); |
| 578 | } | 662 | } |
| 579 | 663 | ||
| 580 | BTF_ID_LIST(bpf_get_task_stack_btf_ids) | 664 | BTF_ID_LIST(bpf_get_task_stack_btf_ids) |
| @@ -591,6 +675,70 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { · 10 unmodified lines | |||
| 591 | .btf_id = bpf_get_task_stack_btf_ids, | 675 | .btf_id = bpf_get_task_stack_btf_ids, |
| 592 | }; | 676 | }; |
| 593 | 677 | ||
| 678 | BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, | ||
| 679 | void *, buf, u32, size, u64, flags) | ||
| 680 | { | ||
| 681 | struct perf_event *event = ctx->event; | ||
| 682 | struct perf_callchain_entry *trace; | ||
| 683 | bool kernel, user; | ||
| 684 | int err = -EINVAL; | ||
| 685 | __u64 nr_kernel; | ||
| 686 | |||
| 687 | if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) | ||
| 688 | return __bpf_get_stack(ctx->regs, NULL, NULL, buf, size, flags); | ||
| 689 | |||
| 690 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | | ||
| 691 | BPF_F_USER_BUILD_ID))) | ||
| 692 | goto clear; | ||
| 693 | |||
| 694 | user = flags & BPF_F_USER_STACK; | ||
| 695 | kernel = !user; | ||
| 696 | |||
| 697 | err = -EFAULT; | ||
| 698 | trace = ctx->data->callchain; | ||
| 699 | if (unlikely(!trace)) | ||
| 700 | goto clear; | ||
| 701 | |||
| 702 | nr_kernel = count_kernel_ip(trace); | ||
| 703 | |||
| 704 | if (kernel) { | ||
| 705 | __u64 nr = trace->nr; | ||
| 706 | |||
| 707 | trace->nr = nr_kernel; | ||
| 708 | err = __bpf_get_stack(ctx->regs, NULL, trace, buf, | ||
| 709 | size, flags); | ||
| 710 | |||
| 711 | /* restore nr */ | ||
| 712 | trace->nr = nr; | ||
| 713 | } else { /* user */ | ||
| 714 | u64 skip = flags & BPF_F_SKIP_FIELD_MASK; | ||
| 715 | |||
| 716 | skip += nr_kernel; | ||
| 717 | if (skip > BPF_F_SKIP_FIELD_MASK) | ||
| 718 | goto clear; | ||
| 719 | |||
| 720 | flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; | ||
| 721 | err = __bpf_get_stack(ctx->regs, NULL, trace, buf, | ||
| 722 | size, flags); | ||
| 723 | } | ||
| 724 | return err; | ||
| 725 | |||
| 726 | clear: | ||
| 727 | memset(buf, 0, size); | ||
| 728 | return err; | ||
| 729 | |||
| 730 | } | ||
| 731 | |||
| 732 | const struct bpf_func_proto bpf_get_stack_proto_pe = { | ||
| 733 | .func = bpf_get_stack_pe, | ||
| 734 | .gpl_only = true, | ||
| 735 | .ret_type = RET_INTEGER, | ||
| 736 | .arg1_type = ARG_PTR_TO_CTX, | ||
| 737 | .arg2_type = ARG_PTR_TO_UNINIT_MEM, | ||
| 738 | .arg3_type = ARG_CONST_SIZE_OR_ZERO, | ||
| 739 | .arg4_type = ARG_ANYTHING, | ||
| 740 | }; | ||
| 741 | |||
| 594 | /* Called from eBPF program */ | 742 | /* Called from eBPF program */ |
| 595 | static void *stack_map_lookup_elem(struct bpf_map *map, void *key) | 743 | static void *stack_map_lookup_elem(struct bpf_map *map, void *key) |
| 596 | { | 744 | { |
| @@ -1411,9 +1411,9 @@ pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1410 unmodified lines | |||
| 1411 | case BPF_FUNC_perf_event_output: | 1411 | case BPF_FUNC_perf_event_output: |
| 1412 | return &bpf_perf_event_output_proto_tp; | 1412 | return &bpf_perf_event_output_proto_tp; |
| 1413 | case BPF_FUNC_get_stackid: | 1413 | case BPF_FUNC_get_stackid: |
| 1414 | return &bpf_get_stackid_proto_tp; | 1414 | return &bpf_get_stackid_proto_pe; |
| 1415 | case BPF_FUNC_get_stack: | 1415 | case BPF_FUNC_get_stack: |
| 1416 | return &bpf_get_stack_proto_tp; | 1416 | return &bpf_get_stack_proto_pe; |
| 1417 | case BPF_FUNC_perf_prog_read_value: | 1417 | case BPF_FUNC_perf_prog_read_value: |
| 1418 | return &bpf_perf_prog_read_value_proto; | 1418 | return &bpf_perf_prog_read_value_proto; |
| 1419 | case BPF_FUNC_read_branch_records: | 1419 | case BPF_FUNC_read_branch_records: |