The new stackid_fastpath does the fast stack hash and trace check, that does not need new bucket allocation. It covers both just-ip and buildid code paths.
Unified · Split
| @@ -511,6 +511,7 @@ struct stackid { · 510 unmodified lines | ||
| 511 | 511 | u32 len; |
| 512 | 512 | u32 hash; |
| 513 | 513 | u32 id; |
| 514 | + bool hash_matches; | |
| 514 | 515 | }; |
| 515 | 516 | |
| 516 | 517 | static int stackid_init(struct stackid *stackid, struct bpf_map *map, |
| @@ -531,28 +532,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map, · 14 unmodified lines | ||
| 531 | 532 | stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); |
| 532 | 533 | stackid->id = stackid->hash & (smap->n_buckets - 1); |
| 533 | 534 | stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); |
| 535 | + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; | |
| 534 | 536 | return 0; |
| 535 | 537 | } |
| 536 | 538 | |
| 539 | +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, | |
| 540 | + struct perf_callchain_entry *trace, u64 flags) | |
| 541 | +{ | |
| 542 | + int err; | |
| 543 | + | |
| 544 | + err = stackid_init(stackid, map, trace, flags); | |
| 545 | + if (err) | |
| 546 | + return err; | |
| 547 | + | |
| 548 | + /* fast cmp */ | |
| 549 | + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) | |
| 550 | + return stackid->id; | |
| 551 | + | |
| 552 | + if (stack_map_use_build_id(map)) | |
| 553 | + return -ENOENT; | |
| 554 | + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && | |
| 555 | + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) | |
| 556 | + return stackid->id; | |
| 557 | + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) | |
| 558 | + return -EEXIST; | |
| 559 | + return -ENOENT; | |
| 560 | +} | |
| 561 | + | |
| 537 | 562 | static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, |
| 538 | 563 | struct perf_callchain_entry *trace, u64 flags) |
| 539 | 564 | { |
| 540 | 565 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); |
| 541 | 566 | struct stack_map_bucket *new_bucket, *old_bucket; |
| 542 | 567 | bool user = flags & BPF_F_USER_STACK; |
| 543 | − bool hash_matches; | |
| 544 | 568 | u32 trace_len, i; |
| 545 | 569 | int err; |
| 546 | 570 | |
| 547 | − err = stackid_init(stackid, map, trace, flags); | |
| 548 | − if (err) | |
| 571 | + err = stackid_fastpath(stackid, map, trace, flags); | |
| 572 | + if (err != -ENOENT) | |
| 549 | 573 | return err; |
| 550 | 574 | |
| 551 | − hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; | |
| 552 | − /* fast cmp */ | |
| 553 | − if (hash_matches && flags & BPF_F_FAST_STACK_CMP) | |
| 554 | − return stackid->id; | |
| 555 | − | |
| 556 | 575 | if (stack_map_use_build_id(map)) { |
| 557 | 576 | struct bpf_stack_build_id *id_offs; |
| 558 | 577 | |
| @@ -567,7 +586,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, · 8 unmodified lines | ||
| 567 | 586 | id_offs[i].ip = stackid->ips[i]; |
| 568 | 587 | stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); |
| 569 | 588 | trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); |
| 570 | − if (hash_matches && stackid->bucket->nr == stackid->nr && | |
| 589 | + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && | |
| 571 | 590 | memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { |
| 572 | 591 | pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); |
| 573 | 592 | return stackid->id; |
| @@ -577,12 +596,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, · 3 unmodified lines | ||
| 577 | 596 | return -EEXIST; |
| 578 | 597 | } |
| 579 | 598 | } else { |
| 580 | − if (hash_matches && stackid->bucket->nr == stackid->nr && | |
| 581 | − memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) | |
| 582 | − return stackid->id; | |
| 583 | − if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) | |
| 584 | − return -EEXIST; | |
| 585 | − | |
| 586 | 599 | new_bucket = (struct stack_map_bucket *) |
| 587 | 600 | pcpu_freelist_pop(&smap->freelist); |
| 588 | 601 | if (unlikely(!new_bucket)) |