Separating /proc/kallsyms add/del code and adding bpf_ksym_add/del functions for that. Moving bpf_prog_ksym_node_add/del functions to __bpf_ksym_add/del and changing their argument to 'struct bpf_ksym' object. This way we can call them for other bpf objects types like trampoline and dispatcher.
Unified · Split
| @@ -584,6 +584,9 @@ struct bpf_image { · 583 unmodified lines | ||
| 584 | 584 | #define BPF_IMAGE_SIZE (PAGE_SIZE - sizeof(struct bpf_image)) |
| 585 | 585 | bool is_bpf_image_address(unsigned long address); |
| 586 | 586 | void *bpf_image_alloc(void); |
| 587 | +/* Called only from JIT-enabled code, so there's no need for stubs. */ | |
| 588 | +void bpf_ksym_add(struct bpf_ksym *ksym); | |
| 589 | +void bpf_ksym_del(struct bpf_ksym *ksym); | |
| 587 | 590 | #else |
| 588 | 591 | static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key) |
| 589 | 592 | { |
| @@ -607,20 +607,29 @@ static DEFINE_SPINLOCK(bpf_lock); · 606 unmodified lines | ||
| 607 | 607 | static LIST_HEAD(bpf_kallsyms); |
| 608 | 608 | static struct latch_tree_root bpf_tree __cacheline_aligned; |
| 609 | 609 | |
| 610 | −static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) | |
| 610 | +void bpf_ksym_add(struct bpf_ksym *ksym) | |
| 611 | 611 | { |
| 612 | − WARN_ON_ONCE(!list_empty(&aux->ksym.lnode)); | |
| 613 | − list_add_tail_rcu(&aux->ksym.lnode, &bpf_kallsyms); | |
| 614 | − latch_tree_insert(&aux->ksym.tnode, &bpf_tree, &bpf_tree_ops); | |
| 612 | + spin_lock_bh(&bpf_lock); | |
| 613 | + WARN_ON_ONCE(!list_empty(&ksym->lnode)); | |
| 614 | + list_add_tail_rcu(&ksym->lnode, &bpf_kallsyms); | |
| 615 | + latch_tree_insert(&ksym->tnode, &bpf_tree, &bpf_tree_ops); | |
| 616 | + spin_unlock_bh(&bpf_lock); | |
| 615 | 617 | } |
| 616 | 618 | |
| 617 | −static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux) | |
| 619 | +static void __bpf_ksym_del(struct bpf_ksym *ksym) | |
| 618 | 620 | { |
| 619 | − if (list_empty(&aux->ksym.lnode)) | |
| 621 | + if (list_empty(&ksym->lnode)) | |
| 620 | 622 | return; |
| 621 | 623 | |
| 622 | − latch_tree_erase(&aux->ksym.tnode, &bpf_tree, &bpf_tree_ops); | |
| 623 | − list_del_rcu(&aux->ksym.lnode); | |
| 624 | + latch_tree_erase(&ksym->tnode, &bpf_tree, &bpf_tree_ops); | |
| 625 | + list_del_rcu(&ksym->lnode); | |
| 626 | +} | |
| 627 | + | |
| 628 | +void bpf_ksym_del(struct bpf_ksym *ksym) | |
| 629 | +{ | |
| 630 | + spin_lock_bh(&bpf_lock); | |
| 631 | + __bpf_ksym_del(ksym); | |
| 632 | + spin_unlock_bh(&bpf_lock); | |
| 624 | 633 | } |
| 625 | 634 | |
| 626 | 635 | static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) |
| @@ -644,9 +653,7 @@ void bpf_prog_kallsyms_add(struct bpf_prog *fp) · 17 unmodified lines | ||
| 644 | 653 | bpf_prog_ksym_set_name(fp); |
| 645 | 654 | fp->aux->ksym.prog = true; |
| 646 | 655 | |
| 647 | − spin_lock_bh(&bpf_lock); | |
| 648 | − bpf_prog_ksym_node_add(fp->aux); | |
| 649 | − spin_unlock_bh(&bpf_lock); | |
| 656 | + bpf_ksym_add(&fp->aux->ksym); | |
| 650 | 657 | } |
| 651 | 658 | |
| 652 | 659 | void bpf_prog_kallsyms_del(struct bpf_prog *fp) |
| @@ -654,9 +661,7 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp) · 1 unmodified line | ||
| 654 | 661 | if (!bpf_prog_kallsyms_candidate(fp)) |
| 655 | 662 | return; |
| 656 | 663 | |
| 657 | − spin_lock_bh(&bpf_lock); | |
| 658 | − bpf_prog_ksym_node_del(fp->aux); | |
| 659 | − spin_unlock_bh(&bpf_lock); | |
| 664 | + bpf_ksym_del(&fp->aux->ksym); | |
| 660 | 665 | } |
| 661 | 666 | |
| 662 | 667 | static struct bpf_ksym *bpf_ksym_find(unsigned long addr) |