To avoid adding forward declarations in the main patch, shuffle some code around. No functional changes.
Unified · Split
| @@ -74,6 +74,121 @@ bpf_offload_find_netdev(struct net_device *netdev) · 73 unmodified lines | ||
| 74 | 74 | return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); |
| 75 | 75 | } |
| 76 | 76 | |
| 77 | +static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, | |
| 78 | + struct net_device *netdev) | |
| 79 | +{ | |
| 80 | + struct bpf_offload_netdev *ondev; | |
| 81 | + int err; | |
| 82 | + | |
| 83 | + ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); | |
| 84 | + if (!ondev) | |
| 85 | + return -ENOMEM; | |
| 86 | + | |
| 87 | + ondev->netdev = netdev; | |
| 88 | + ondev->offdev = offdev; | |
| 89 | + INIT_LIST_HEAD(&ondev->progs); | |
| 90 | + INIT_LIST_HEAD(&ondev->maps); | |
| 91 | + | |
| 92 | + down_write(&bpf_devs_lock); | |
| 93 | + err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); | |
| 94 | + if (err) { | |
| 95 | + netdev_warn(netdev, "failed to register for BPF offload\n"); | |
| 96 | + goto err_unlock_free; | |
| 97 | + } | |
| 98 | + | |
| 99 | + list_add(&ondev->offdev_netdevs, &offdev->netdevs); | |
| 100 | + up_write(&bpf_devs_lock); | |
| 101 | + return 0; | |
| 102 | + | |
| 103 | +err_unlock_free: | |
| 104 | + up_write(&bpf_devs_lock); | |
| 105 | + kfree(ondev); | |
| 106 | + return err; | |
| 107 | +} | |
| 108 | + | |
| 109 | +static void __bpf_prog_offload_destroy(struct bpf_prog *prog) | |
| 110 | +{ | |
| 111 | + struct bpf_prog_offload *offload = prog->aux->offload; | |
| 112 | + | |
| 113 | + if (offload->dev_state) | |
| 114 | + offload->offdev->ops->destroy(prog); | |
| 115 | + | |
| 116 | + /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ | |
| 117 | + bpf_prog_free_id(prog, true); | |
| 118 | + | |
| 119 | + list_del_init(&offload->offloads); | |
| 120 | + kfree(offload); | |
| 121 | + prog->aux->offload = NULL; | |
| 122 | +} | |
| 123 | + | |
| 124 | +static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, | |
| 125 | + enum bpf_netdev_command cmd) | |
| 126 | +{ | |
| 127 | + struct netdev_bpf data = {}; | |
| 128 | + struct net_device *netdev; | |
| 129 | + | |
| 130 | + ASSERT_RTNL(); | |
| 131 | + | |
| 132 | + data.command = cmd; | |
| 133 | + data.offmap = offmap; | |
| 134 | + /* Caller must make sure netdev is valid */ | |
| 135 | + netdev = offmap->netdev; | |
| 136 | + | |
| 137 | + return netdev->netdev_ops->ndo_bpf(netdev, &data); | |
| 138 | +} | |
| 139 | + | |
| 140 | +static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) | |
| 141 | +{ | |
| 142 | + WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); | |
| 143 | + /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ | |
| 144 | + bpf_map_free_id(&offmap->map, true); | |
| 145 | + list_del_init(&offmap->offloads); | |
| 146 | + offmap->netdev = NULL; | |
| 147 | +} | |
| 148 | + | |
| 149 | +static void __bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, | |
| 150 | + struct net_device *netdev) | |
| 151 | +{ | |
| 152 | + struct bpf_offload_netdev *ondev, *altdev; | |
| 153 | + struct bpf_offloaded_map *offmap, *mtmp; | |
| 154 | + struct bpf_prog_offload *offload, *ptmp; | |
| 155 | + | |
| 156 | + ASSERT_RTNL(); | |
| 157 | + | |
| 158 | + down_write(&bpf_devs_lock); | |
| 159 | + ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
| 160 | + if (WARN_ON(!ondev)) | |
| 161 | + goto unlock; | |
| 162 | + | |
| 163 | + WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); | |
| 164 | + list_del(&ondev->offdev_netdevs); | |
| 165 | + | |
| 166 | + /* Try to move the objects to another netdev of the device */ | |
| 167 | + altdev = list_first_entry_or_null(&offdev->netdevs, | |
| 168 | + struct bpf_offload_netdev, | |
| 169 | + offdev_netdevs); | |
| 170 | + if (altdev) { | |
| 171 | + list_for_each_entry(offload, &ondev->progs, offloads) | |
| 172 | + offload->netdev = altdev->netdev; | |
| 173 | + list_splice_init(&ondev->progs, &altdev->progs); | |
| 174 | + | |
| 175 | + list_for_each_entry(offmap, &ondev->maps, offloads) | |
| 176 | + offmap->netdev = altdev->netdev; | |
| 177 | + list_splice_init(&ondev->maps, &altdev->maps); | |
| 178 | + } else { | |
| 179 | + list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) | |
| 180 | + __bpf_prog_offload_destroy(offload->prog); | |
| 181 | + list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) | |
| 182 | + __bpf_map_offload_destroy(offmap); | |
| 183 | + } | |
| 184 | + | |
| 185 | + WARN_ON(!list_empty(&ondev->progs)); | |
| 186 | + WARN_ON(!list_empty(&ondev->maps)); | |
| 187 | + kfree(ondev); | |
| 188 | +unlock: | |
| 189 | + up_write(&bpf_devs_lock); | |
| 190 | +} | |
| 191 | + | |
| 77 | 192 | int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) |
| 78 | 193 | { |
| 79 | 194 | struct bpf_offload_netdev *ondev; |
| @@ -206,21 +321,6 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) · 126 unmodified lines | ||
| 206 | 321 | up_read(&bpf_devs_lock); |
| 207 | 322 | } |
| 208 | 323 | |
| 209 | −static void __bpf_prog_offload_destroy(struct bpf_prog *prog) | |
| 210 | −{ | |
| 211 | − struct bpf_prog_offload *offload = prog->aux->offload; | |
| 212 | − | |
| 213 | − if (offload->dev_state) | |
| 214 | − offload->offdev->ops->destroy(prog); | |
| 215 | − | |
| 216 | − /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ | |
| 217 | − bpf_prog_free_id(prog, true); | |
| 218 | − | |
| 219 | − list_del_init(&offload->offloads); | |
| 220 | − kfree(offload); | |
| 221 | − prog->aux->offload = NULL; | |
| 222 | −} | |
| 223 | − | |
| 224 | 324 | void bpf_prog_offload_destroy(struct bpf_prog *prog) |
| 225 | 325 | { |
| 226 | 326 | down_write(&bpf_devs_lock); |
| @@ -340,22 +440,6 @@ int bpf_prog_offload_info_fill(struct bpf_prog_info *info, · 113 unmodified lines | ||
| 340 | 440 | const struct bpf_prog_ops bpf_offload_prog_ops = { |
| 341 | 441 | }; |
| 342 | 442 | |
| 343 | −static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, | |
| 344 | − enum bpf_netdev_command cmd) | |
| 345 | −{ | |
| 346 | − struct netdev_bpf data = {}; | |
| 347 | − struct net_device *netdev; | |
| 348 | − | |
| 349 | − ASSERT_RTNL(); | |
| 350 | − | |
| 351 | − data.command = cmd; | |
| 352 | − data.offmap = offmap; | |
| 353 | − /* Caller must make sure netdev is valid */ | |
| 354 | − netdev = offmap->netdev; | |
| 355 | − | |
| 356 | − return netdev->netdev_ops->ndo_bpf(netdev, &data); | |
| 357 | −} | |
| 358 | − | |
| 359 | 443 | struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) |
| 360 | 444 | { |
| 361 | 445 | struct net *net = current->nsproxy->net_ns; |
| @@ -405,15 +489,6 @@ err_unlock: · 43 unmodified lines | ||
| 405 | 489 | return ERR_PTR(err); |
| 406 | 490 | } |
| 407 | 491 | |
| 408 | −static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) | |
| 409 | −{ | |
| 410 | − WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); | |
| 411 | − /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ | |
| 412 | − bpf_map_free_id(&offmap->map, true); | |
| 413 | − list_del_init(&offmap->offloads); | |
| 414 | − offmap->netdev = NULL; | |
| 415 | −} | |
| 416 | − | |
| 417 | 492 | void bpf_map_offload_map_free(struct bpf_map *map) |
| 418 | 493 | { |
| 419 | 494 | struct bpf_offloaded_map *offmap = map_to_offmap(map); |
| @@ -592,77 +667,14 @@ bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map) · 172 unmodified lines | ||
| 592 | 667 | int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, |
| 593 | 668 | struct net_device *netdev) |
| 594 | 669 | { |
| 595 | − struct bpf_offload_netdev *ondev; | |
| 596 | − int err; | |
| 597 | − | |
| 598 | − ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); | |
| 599 | − if (!ondev) | |
| 600 | − return -ENOMEM; | |
| 601 | − | |
| 602 | − ondev->netdev = netdev; | |
| 603 | − ondev->offdev = offdev; | |
| 604 | − INIT_LIST_HEAD(&ondev->progs); | |
| 605 | − INIT_LIST_HEAD(&ondev->maps); | |
| 606 | − | |
| 607 | − down_write(&bpf_devs_lock); | |
| 608 | − err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); | |
| 609 | − if (err) { | |
| 610 | − netdev_warn(netdev, "failed to register for BPF offload\n"); | |
| 611 | − goto err_unlock_free; | |
| 612 | − } | |
| 613 | − | |
| 614 | − list_add(&ondev->offdev_netdevs, &offdev->netdevs); | |
| 615 | − up_write(&bpf_devs_lock); | |
| 616 | − return 0; | |
| 617 | − | |
| 618 | −err_unlock_free: | |
| 619 | − up_write(&bpf_devs_lock); | |
| 620 | − kfree(ondev); | |
| 621 | − return err; | |
| 670 | + return __bpf_offload_dev_netdev_register(offdev, netdev); | |
| 622 | 671 | } |
| 623 | 672 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register); |
| 624 | 673 | |
| 625 | 674 | void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, |
| 626 | 675 | struct net_device *netdev) |
| 627 | 676 | { |
| 628 | − struct bpf_offload_netdev *ondev, *altdev; | |
| 629 | − struct bpf_offloaded_map *offmap, *mtmp; | |
| 630 | − struct bpf_prog_offload *offload, *ptmp; | |
| 631 | − | |
| 632 | − ASSERT_RTNL(); | |
| 633 | − | |
| 634 | − down_write(&bpf_devs_lock); | |
| 635 | − ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); | |
| 636 | − if (WARN_ON(!ondev)) | |
| 637 | − goto unlock; | |
| 638 | − | |
| 639 | − WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); | |
| 640 | − list_del(&ondev->offdev_netdevs); | |
| 641 | − | |
| 642 | − /* Try to move the objects to another netdev of the device */ | |
| 643 | − altdev = list_first_entry_or_null(&offdev->netdevs, | |
| 644 | − struct bpf_offload_netdev, | |
| 645 | − offdev_netdevs); | |
| 646 | − if (altdev) { | |
| 647 | − list_for_each_entry(offload, &ondev->progs, offloads) | |
| 648 | − offload->netdev = altdev->netdev; | |
| 649 | − list_splice_init(&ondev->progs, &altdev->progs); | |
| 650 | − | |
| 651 | − list_for_each_entry(offmap, &ondev->maps, offloads) | |
| 652 | − offmap->netdev = altdev->netdev; | |
| 653 | − list_splice_init(&ondev->maps, &altdev->maps); | |
| 654 | − } else { | |
| 655 | − list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) | |
| 656 | − __bpf_prog_offload_destroy(offload->prog); | |
| 657 | − list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) | |
| 658 | − __bpf_map_offload_destroy(offmap); | |
| 659 | − } | |
| 660 | − | |
| 661 | − WARN_ON(!list_empty(&ondev->progs)); | |
| 662 | − WARN_ON(!list_empty(&ondev->maps)); | |
| 663 | − kfree(ondev); | |
| 664 | −unlock: | |
| 665 | − up_write(&bpf_devs_lock); | |
| 677 | + __bpf_offload_dev_netdev_unregister(offdev, netdev); | |
| 666 | 678 | } |
| 667 | 679 | EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); |
| 668 | 680 | |