History

bpf: Convert bpf_prog refcnt to atomic64_t

Similarly to bpf_map's refcnt/usercnt, convert bpf_prog's refcnt to atomic64 and remove artificial 32k limit. This allows to make bpf_prog's refcounting non-failing, simplifying logic of users of bpf_prog_add/bpf_prog_inc. Validated compilation by running allyesconfig kernel build.

Andrii Nakryiko committed 6 years ago

Commit 85192db1 parent 1e0bd5a11 files · +40 −97Browse files

Unified · Split

11 files
  1. drivers/net/ethernet/broadcom/bnxt/bnxt.c +2 −7
  2. drivers/net/ethernet/cavium/thunder/nicvf_main.c +2 −7
  3. drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +2 −5
  4. drivers/net/ethernet/mellanox/mlx4/en_netdev.c +6 −18
  5. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +5 −13
  6. drivers/net/ethernet/qlogic/qede/qede_main.c +2 −6
  7. drivers/net/virtio_net.c +2 −5
  8. include/linux/bpf.h +5 −8
  9. kernel/bpf/inode.c +3 −2
  10. kernel/bpf/syscall.c +9 −21
  11. kernel/events/core.c +2 −5
drivers/net/ethernet/broadcom/bnxt/bnxt.c+2−7
@@ -3171,13 +3171,8 @@ static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) · 3170 unmodified lines
31713171 bnxt_init_rxbd_pages(ring, type);
31723172
31733173 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3174− rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
3175− if (IS_ERR(rxr->xdp_prog)) {
3176− int rc = PTR_ERR(rxr->xdp_prog);
3177−
3178− rxr->xdp_prog = NULL;
3179− return rc;
3180− }
3174+ bpf_prog_add(bp->xdp_prog, 1);
3175+ rxr->xdp_prog = bp->xdp_prog;
31813176 }
31823177 prod = rxr->rx_prod;
31833178 for (i = 0; i < bp->rx_ring_size; i++) {
drivers/net/ethernet/cavium/thunder/nicvf_main.c+2−7
@@ -1876,13 +1876,8 @@ static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog) · 1875 unmodified lines
18761876
18771877 if (nic->xdp_prog) {
18781878 /* Attach BPF program */
1879− nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
1880− if (!IS_ERR(nic->xdp_prog)) {
1881− bpf_attached = true;
1882− } else {
1883− ret = PTR_ERR(nic->xdp_prog);
1884− nic->xdp_prog = NULL;
1885− }
1879+ bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
1880+ bpf_attached = true;
18861881 }
18871882
18881883 /* Calculate Tx queues needed for XDP and network stack */
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c+2−5
@@ -1807,11 +1807,8 @@ static int setup_xdp(struct net_device *dev, struct bpf_prog *prog) · 1806 unmodified lines
18071807 if (prog && !xdp_mtu_valid(priv, dev->mtu))
18081808 return -EINVAL;
18091809
1810− if (prog) {
1811− prog = bpf_prog_add(prog, priv->num_channels);
1812− if (IS_ERR(prog))
1813− return PTR_ERR(prog);
1814− }
1810+ if (prog)
1811+ bpf_prog_add(prog, priv->num_channels);
18151812
18161813 up = netif_running(dev);
18171814 need_update = (!!priv->xdp_prog != !!prog);
drivers/net/ethernet/mellanox/mlx4/en_netdev.c+6−18
@@ -2286,11 +2286,7 @@ int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, · 2285 unmodified lines
22862286 lockdep_is_held(&priv->mdev->state_lock));
22872287
22882288 if (xdp_prog && carry_xdp_prog) {
2289− xdp_prog = bpf_prog_add(xdp_prog, tmp->rx_ring_num);
2290− if (IS_ERR(xdp_prog)) {
2291− mlx4_en_free_resources(tmp);
2292− return PTR_ERR(xdp_prog);
2293− }
2289+ bpf_prog_add(xdp_prog, tmp->rx_ring_num);
22942290 for (i = 0; i < tmp->rx_ring_num; i++)
22952291 rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog,
22962292 xdp_prog);
@@ -2782,11 +2778,9 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 485 unmodified lines
27822778 * program for a new one.
27832779 */
27842780 if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) {
2785− if (prog) {
2786− prog = bpf_prog_add(prog, priv->rx_ring_num - 1);
2787− if (IS_ERR(prog))
2788− return PTR_ERR(prog);
2789− }
2781+ if (prog)
2782+ bpf_prog_add(prog, priv->rx_ring_num - 1);
2783+
27902784 mutex_lock(&mdev->state_lock);
27912785 for (i = 0; i < priv->rx_ring_num; i++) {
27922786 old_prog = rcu_dereference_protected(
@@ -2807,13 +2801,8 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 14 unmodified lines
28072801 if (!tmp)
28082802 return -ENOMEM;
28092803
2810− if (prog) {
2811− prog = bpf_prog_add(prog, priv->rx_ring_num - 1);
2812− if (IS_ERR(prog)) {
2813− err = PTR_ERR(prog);
2814− goto out;
2815− }
2816− }
2804+ if (prog)
2805+ bpf_prog_add(prog, priv->rx_ring_num - 1);
28172806
28182807 mutex_lock(&mdev->state_lock);
28192808 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile));
@@ -2862,7 +2851,6 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 42 unmodified lines
28622851
28632852 unlock_out:
28642853 mutex_unlock(&mdev->state_lock);
2865−out:
28662854 kfree(tmp);
28672855 return err;
28682856 }
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+5−13
@@ -408,12 +408,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 407 unmodified lines
408408 rq->stats = &c->priv->channel_stats[c->ix].rq;
409409 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
410410
411− rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
412− if (IS_ERR(rq->xdp_prog)) {
413− err = PTR_ERR(rq->xdp_prog);
414− rq->xdp_prog = NULL;
415− goto err_rq_wq_destroy;
416− }
411+ if (params->xdp_prog)
412+ bpf_prog_inc(params->xdp_prog);
413+ rq->xdp_prog = params->xdp_prog;
417414
418415 rq_xdp_ix = rq->ix;
419416 if (xsk)
@@ -4406,16 +4403,11 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) · 3986 unmodified lines
44064403 /* no need for full reset when exchanging programs */
44074404 reset = (!priv->channels.params.xdp_prog || !prog);
44084405
4409− if (was_opened && !reset) {
4406+ if (was_opened && !reset)
44104407 /* num_channels is invariant here, so we can take the
44114408 * batched reference right upfront.
44124409 */
4413− prog = bpf_prog_add(prog, priv->channels.num);
4414− if (IS_ERR(prog)) {
4415− err = PTR_ERR(prog);
4416− goto unlock;
4417− }
4418− }
4410+ bpf_prog_add(prog, priv->channels.num);
44194411
44204412 if (was_opened && reset) {
44214413 struct mlx5e_channels new_channels = {};
drivers/net/ethernet/qlogic/qede/qede_main.c+2−6
@@ -2107,12 +2107,8 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats) · 2106 unmodified lines
21072107 if (rc)
21082108 goto out;
21092109
2110− fp->rxq->xdp_prog = bpf_prog_add(edev->xdp_prog, 1);
2111− if (IS_ERR(fp->rxq->xdp_prog)) {
2112− rc = PTR_ERR(fp->rxq->xdp_prog);
2113− fp->rxq->xdp_prog = NULL;
2114− goto out;
2115− }
2110+ bpf_prog_add(edev->xdp_prog, 1);
2111+ fp->rxq->xdp_prog = edev->xdp_prog;
21162112 }
21172113
21182114 if (fp->type & QEDE_FASTPATH_TX) {
drivers/net/virtio_net.c+2−5
@@ -2445,11 +2445,8 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, · 2444 unmodified lines
24452445 if (!prog && !old_prog)
24462446 return 0;
24472447
2448− if (prog) {
2449− prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);
2450− if (IS_ERR(prog))
2451− return PTR_ERR(prog);
2452− }
2448+ if (prog)
2449+ bpf_prog_add(prog, vi->max_queue_pairs - 1);
24532450
24542451 /* Make sure NAPI is not using any XDP TX queues for RX. */
24552452 if (netif_running(dev)) {
include/linux/bpf.h+5−8
@@ -485,7 +485,7 @@ struct bpf_func_info_aux { · 484 unmodified lines
485485 };
486486
487487 struct bpf_prog_aux {
488− atomic_t refcnt;
488+ atomic64_t refcnt;
489489 u32 used_map_cnt;
490490 u32 max_ctx_offset;
491491 u32 max_pkt_offset;
@@ -770,9 +770,9 @@ extern const struct bpf_verifier_ops xdp_analyzer_ops; · 278 unmodified lines
770770 struct bpf_prog *bpf_prog_get(u32 ufd);
771771 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
772772 bool attach_drv);
773−struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);
773+void bpf_prog_add(struct bpf_prog *prog, int i);
774774 void bpf_prog_sub(struct bpf_prog *prog, int i);
775−struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);
775+void bpf_prog_inc(struct bpf_prog *prog);
776776 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
777777 void bpf_prog_put(struct bpf_prog *prog);
778778 int __bpf_prog_charge(struct user_struct *user, u32 pages);
@@ -912,10 +912,8 @@ static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, · 133 unmodified lines
912912 return ERR_PTR(-EOPNOTSUPP);
913913 }
914914
915−static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog,
916− int i)
915+static inline void bpf_prog_add(struct bpf_prog *prog, int i)
917916 {
918− return ERR_PTR(-EOPNOTSUPP);
919917 }
920918
921919 static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
@@ -926,9 +924,8 @@ static inline void bpf_prog_put(struct bpf_prog *prog) · 4 unmodified lines
926924 {
927925 }
928926
929−static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)
927+static inline void bpf_prog_inc(struct bpf_prog *prog)
930928 {
931− return ERR_PTR(-EOPNOTSUPP);
932929 }
933930
934931 static inline struct bpf_prog *__must_check
kernel/bpf/inode.c+3−2
@@ -31,7 +31,7 @@ static void *bpf_any_get(void *raw, enum bpf_type type) · 30 unmodified lines
3131 {
3232 switch (type) {
3333 case BPF_TYPE_PROG:
34− raw = bpf_prog_inc(raw);
34+ bpf_prog_inc(raw);
3535 break;
3636 case BPF_TYPE_MAP:
3737 bpf_map_inc_with_uref(raw);
@@ -534,7 +534,8 @@ static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type · 496 unmodified lines
534534 if (!bpf_prog_get_ok(prog, &type, false))
535535 return ERR_PTR(-EINVAL);
536536
537− return bpf_prog_inc(prog);
537+ bpf_prog_inc(prog);
538+ return prog;
538539 }
539540
540541 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type)
kernel/bpf/syscall.c+9−21
@@ -1339,7 +1339,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1338 unmodified lines
13391339
13401340 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
13411341 {
1342− if (atomic_dec_and_test(&prog->aux->refcnt)) {
1342+ if (atomic64_dec_and_test(&prog->aux->refcnt)) {
13431343 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
13441344 /* bpf_prog_free_id() must be called first */
13451345 bpf_prog_free_id(prog, do_idr_lock);
@@ -1445,16 +1445,9 @@ static struct bpf_prog *____bpf_prog_get(struct fd f) · 99 unmodified lines
14451445 return f.file->private_data;
14461446 }
14471447
1448−/* prog's refcnt limit */
1449−#define BPF_MAX_REFCNT 32768
1450−
1451−struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
1448+void bpf_prog_add(struct bpf_prog *prog, int i)
14521449 {
1453− if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
1454− atomic_sub(i, &prog->aux->refcnt);
1455− return ERR_PTR(-EBUSY);
1456− }
1457− return prog;
1450+ atomic64_add(i, &prog->aux->refcnt);
14581451 }
14591452 EXPORT_SYMBOL_GPL(bpf_prog_add);
14601453
@@ -1465,13 +1458,13 @@ void bpf_prog_sub(struct bpf_prog *prog, int i) · 4 unmodified lines
14651458 * path holds a reference to the program, thus atomic_sub() can
14661459 * be safely used in such cases!
14671460 */
1468− WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
1461+ WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
14691462 }
14701463 EXPORT_SYMBOL_GPL(bpf_prog_sub);
14711464
1472−struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
1465+void bpf_prog_inc(struct bpf_prog *prog)
14731466 {
1474− return bpf_prog_add(prog, 1);
1467+ atomic64_inc(&prog->aux->refcnt);
14751468 }
14761469 EXPORT_SYMBOL_GPL(bpf_prog_inc);
14771470
@@ -1480,12 +1473,7 @@ struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) · 2 unmodified lines
14801473 {
14811474 int refold;
14821475
1483− refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1484−
1485− if (refold >= BPF_MAX_REFCNT) {
1486− __bpf_prog_put(prog, false);
1487− return ERR_PTR(-EBUSY);
1488− }
1476+ refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
14891477
14901478 if (!refold)
14911479 return ERR_PTR(-ENOENT);
@@ -1523,7 +1511,7 @@ static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, · 31 unmodified lines
15231511 goto out;
15241512 }
15251513
1526− prog = bpf_prog_inc(prog);
1514+ bpf_prog_inc(prog);
15271515 out:
15281516 fdput(f);
15291517 return prog;
@@ -1714,7 +1702,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) · 184 unmodified lines
17141702 prog->orig_prog = NULL;
17151703 prog->jited = 0;
17161704
1717− atomic_set(&prog->aux->refcnt, 1);
1705+ atomic64_set(&prog->aux->refcnt, 1);
17181706 prog->gpl_compatible = is_gpl ? 1 : 0;
17191707
17201708 if (bpf_prog_is_dev_bound(prog->aux)) {
kernel/events/core.c+2−5
@@ -10477,12 +10477,9 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 10476 unmodified lines
1047710477 context = parent_event->overflow_handler_context;
1047810478 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
1047910479 if (overflow_handler == bpf_overflow_handler) {
10480− struct bpf_prog *prog = bpf_prog_inc(parent_event->prog);
10480+ struct bpf_prog *prog = parent_event->prog;
1048110481
10482− if (IS_ERR(prog)) {
10483− err = PTR_ERR(prog);
10484− goto err_ns;
10485− }
10482+ bpf_prog_inc(prog);
1048610483 event->prog = prog;
1048710484 event->orig_overflow_handler =
1048810485 parent_event->orig_overflow_handler;