History

bpf: Convert bpf_prog refcnt to atomic64_t

Similarly to bpf_map's refcnt/usercnt, convert bpf_prog's refcnt to atomic64 and remove artificial 32k limit. This allows to make bpf_prog's refcounting non-failing, simplifying logic of users of bpf_prog_add/bpf_prog_inc. Validated compilation by running allyesconfig kernel build.

Andrii Nakryiko committed 6 years ago

Commit 85192db1 parent 1e0bd5a11 files · +40 −97Browse files

Unified · Split

11 files
  1. drivers/net/ethernet/broadcom/bnxt/bnxt.c +2 −7
  2. drivers/net/ethernet/cavium/thunder/nicvf_main.c +2 −7
  3. drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +2 −5
  4. drivers/net/ethernet/mellanox/mlx4/en_netdev.c +6 −18
  5. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +5 −13
  6. drivers/net/ethernet/qlogic/qede/qede_main.c +2 −6
  7. drivers/net/virtio_net.c +2 −5
  8. include/linux/bpf.h +5 −8
  9. kernel/bpf/inode.c +3 −2
  10. kernel/bpf/syscall.c +9 −21
  11. kernel/events/core.c +2 −5
drivers/net/ethernet/broadcom/bnxt/bnxt.c+2−7
@@ -3171,13 +3171,8 @@ static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) · 3170 unmodified lines
3171 bnxt_init_rxbd_pages(ring, type);3171 bnxt_init_rxbd_pages(ring, type);
3172 3172
3173 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {3173 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3174 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);3174 bpf_prog_add(bp->xdp_prog, 1);
3175 if (IS_ERR(rxr->xdp_prog)) {3175 rxr->xdp_prog = bp->xdp_prog;
3176 int rc = PTR_ERR(rxr->xdp_prog);
3177
3178 rxr->xdp_prog = NULL;
3179 return rc;
3180 }
3181 }3176 }
3182 prod = rxr->rx_prod;3177 prod = rxr->rx_prod;
3183 for (i = 0; i < bp->rx_ring_size; i++) {3178 for (i = 0; i < bp->rx_ring_size; i++) {
drivers/net/ethernet/cavium/thunder/nicvf_main.c+2−7
@@ -1876,13 +1876,8 @@ static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog) · 1875 unmodified lines
1876 1876
1877 if (nic->xdp_prog) {1877 if (nic->xdp_prog) {
1878 /* Attach BPF program */1878 /* Attach BPF program */
1879 nic->xdp_prog = bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);1879 bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1);
1880 if (!IS_ERR(nic->xdp_prog)) {1880 bpf_attached = true;
1881 bpf_attached = true;
1882 } else {
1883 ret = PTR_ERR(nic->xdp_prog);
1884 nic->xdp_prog = NULL;
1885 }
1886 }1881 }
1887 1882
1888 /* Calculate Tx queues needed for XDP and network stack */1883 /* Calculate Tx queues needed for XDP and network stack */
drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c+2−5
@@ -1807,11 +1807,8 @@ static int setup_xdp(struct net_device *dev, struct bpf_prog *prog) · 1806 unmodified lines
1807 if (prog && !xdp_mtu_valid(priv, dev->mtu))1807 if (prog && !xdp_mtu_valid(priv, dev->mtu))
1808 return -EINVAL;1808 return -EINVAL;
1809 1809
1810 if (prog) {1810 if (prog)
1811 prog = bpf_prog_add(prog, priv->num_channels);1811 bpf_prog_add(prog, priv->num_channels);
1812 if (IS_ERR(prog))
1813 return PTR_ERR(prog);
1814 }
1815 1812
1816 up = netif_running(dev);1813 up = netif_running(dev);
1817 need_update = (!!priv->xdp_prog != !!prog);1814 need_update = (!!priv->xdp_prog != !!prog);
drivers/net/ethernet/mellanox/mlx4/en_netdev.c+6−18
@@ -2286,11 +2286,7 @@ int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, · 2285 unmodified lines
2286 lockdep_is_held(&priv->mdev->state_lock));2286 lockdep_is_held(&priv->mdev->state_lock));
2287 2287
2288 if (xdp_prog && carry_xdp_prog) {2288 if (xdp_prog && carry_xdp_prog) {
2289 xdp_prog = bpf_prog_add(xdp_prog, tmp->rx_ring_num);2289 bpf_prog_add(xdp_prog, tmp->rx_ring_num);
2290 if (IS_ERR(xdp_prog)) {
2291 mlx4_en_free_resources(tmp);
2292 return PTR_ERR(xdp_prog);
2293 }
2294 for (i = 0; i < tmp->rx_ring_num; i++)2290 for (i = 0; i < tmp->rx_ring_num; i++)
2295 rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog,2291 rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog,
2296 xdp_prog);2292 xdp_prog);
@@ -2782,11 +2778,9 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 485 unmodified lines
2782 * program for a new one.2778 * program for a new one.
2783 */2779 */
2784 if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) {2780 if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) {
2785 if (prog) {2781 if (prog)
2786 prog = bpf_prog_add(prog, priv->rx_ring_num - 1);2782 bpf_prog_add(prog, priv->rx_ring_num - 1);
2787 if (IS_ERR(prog))2783
2788 return PTR_ERR(prog);
2789 }
2790 mutex_lock(&mdev->state_lock);2784 mutex_lock(&mdev->state_lock);
2791 for (i = 0; i < priv->rx_ring_num; i++) {2785 for (i = 0; i < priv->rx_ring_num; i++) {
2792 old_prog = rcu_dereference_protected(2786 old_prog = rcu_dereference_protected(
@@ -2807,13 +2801,8 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 14 unmodified lines
2807 if (!tmp)2801 if (!tmp)
2808 return -ENOMEM;2802 return -ENOMEM;
2809 2803
2810 if (prog) {2804 if (prog)
2811 prog = bpf_prog_add(prog, priv->rx_ring_num - 1);2805 bpf_prog_add(prog, priv->rx_ring_num - 1);
2812 if (IS_ERR(prog)) {
2813 err = PTR_ERR(prog);
2814 goto out;
2815 }
2816 }
2817 2806
2818 mutex_lock(&mdev->state_lock);2807 mutex_lock(&mdev->state_lock);
2819 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile));2808 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile));
@@ -2862,7 +2851,6 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) · 42 unmodified lines
2862 2851
2863unlock_out:2852unlock_out:
2864 mutex_unlock(&mdev->state_lock);2853 mutex_unlock(&mdev->state_lock);
2865out:
2866 kfree(tmp);2854 kfree(tmp);
2867 return err;2855 return err;
2868}2856}
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+5−13
@@ -408,12 +408,9 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 407 unmodified lines
408 rq->stats = &c->priv->channel_stats[c->ix].rq;408 rq->stats = &c->priv->channel_stats[c->ix].rq;
409 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);409 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
410 410
411 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;411 if (params->xdp_prog)
412 if (IS_ERR(rq->xdp_prog)) {412 bpf_prog_inc(params->xdp_prog);
413 err = PTR_ERR(rq->xdp_prog);413 rq->xdp_prog = params->xdp_prog;
414 rq->xdp_prog = NULL;
415 goto err_rq_wq_destroy;
416 }
417 414
418 rq_xdp_ix = rq->ix;415 rq_xdp_ix = rq->ix;
419 if (xsk)416 if (xsk)
@@ -4406,16 +4403,11 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) · 3986 unmodified lines
4406 /* no need for full reset when exchanging programs */4403 /* no need for full reset when exchanging programs */
4407 reset = (!priv->channels.params.xdp_prog || !prog);4404 reset = (!priv->channels.params.xdp_prog || !prog);
4408 4405
4409 if (was_opened && !reset) {4406 if (was_opened && !reset)
4410 /* num_channels is invariant here, so we can take the4407 /* num_channels is invariant here, so we can take the
4411 * batched reference right upfront.4408 * batched reference right upfront.
4412 */4409 */
4413 prog = bpf_prog_add(prog, priv->channels.num);4410 bpf_prog_add(prog, priv->channels.num);
4414 if (IS_ERR(prog)) {
4415 err = PTR_ERR(prog);
4416 goto unlock;
4417 }
4418 }
4419 4411
4420 if (was_opened && reset) {4412 if (was_opened && reset) {
4421 struct mlx5e_channels new_channels = {};4413 struct mlx5e_channels new_channels = {};
drivers/net/ethernet/qlogic/qede/qede_main.c+2−6
@@ -2107,12 +2107,8 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats) · 2106 unmodified lines
2107 if (rc)2107 if (rc)
2108 goto out;2108 goto out;
2109 2109
2110 fp->rxq->xdp_prog = bpf_prog_add(edev->xdp_prog, 1);2110 bpf_prog_add(edev->xdp_prog, 1);
2111 if (IS_ERR(fp->rxq->xdp_prog)) {2111 fp->rxq->xdp_prog = edev->xdp_prog;
2112 rc = PTR_ERR(fp->rxq->xdp_prog);
2113 fp->rxq->xdp_prog = NULL;
2114 goto out;
2115 }
2116 }2112 }
2117 2113
2118 if (fp->type & QEDE_FASTPATH_TX) {2114 if (fp->type & QEDE_FASTPATH_TX) {
drivers/net/virtio_net.c+2−5
@@ -2445,11 +2445,8 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, · 2444 unmodified lines
2445 if (!prog && !old_prog)2445 if (!prog && !old_prog)
2446 return 0;2446 return 0;
2447 2447
2448 if (prog) {2448 if (prog)
2449 prog = bpf_prog_add(prog, vi->max_queue_pairs - 1);2449 bpf_prog_add(prog, vi->max_queue_pairs - 1);
2450 if (IS_ERR(prog))
2451 return PTR_ERR(prog);
2452 }
2453 2450
2454 /* Make sure NAPI is not using any XDP TX queues for RX. */2451 /* Make sure NAPI is not using any XDP TX queues for RX. */
2455 if (netif_running(dev)) {2452 if (netif_running(dev)) {
include/linux/bpf.h+5−8
@@ -485,7 +485,7 @@ struct bpf_func_info_aux { · 484 unmodified lines
485};485};
486 486
487struct bpf_prog_aux {487struct bpf_prog_aux {
488 atomic_t refcnt;488 atomic64_t refcnt;
489 u32 used_map_cnt;489 u32 used_map_cnt;
490 u32 max_ctx_offset;490 u32 max_ctx_offset;
491 u32 max_pkt_offset;491 u32 max_pkt_offset;
@@ -770,9 +770,9 @@ extern const struct bpf_verifier_ops xdp_analyzer_ops; · 278 unmodified lines
770struct bpf_prog *bpf_prog_get(u32 ufd);770struct bpf_prog *bpf_prog_get(u32 ufd);
771struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,771struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
772 bool attach_drv);772 bool attach_drv);
773struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);773void bpf_prog_add(struct bpf_prog *prog, int i);
774void bpf_prog_sub(struct bpf_prog *prog, int i);774void bpf_prog_sub(struct bpf_prog *prog, int i);
775struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);775void bpf_prog_inc(struct bpf_prog *prog);
776struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);776struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
777void bpf_prog_put(struct bpf_prog *prog);777void bpf_prog_put(struct bpf_prog *prog);
778int __bpf_prog_charge(struct user_struct *user, u32 pages);778int __bpf_prog_charge(struct user_struct *user, u32 pages);
@@ -912,10 +912,8 @@ static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, · 133 unmodified lines
912 return ERR_PTR(-EOPNOTSUPP);912 return ERR_PTR(-EOPNOTSUPP);
913}913}
914 914
915static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog,915static inline void bpf_prog_add(struct bpf_prog *prog, int i)
916 int i)
917{916{
918 return ERR_PTR(-EOPNOTSUPP);
919}917}
920 918
921static inline void bpf_prog_sub(struct bpf_prog *prog, int i)919static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
@@ -926,9 +924,8 @@ static inline void bpf_prog_put(struct bpf_prog *prog) · 4 unmodified lines
926{924{
927}925}
928 926
929static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)927static inline void bpf_prog_inc(struct bpf_prog *prog)
930{928{
931 return ERR_PTR(-EOPNOTSUPP);
932}929}
933 930
934static inline struct bpf_prog *__must_check931static inline struct bpf_prog *__must_check
kernel/bpf/inode.c+3−2
@@ -31,7 +31,7 @@ static void *bpf_any_get(void *raw, enum bpf_type type) · 30 unmodified lines
31{31{
32 switch (type) {32 switch (type) {
33 case BPF_TYPE_PROG:33 case BPF_TYPE_PROG:
34 raw = bpf_prog_inc(raw);34 bpf_prog_inc(raw);
35 break;35 break;
36 case BPF_TYPE_MAP:36 case BPF_TYPE_MAP:
37 bpf_map_inc_with_uref(raw);37 bpf_map_inc_with_uref(raw);
@@ -534,7 +534,8 @@ static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type · 496 unmodified lines
534 if (!bpf_prog_get_ok(prog, &type, false))534 if (!bpf_prog_get_ok(prog, &type, false))
535 return ERR_PTR(-EINVAL);535 return ERR_PTR(-EINVAL);
536 536
537 return bpf_prog_inc(prog);537 bpf_prog_inc(prog);
538 return prog;
538}539}
539 540
540struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type)541struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type)
kernel/bpf/syscall.c+9−21
@@ -1339,7 +1339,7 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) · 1338 unmodified lines
1339 1339
1340static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)1340static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1341{1341{
1342 if (atomic_dec_and_test(&prog->aux->refcnt)) {1342 if (atomic64_dec_and_test(&prog->aux->refcnt)) {
1343 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);1343 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1344 /* bpf_prog_free_id() must be called first */1344 /* bpf_prog_free_id() must be called first */
1345 bpf_prog_free_id(prog, do_idr_lock);1345 bpf_prog_free_id(prog, do_idr_lock);
@@ -1445,16 +1445,9 @@ static struct bpf_prog *____bpf_prog_get(struct fd f) · 99 unmodified lines
1445 return f.file->private_data;1445 return f.file->private_data;
1446}1446}
1447 1447
1448/* prog's refcnt limit */1448void bpf_prog_add(struct bpf_prog *prog, int i)
1449#define BPF_MAX_REFCNT 32768
1450
1451struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
1452{1449{
1453 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {1450 atomic64_add(i, &prog->aux->refcnt);
1454 atomic_sub(i, &prog->aux->refcnt);
1455 return ERR_PTR(-EBUSY);
1456 }
1457 return prog;
1458}1451}
1459EXPORT_SYMBOL_GPL(bpf_prog_add);1452EXPORT_SYMBOL_GPL(bpf_prog_add);
1460 1453
@@ -1465,13 +1458,13 @@ void bpf_prog_sub(struct bpf_prog *prog, int i) · 4 unmodified lines
1465 * path holds a reference to the program, thus atomic_sub() can1458 * path holds a reference to the program, thus atomic_sub() can
1466 * be safely used in such cases!1459 * be safely used in such cases!
1467 */1460 */
1468 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);1461 WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1469}1462}
1470EXPORT_SYMBOL_GPL(bpf_prog_sub);1463EXPORT_SYMBOL_GPL(bpf_prog_sub);
1471 1464
1472struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)1465void bpf_prog_inc(struct bpf_prog *prog)
1473{1466{
1474 return bpf_prog_add(prog, 1);1467 atomic64_inc(&prog->aux->refcnt);
1475}1468}
1476EXPORT_SYMBOL_GPL(bpf_prog_inc);1469EXPORT_SYMBOL_GPL(bpf_prog_inc);
1477 1470
@@ -1480,12 +1473,7 @@ struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) · 2 unmodified lines
1480{1473{
1481 int refold;1474 int refold;
1482 1475
1483 refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);1476 refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1484
1485 if (refold >= BPF_MAX_REFCNT) {
1486 __bpf_prog_put(prog, false);
1487 return ERR_PTR(-EBUSY);
1488 }
1489 1477
1490 if (!refold)1478 if (!refold)
1491 return ERR_PTR(-ENOENT);1479 return ERR_PTR(-ENOENT);
@@ -1523,7 +1511,7 @@ static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, · 31 unmodified lines
1523 goto out;1511 goto out;
1524 }1512 }
1525 1513
1526 prog = bpf_prog_inc(prog);1514 bpf_prog_inc(prog);
1527out:1515out:
1528 fdput(f);1516 fdput(f);
1529 return prog;1517 return prog;
@@ -1714,7 +1702,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) · 184 unmodified lines
1714 prog->orig_prog = NULL;1702 prog->orig_prog = NULL;
1715 prog->jited = 0;1703 prog->jited = 0;
1716 1704
1717 atomic_set(&prog->aux->refcnt, 1);1705 atomic64_set(&prog->aux->refcnt, 1);
1718 prog->gpl_compatible = is_gpl ? 1 : 0;1706 prog->gpl_compatible = is_gpl ? 1 : 0;
1719 1707
1720 if (bpf_prog_is_dev_bound(prog->aux)) {1708 if (bpf_prog_is_dev_bound(prog->aux)) {
kernel/events/core.c+2−5
@@ -10477,12 +10477,9 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, · 10476 unmodified lines
10477 context = parent_event->overflow_handler_context;10477 context = parent_event->overflow_handler_context;
10478#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)10478#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING)
10479 if (overflow_handler == bpf_overflow_handler) {10479 if (overflow_handler == bpf_overflow_handler) {
10480 struct bpf_prog *prog = bpf_prog_inc(parent_event->prog);10480 struct bpf_prog *prog = parent_event->prog;
10481 10481
10482 if (IS_ERR(prog)) {10482 bpf_prog_inc(prog);
10483 err = PTR_ERR(prog);
10484 goto err_ns;
10485 }
10486 event->prog = prog;10483 event->prog = prog;
10487 event->orig_overflow_handler =10484 event->orig_overflow_handler =
10488 parent_event->orig_overflow_handler;10485 parent_event->orig_overflow_handler;