History

(IB, net)/mlx4: Add resource utilization support

Adding visibility of resource usage of QPs, CQs and counters used by virtual functions. This feature will be used to give the PF administrator more data while debugging VF status. Usage info was added to ALLOC_RES command, to notify the PF if the resource which is being reserved or allocated for the VF will be used by kernel driver or by user verbs. Updated reservation and allocation functions of QP, CQ and counter with additional usage parameter.

Moshe Shemesh committed 9 years ago

Commit f3301871 parent ea30b9611 files · +45 −20Browse files

Unified · Split

11 files
  1. drivers/infiniband/hw/mlx4/cq.c +2 −0
  2. drivers/infiniband/hw/mlx4/main.c +4 −2
  3. drivers/infiniband/hw/mlx4/qp.c +9 −4
  4. drivers/net/ethernet/mellanox/mlx4/cq.c +4 −3
  5. drivers/net/ethernet/mellanox/mlx4/en_cq.c +1 −0
  6. drivers/net/ethernet/mellanox/mlx4/en_netdev.c +2 −1
  7. drivers/net/ethernet/mellanox/mlx4/en_rx.c +4 −2
  8. drivers/net/ethernet/mellanox/mlx4/en_tx.c +2 −1
  9. drivers/net/ethernet/mellanox/mlx4/main.c +4 −3
  10. drivers/net/ethernet/mellanox/mlx4/qp.c +3 −2
  11. include/linux/mlx4/device.h +10 −2
drivers/infiniband/hw/mlx4/cq.c+2−0
@@ -218,6 +218,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, · 217 unmodified lines
218 goto err_mtt;218 goto err_mtt;
219 219
220 uar = &to_mucontext(context)->uar;220 uar = &to_mucontext(context)->uar;
221 cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
221 } else {222 } else {
222 err = mlx4_db_alloc(dev->dev, &cq->db, 1);223 err = mlx4_db_alloc(dev->dev, &cq->db, 1);
223 if (err)224 if (err)
@@ -233,6 +234,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, · 9 unmodified lines
233 goto err_db;234 goto err_db;
234 235
235 uar = &dev->priv_uar;236 uar = &dev->priv_uar;
237 cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
236 }238 }
237 239
238 if (dev->eq_table)240 if (dev->eq_table)
drivers/infiniband/hw/mlx4/main.c+4−2
@@ -2779,7 +2779,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) · 2778 unmodified lines
2779 allocated = 0;2779 allocated = 0;
2780 if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==2780 if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
2781 IB_LINK_LAYER_ETHERNET) {2781 IB_LINK_LAYER_ETHERNET) {
2782 err = mlx4_counter_alloc(ibdev->dev, &counter_index);2782 err = mlx4_counter_alloc(ibdev->dev, &counter_index,
2783 MLX4_RES_USAGE_DRIVER);
2783 /* if failed to allocate a new counter, use default */2784 /* if failed to allocate a new counter, use default */
2784 if (err)2785 if (err)
2785 counter_index =2786 counter_index =
@@ -2834,7 +2835,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) · 48 unmodified lines
2834 ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;2835 ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;
2835 err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,2836 err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,
2836 MLX4_IB_UC_STEER_QPN_ALIGN,2837 MLX4_IB_UC_STEER_QPN_ALIGN,
2837 &ibdev->steer_qpn_base, 0);2838 &ibdev->steer_qpn_base, 0,
2839 MLX4_RES_USAGE_DRIVER);
2838 if (err)2840 if (err)
2839 goto err_counter;2841 goto err_counter;
2840 2842
drivers/infiniband/hw/mlx4/qp.c+9−4
@@ -769,6 +769,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, · 768 unmodified lines
769 if (err)769 if (err)
770 goto err_mtt;770 goto err_mtt;
771 }771 }
772 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
772 } else {773 } else {
773 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,774 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
774 qp_has_rq(init_attr), qp, 0);775 qp_has_rq(init_attr), qp, 0);
@@ -841,6 +842,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, · 66 unmodified lines
841 err = -ENOMEM;842 err = -ENOMEM;
842 goto err_wrid;843 goto err_wrid;
843 }844 }
845 qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
844 }846 }
845 847
846 if (sqpn) {848 if (sqpn) {
@@ -860,13 +862,14 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd, · 13 unmodified lines
860 (init_attr->cap.max_send_wr ?862 (init_attr->cap.max_send_wr ?
861 MLX4_RESERVE_ETH_BF_QP : 0) |863 MLX4_RESERVE_ETH_BF_QP : 0) |
862 (init_attr->cap.max_recv_wr ?864 (init_attr->cap.max_recv_wr ?
863 MLX4_RESERVE_A0_QP : 0));865 MLX4_RESERVE_A0_QP : 0),
866 qp->mqp.usage);
864 else867 else
865 if (qp->flags & MLX4_IB_QP_NETIF)868 if (qp->flags & MLX4_IB_QP_NETIF)
866 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);869 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
867 else870 else
868 err = mlx4_qp_reserve_range(dev->dev, 1, 1,871 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
869 &qpn, 0);872 &qpn, 0, qp->mqp.usage);
870 if (err)873 if (err)
871 goto err_proxy;874 goto err_proxy;
872 }875 }
@@ -1218,7 +1221,9 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd, · 345 unmodified lines
1218 if (udata)1221 if (udata)
1219 return ERR_PTR(-EINVAL);1222 return ERR_PTR(-EINVAL);
1220 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {1223 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1221 int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0);1224 int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1225 1, 1, &sqpn, 0,
1226 MLX4_RES_USAGE_DRIVER);
1222 1227
1223 if (res)1228 if (res)
1224 return ERR_PTR(res);1229 return ERR_PTR(res);
@@ -1581,7 +1586,7 @@ static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp) · 356 unmodified lines
1581 !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))1586 !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
1582 return 0;1587 return 0;
1583 1588
1584 err = mlx4_counter_alloc(dev->dev, &tmp_idx);1589 err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
1585 if (err)1590 if (err)
1586 return err;1591 return err;
1587 1592
drivers/net/ethernet/mellanox/mlx4/cq.c+4−3
@@ -241,13 +241,14 @@ err_out: · 240 unmodified lines
241 return err;241 return err;
242}242}
243 243
244static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)244static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn, u8 usage)
245{245{
246 u32 in_modifier = RES_CQ | (((u32)usage & 3) << 30);
246 u64 out_param;247 u64 out_param;
247 int err;248 int err;
248 249
249 if (mlx4_is_mfunc(dev)) {250 if (mlx4_is_mfunc(dev)) {
250 err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,251 err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
251 RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,252 RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
252 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);253 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
253 if (err)254 if (err)
@@ -303,7 +304,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, · 49 unmodified lines
303 304
304 cq->vector = vector;305 cq->vector = vector;
305 306
306 err = mlx4_cq_alloc_icm(dev, &cq->cqn);307 err = mlx4_cq_alloc_icm(dev, &cq->cqn, cq->usage);
307 if (err)308 if (err)
308 return err;309 return err;
309 310
drivers/net/ethernet/mellanox/mlx4/en_cq.c+1−0
@@ -140,6 +140,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, · 139 unmodified lines
140 (cq->type == RX && priv->hwtstamp_config.rx_filter))140 (cq->type == RX && priv->hwtstamp_config.rx_filter))
141 timestamp_en = 1;141 timestamp_en = 1;
142 142
143 cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
143 err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,144 err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
144 &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,145 &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
145 cq->vector, 0, timestamp_en);146 cq->vector, 0, timestamp_en);
drivers/net/ethernet/mellanox/mlx4/en_netdev.c+2−1
@@ -651,7 +651,8 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv) · 650 unmodified lines
651 return 0;651 return 0;
652 }652 }
653 653
654 err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);654 err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP,
655 MLX4_RES_USAGE_DRIVER);
655 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);656 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
656 if (err) {657 if (err) {
657 en_err(priv, "Failed to reserve qp for mac registration\n");658 en_err(priv, "Failed to reserve qp for mac registration\n");
drivers/net/ethernet/mellanox/mlx4/en_rx.c+4−2
@@ -1081,7 +1081,8 @@ int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv) · 1080 unmodified lines
1081 u32 qpn;1081 u32 qpn;
1082 1082
1083 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,1083 err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
1084 MLX4_RESERVE_A0_QP);1084 MLX4_RESERVE_A0_QP,
1085 MLX4_RES_USAGE_DRIVER);
1085 if (err) {1086 if (err) {
1086 en_err(priv, "Failed reserving drop qpn\n");1087 en_err(priv, "Failed reserving drop qpn\n");
1087 return err;1088 return err;
@@ -1127,7 +1128,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv) · 39 unmodified lines
1127 flags = priv->rx_ring_num == 1 ? MLX4_RESERVE_A0_QP : 0;1128 flags = priv->rx_ring_num == 1 ? MLX4_RESERVE_A0_QP : 0;
1128 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,1129 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
1129 priv->rx_ring_num,1130 priv->rx_ring_num,
1130 &rss_map->base_qpn, flags);1131 &rss_map->base_qpn, flags,
1132 MLX4_RES_USAGE_DRIVER);
1131 if (err) {1133 if (err) {
1132 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);1134 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
1133 return err;1135 return err;
drivers/net/ethernet/mellanox/mlx4/en_tx.c+2−1
@@ -105,7 +105,8 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, · 104 unmodified lines
105 (unsigned long long) ring->sp_wqres.buf.direct.map);105 (unsigned long long) ring->sp_wqres.buf.direct.map);
106 106
107 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,107 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
108 MLX4_RESERVE_ETH_BF_QP);108 MLX4_RESERVE_ETH_BF_QP,
109 MLX4_RES_USAGE_DRIVER);
109 if (err) {110 if (err) {
110 en_err(priv, "failed reserving qp for TX ring\n");111 en_err(priv, "failed reserving qp for TX ring\n");
111 goto err_hwq_res;112 goto err_hwq_res;
drivers/net/ethernet/mellanox/mlx4/main.c+4−3
@@ -2475,7 +2475,7 @@ static int mlx4_allocate_default_counters(struct mlx4_dev *dev) · 2474 unmodified lines
2475 priv->def_counter[port] = -1;2475 priv->def_counter[port] = -1;
2476 2476
2477 for (port = 0; port < dev->caps.num_ports; port++) {2477 for (port = 0; port < dev->caps.num_ports; port++) {
2478 err = mlx4_counter_alloc(dev, &idx);2478 err = mlx4_counter_alloc(dev, &idx, MLX4_RES_USAGE_DRIVER);
2479 2479
2480 if (!err || err == -ENOSPC) {2480 if (!err || err == -ENOSPC) {
2481 priv->def_counter[port] = idx;2481 priv->def_counter[port] = idx;
@@ -2517,13 +2517,14 @@ int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx) · 35 unmodified lines
2517 return 0;2517 return 0;
2518}2518}
2519 2519
2520int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)2520int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage)
2521{2521{
2522 u32 in_modifier = RES_COUNTER | (((u32)usage & 3) << 30);
2522 u64 out_param;2523 u64 out_param;
2523 int err;2524 int err;
2524 2525
2525 if (mlx4_is_mfunc(dev)) {2526 if (mlx4_is_mfunc(dev)) {
2526 err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER,2527 err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
2527 RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,2528 RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,
2528 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);2529 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
2529 if (!err)2530 if (!err)
drivers/net/ethernet/mellanox/mlx4/qp.c+3−2
@@ -245,8 +245,9 @@ int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, · 244 unmodified lines
245}245}
246 246
247int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,247int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
248 int *base, u8 flags)248 int *base, u8 flags, u8 usage)
249{249{
250 u32 in_modifier = RES_QP | (((u32)usage & 3) << 30);
250 u64 in_param = 0;251 u64 in_param = 0;
251 u64 out_param;252 u64 out_param;
252 int err;253 int err;
@@ -258,7 +259,7 @@ int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, · 5 unmodified lines
258 set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt);259 set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt);
259 set_param_h(&in_param, align);260 set_param_h(&in_param, align);
260 err = mlx4_cmd_imm(dev, in_param, &out_param,261 err = mlx4_cmd_imm(dev, in_param, &out_param,
261 RES_QP, RES_OP_RESERVE,262 in_modifier, RES_OP_RESERVE,
262 MLX4_CMD_ALLOC_RES,263 MLX4_CMD_ALLOC_RES,
263 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);264 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
264 if (err)265 if (err)
include/linux/mlx4/device.h+10−2
@@ -428,6 +428,12 @@ enum mlx4_steer_type { · 427 unmodified lines
428 MLX4_NUM_STEERS428 MLX4_NUM_STEERS
429};429};
430 430
431enum mlx4_resource_usage {
432 MLX4_RES_USAGE_NONE,
433 MLX4_RES_USAGE_DRIVER,
434 MLX4_RES_USAGE_USER_VERBS,
435};
436
431enum {437enum {
432 MLX4_NUM_FEXCH = 64 * 1024,438 MLX4_NUM_FEXCH = 64 * 1024,
433};439};
@@ -748,6 +754,7 @@ struct mlx4_cq { · 314 unmodified lines
748 } tasklet_ctx;754 } tasklet_ctx;
749 int reset_notify_added;755 int reset_notify_added;
750 struct list_head reset_notify;756 struct list_head reset_notify;
757 u8 usage;
751};758};
752 759
753struct mlx4_qp {760struct mlx4_qp {
@@ -757,6 +764,7 @@ struct mlx4_qp { · 3 unmodified lines
757 764
758 atomic_t refcount;765 atomic_t refcount;
759 struct completion free;766 struct completion free;
767 u8 usage;
760};768};
761 769
762struct mlx4_srq {770struct mlx4_srq {
@@ -1120,7 +1128,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, · 357 unmodified lines
1120 unsigned vector, int collapsed, int timestamp_en);1128 unsigned vector, int collapsed, int timestamp_en);
1121void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq);1129void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq);
1122int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,1130int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
1123 int *base, u8 flags);1131 int *base, u8 flags, u8 usage);
1124void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt);1132void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt);
1125 1133
1126int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp);1134int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp);
@@ -1417,7 +1425,7 @@ int mlx4_get_phys_port_id(struct mlx4_dev *dev); · 290 unmodified lines
1417int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port);1425int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port);
1418int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port);1426int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port);
1419 1427
1420int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx);1428int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage);
1421void mlx4_counter_free(struct mlx4_dev *dev, u32 idx);1429void mlx4_counter_free(struct mlx4_dev *dev, u32 idx);
1422int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port);1430int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port);
1423 1431

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t