History

Merge tag 'mlx5-updates-2019-04-22' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says: ==================== mlx5-updates-2019-04-22 This series includes updates to mlx5e driver RX data path and some significant XDP RX/TX improvements to overcome/mitigate HW and PCIE bottlenecks. From Tariq: 1) Some Enhancements in rq->flags 2) Stabilize RX packet rate (on Striding RQ) with multiple outstanding UMR posts In this patch, we add support for multiple outstanding UMR posts, to allow faster gap closure between consuming MPWQEs and reposting them back into the WQ. Performance test: As expected, huge improvement in large-scale (48 cores). xdp_redirect_map, 64B UDP multi-stream. Redirect from ConnectX-5 100Gbps to ConnectX-6 100Gbps. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz. Before: Unstable, 7 to 30 Mpps After: Stable, at 70.5 Mpps From Shay: 3) XDP, Inline small packets into the TX MPWQE in XDP xmit flow Upon high packet rate with multiple CPUs TX workloads, much of the HCA's resources are spent on prefetching TX descriptors, thus affecting transmission rates. This patch comes to mitigate this problem by moving some workload to the CPU and reducing the HW data prefetch overhead for small packets (<= 256B). When forwarding packets with XDP, a packet that is smaller than a certain size (set to ~256 bytes) would be sent inline within its WQE TX descrptor (mem-copied), when the hardware tx queue is congested beyond a pre-defined water-mark. Performance: Tested packet rate for UDP 64Byte multi-stream over two dual port ConnectX-5 100Gbps NICs. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz * Tested with hyper-threading disabled XDP_TX: | | before | after | | | 24 rings | 51Mpps | 116Mpps | +126% | | 1 ring | 12Mpps | 12Mpps | same | XDP_REDIRECT: ** Below is the transmit rate, not the redirection rate which might be larger, and is not affected by this patch. | | before | after | | | 32 rings | 64Mpps | 92Mpps | +43% | | 1 ring | 6.4Mpps | 6.4Mpps | same | As we can see, feature significantly improves scaling, without hurting single ring performance. From Maxim: 4) Some trivial refactoring and code improvements prior to a larger series to support AF_XDP. ====================

David S. Miller committed 7 years ago

Commit 20eb08b2 parents 539b593 f8ebecf16 files · +406 −202Browse files

Unified · Split

16 files
  1. drivers/net/ethernet/mellanox/mlx5/core/Makefile +2 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/en.h +13 −7
  3. drivers/net/ethernet/mellanox/mlx5/core/en/params.c +104 −0
  4. drivers/net/ethernet/mellanox/mlx5/core/en/params.h +22 −0
  5. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +18 −12
  6. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +54 −3
  7. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +48 −124
  8. drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +87 −43
  9. drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +12 −3
  10. drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +6 −2
  11. drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +11 −0
  12. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +0 −5
  13. drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +10 −2
  14. drivers/net/ethernet/mellanox/mlx5/core/wq.h +12 −0
  15. include/linux/mlx5/fs.h +6 −0
  16. include/linux/mlx5/qp.h +1 −0
drivers/net/ethernet/mellanox/mlx5/core/Makefile+2−1
@@ -22,7 +22,8 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ · 21 unmodified lines
22#22#
23mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \23mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
24 en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \24 en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \
25 en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o25 en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \
26 en/params.o
26 27
27#28#
28# Netdev extra29# Netdev extra
drivers/net/ethernet/mellanox/mlx5/core/en.h+13−7
@@ -409,14 +409,17 @@ struct mlx5e_xdp_info_fifo { · 408 unmodified lines
409 409
410struct mlx5e_xdp_wqe_info {410struct mlx5e_xdp_wqe_info {
411 u8 num_wqebbs;411 u8 num_wqebbs;
412 u8 num_ds;412 u8 num_pkts;
413};413};
414 414
415struct mlx5e_xdp_mpwqe {415struct mlx5e_xdp_mpwqe {
416 /* Current MPWQE session */416 /* Current MPWQE session */
417 struct mlx5e_tx_wqe *wqe;417 struct mlx5e_tx_wqe *wqe;
418 u8 ds_count;418 u8 ds_count;
419 u8 pkt_count;
419 u8 max_ds_count;420 u8 max_ds_count;
421 u8 complete;
422 u8 inline_on;
420};423};
421 424
422struct mlx5e_xdpsq;425struct mlx5e_xdpsq;
@@ -428,7 +431,6 @@ struct mlx5e_xdpsq { · 5 unmodified lines
428 /* dirtied @completion */431 /* dirtied @completion */
429 u32 xdpi_fifo_cc;432 u32 xdpi_fifo_cc;
430 u16 cc;433 u16 cc;
431 bool redirect_flush;
432 434
433 /* dirtied @xmit */435 /* dirtied @xmit */
434 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;436 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
@@ -461,10 +463,10 @@ struct mlx5e_xdpsq { · 26 unmodified lines
461 463
462struct mlx5e_icosq {464struct mlx5e_icosq {
463 /* data path */465 /* data path */
466 u16 cc;
467 u16 pc;
464 468
465 /* dirtied @xmit */469 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
466 u16 pc ____cacheline_aligned_in_smp;
467
468 struct mlx5e_cq cq;470 struct mlx5e_cq cq;
469 471
470 /* write@xmit, read@completion */472 /* write@xmit, read@completion */
@@ -531,7 +533,8 @@ typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); · 60 unmodified lines
531typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);533typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
532 534
533enum mlx5e_rq_flag {535enum mlx5e_rq_flag {
534 MLX5E_RQ_FLAG_XDP_XMIT = BIT(0),536 MLX5E_RQ_FLAG_XDP_XMIT,
537 MLX5E_RQ_FLAG_XDP_REDIRECT,
535};538};
536 539
537struct mlx5e_rq_frag_info {540struct mlx5e_rq_frag_info {
@@ -562,8 +565,10 @@ struct mlx5e_rq { · 24 unmodified lines
562 struct mlx5e_mpw_info *info;565 struct mlx5e_mpw_info *info;
563 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;566 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
564 u16 num_strides;567 u16 num_strides;
568 u16 actual_wq_head;
565 u8 log_stride_sz;569 u8 log_stride_sz;
566 bool umr_in_progress;570 u8 umr_in_progress;
571 u8 umr_last_bulk;
567 } mpwqe;572 } mpwqe;
568 };573 };
569 struct {574 struct {
@@ -773,6 +778,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); · 203 unmodified lines
773netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,778netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
774 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more);779 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more);
775 780
781void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
776void mlx5e_completion_event(struct mlx5_core_cq *mcq);782void mlx5e_completion_event(struct mlx5_core_cq *mcq);
777void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);783void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
778int mlx5e_napi_poll(struct napi_struct *napi, int budget);784int mlx5e_napi_poll(struct napi_struct *napi, int budget);
drivers/net/ethernet/mellanox/mlx5/core/en/params.c+104−0
@@ -0,0 +1,104 @@
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#include "en/params.h"
5
6u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
7{
8 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
9 u16 linear_rq_headroom = params->xdp_prog ?
10 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
11 u32 frag_sz;
12
13 linear_rq_headroom += NET_IP_ALIGN;
14
15 frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
16
17 if (params->xdp_prog && frag_sz < PAGE_SIZE)
18 frag_sz = PAGE_SIZE;
19
20 return frag_sz;
21}
22
23u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
24{
25 u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
26
27 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
28}
29
30bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params)
31{
32 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
33
34 return !params->lro_en && frag_sz <= PAGE_SIZE;
35}
36
37#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \
38 MLX5_MPWQE_LOG_STRIDE_SZ_BASE)
39bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
40 struct mlx5e_params *params)
41{
42 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
43 s8 signed_log_num_strides_param;
44 u8 log_num_strides;
45
46 if (!mlx5e_rx_is_linear_skb(params))
47 return false;
48
49 if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)
50 return false;
51
52 if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
53 return true;
54
55 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
56 signed_log_num_strides_param =
57 (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
58
59 return signed_log_num_strides_param >= 0;
60}
61
62u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
63{
64 u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params);
65
66 /* Numbers are unsigned, don't subtract to avoid underflow. */
67 if (params->log_rq_mtu_frames <
68 log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
69 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
70
71 return params->log_rq_mtu_frames - log_pkts_per_wqe;
72}
73
74u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
75 struct mlx5e_params *params)
76{
77 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
78 return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
79
80 return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
81}
82
83u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
84 struct mlx5e_params *params)
85{
86 return MLX5_MPWRQ_LOG_WQE_SZ -
87 mlx5e_mpwqe_get_log_stride_size(mdev, params);
88}
89
90u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
91 struct mlx5e_params *params)
92{
93 u16 linear_rq_headroom = params->xdp_prog ?
94 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
95 bool is_linear_skb;
96
97 linear_rq_headroom += NET_IP_ALIGN;
98
99 is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
100 mlx5e_rx_is_linear_skb(params) :
101 mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
102
103 return is_linear_skb ? linear_rq_headroom : 0;
104}
drivers/net/ethernet/mellanox/mlx5/core/en/params.h+22−0
@@ -0,0 +1,22 @@
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#ifndef __MLX5_EN_PARAMS_H__
5#define __MLX5_EN_PARAMS_H__
6
7#include "en.h"
8
9u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params);
10u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params);
11bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params);
12bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
13 struct mlx5e_params *params);
14u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params);
15u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
16 struct mlx5e_params *params);
17u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
18 struct mlx5e_params *params);
19u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
20 struct mlx5e_params *params);
21
22#endif /* __MLX5_EN_PARAMS_H__ */
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+18−12
@@ -85,7 +85,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, · 84 unmodified lines
85 if (unlikely(err))85 if (unlikely(err))
86 goto xdp_abort;86 goto xdp_abort;
87 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);87 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
88 rq->xdpsq.redirect_flush = true;88 __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
89 mlx5e_page_dma_unmap(rq, di);89 mlx5e_page_dma_unmap(rq, di);
90 rq->stats->xdp_redirect++;90 rq->stats->xdp_redirect++;
91 return true;91 return true;
@@ -105,6 +105,7 @@ xdp_abort: · 13 unmodified lines
105static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq)105static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq)
106{106{
107 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;107 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
108 struct mlx5e_xdpsq_stats *stats = sq->stats;
108 struct mlx5_wq_cyc *wq = &sq->wq;109 struct mlx5_wq_cyc *wq = &sq->wq;
109 u8 wqebbs;110 u8 wqebbs;
110 u16 pi;111 u16 pi;
@@ -112,7 +113,9 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) · 1 unmodified line
112 mlx5e_xdpsq_fetch_wqe(sq, &session->wqe);113 mlx5e_xdpsq_fetch_wqe(sq, &session->wqe);
113 114
114 prefetchw(session->wqe->data);115 prefetchw(session->wqe->data);
115 session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT;116 session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT;
117 session->pkt_count = 0;
118 session->complete = 0;
116 119
117 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);120 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
118 121
@@ -131,6 +134,10 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) · 12 unmodified lines
131 MLX5E_XDP_MPW_MAX_WQEBBS);134 MLX5E_XDP_MPW_MAX_WQEBBS);
132 135
133 session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs;136 session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs;
137
138 mlx5e_xdp_update_inline_state(sq);
139
140 stats->mpwqe++;
134}141}
135 142
136static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq)143static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq)
@@ -147,7 +154,7 @@ static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) · 10 unmodified lines
147 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count);154 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count);
148 155
149 wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS);156 wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS);
150 wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT;157 wi->num_pkts = session->pkt_count;
151 158
152 sq->pc += wi->num_wqebbs;159 sq->pc += wi->num_wqebbs;
153 160
@@ -162,11 +169,9 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, · 8 unmodified lines
162 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;169 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
163 struct mlx5e_xdpsq_stats *stats = sq->stats;170 struct mlx5e_xdpsq_stats *stats = sq->stats;
164 171
165 dma_addr_t dma_addr = xdpi->dma_addr;
166 struct xdp_frame *xdpf = xdpi->xdpf;172 struct xdp_frame *xdpf = xdpi->xdpf;
167 unsigned int dma_len = xdpf->len;
168 173
169 if (unlikely(sq->hw_mtu < dma_len)) {174 if (unlikely(sq->hw_mtu < xdpf->len)) {
170 stats->err++;175 stats->err++;
171 return false;176 return false;
172 }177 }
@@ -183,9 +188,10 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, · 10 unmodified lines
183 mlx5e_xdp_mpwqe_session_start(sq);188 mlx5e_xdp_mpwqe_session_start(sq);
184 }189 }
185 190
186 mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len);191 mlx5e_xdp_mpwqe_add_dseg(sq, xdpi, stats);
187 192
188 if (unlikely(session->ds_count == session->max_ds_count))193 if (unlikely(session->complete ||
194 session->ds_count == session->max_ds_count))
189 mlx5e_xdp_mpwqe_complete(sq);195 mlx5e_xdp_mpwqe_complete(sq);
190 196
191 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);197 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
@@ -299,7 +305,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) · 107 unmodified lines
299 305
300 sqcc += wi->num_wqebbs;306 sqcc += wi->num_wqebbs;
301 307
302 for (j = 0; j < wi->num_ds; j++) {308 for (j = 0; j < wi->num_pkts; j++) {
303 struct mlx5e_xdp_info xdpi =309 struct mlx5e_xdp_info xdpi =
304 mlx5e_xdpi_fifo_pop(xdpi_fifo);310 mlx5e_xdpi_fifo_pop(xdpi_fifo);
305 311
@@ -340,7 +346,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) · 34 unmodified lines
340 346
341 sq->cc += wi->num_wqebbs;347 sq->cc += wi->num_wqebbs;
342 348
343 for (i = 0; i < wi->num_ds; i++) {349 for (i = 0; i < wi->num_pkts; i++) {
344 struct mlx5e_xdp_info xdpi =350 struct mlx5e_xdp_info xdpi =
345 mlx5e_xdpi_fifo_pop(xdpi_fifo);351 mlx5e_xdpi_fifo_pop(xdpi_fifo);
346 352
@@ -419,9 +425,9 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) · 72 unmodified lines
419 425
420 mlx5e_xmit_xdp_doorbell(xdpsq);426 mlx5e_xmit_xdp_doorbell(xdpsq);
421 427
422 if (xdpsq->redirect_flush) {428 if (test_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags)) {
423 xdp_do_flush_map();429 xdp_do_flush_map();
424 xdpsq->redirect_flush = false;430 __clear_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
425 }431 }
426}432}
427 433
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+54−3
@@ -75,16 +75,68 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) · 74 unmodified lines
75 }75 }
76}76}
77 77
78/* Enable inline WQEs to shift some load from a congested HCA (HW) to
79 * a less congested cpu (SW).
80 */
81static inline void mlx5e_xdp_update_inline_state(struct mlx5e_xdpsq *sq)
82{
83 u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc;
84 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
85
86#define MLX5E_XDP_INLINE_WATERMARK_LOW 10
87#define MLX5E_XDP_INLINE_WATERMARK_HIGH 128
88
89 if (session->inline_on) {
90 if (outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW)
91 session->inline_on = 0;
92 return;
93 }
94
95 /* inline is false */
96 if (outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH)
97 session->inline_on = 1;
98}
99
78static inline void100static inline void
79mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, dma_addr_t dma_addr, u16 dma_len)101mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi,
102 struct mlx5e_xdpsq_stats *stats)
80{103{
81 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;104 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
105 dma_addr_t dma_addr = xdpi->dma_addr;
106 struct xdp_frame *xdpf = xdpi->xdpf;
82 struct mlx5_wqe_data_seg *dseg =107 struct mlx5_wqe_data_seg *dseg =
83 (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count++;108 (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
109 u16 dma_len = xdpf->len;
84 110
111 session->pkt_count++;
112
113#define MLX5E_XDP_INLINE_WQE_SZ_THRSD (256 - sizeof(struct mlx5_wqe_inline_seg))
114
115 if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) {
116 struct mlx5_wqe_inline_seg *inline_dseg =
117 (struct mlx5_wqe_inline_seg *)dseg;
118 u16 ds_len = sizeof(*inline_dseg) + dma_len;
119 u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS);
120
121 if (unlikely(session->ds_count + ds_cnt > session->max_ds_count)) {
122 /* Not enough space for inline wqe, send with memory pointer */
123 session->complete = true;
124 goto no_inline;
125 }
126
127 inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG);
128 memcpy(inline_dseg->data, xdpf->data, dma_len);
129
130 session->ds_count += ds_cnt;
131 stats->inlnw++;
132 return;
133 }
134
135no_inline:
85 dseg->addr = cpu_to_be64(dma_addr);136 dseg->addr = cpu_to_be64(dma_addr);
86 dseg->byte_count = cpu_to_be32(dma_len);137 dseg->byte_count = cpu_to_be32(dma_len);
87 dseg->lkey = sq->mkey_be;138 dseg->lkey = sq->mkey_be;
139 session->ds_count++;
88}140}
89 141
90static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq,142static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq,
@@ -111,5 +163,4 @@ mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo) · 20 unmodified lines
111{163{
112 return fifo->xi[(*fifo->cc)++ & fifo->mask];164 return fifo->xi[(*fifo->cc)++ & fifo->mask];
113}165}
114
115#endif166#endif
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+48−124
@@ -55,6 +55,7 @@ · 54 unmodified lines
55#include "lib/eq.h"55#include "lib/eq.h"
56#include "en/monitor_stats.h"56#include "en/monitor_stats.h"
57#include "en/reporter.h"57#include "en/reporter.h"
58#include "en/params.h"
58 59
59struct mlx5e_rq_param {60struct mlx5e_rq_param {
60 u32 rqc[MLX5_ST_SZ_DW(rqc)];61 u32 rqc[MLX5_ST_SZ_DW(rqc)];
@@ -103,104 +104,6 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) · 42 unmodified lines
103 return true;104 return true;
104}105}
105 106
106static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
107{
108 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
109 u16 linear_rq_headroom = params->xdp_prog ?
110 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
111 u32 frag_sz;
112
113 linear_rq_headroom += NET_IP_ALIGN;
114
115 frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
116
117 if (params->xdp_prog && frag_sz < PAGE_SIZE)
118 frag_sz = PAGE_SIZE;
119
120 return frag_sz;
121}
122
123static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
124{
125 u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
126
127 return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
128}
129
130static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
131 struct mlx5e_params *params)
132{
133 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
134
135 return !params->lro_en && frag_sz <= PAGE_SIZE;
136}
137
138#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \
139 MLX5_MPWQE_LOG_STRIDE_SZ_BASE)
140static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
141 struct mlx5e_params *params)
142{
143 u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
144 s8 signed_log_num_strides_param;
145 u8 log_num_strides;
146
147 if (!mlx5e_rx_is_linear_skb(mdev, params))
148 return false;
149
150 if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)
151 return false;
152
153 if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
154 return true;
155
156 log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
157 signed_log_num_strides_param =
158 (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
159
160 return signed_log_num_strides_param >= 0;
161}
162
163static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
164{
165 if (params->log_rq_mtu_frames <
166 mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
167 return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
168
169 return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params);
170}
171
172static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
173 struct mlx5e_params *params)
174{
175 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
176 return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
177
178 return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
179}
180
181static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
182 struct mlx5e_params *params)
183{
184 return MLX5_MPWRQ_LOG_WQE_SZ -
185 mlx5e_mpwqe_get_log_stride_size(mdev, params);
186}
187
188static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
189 struct mlx5e_params *params)
190{
191 u16 linear_rq_headroom = params->xdp_prog ?
192 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
193 bool is_linear_skb;
194
195 linear_rq_headroom += NET_IP_ALIGN;
196
197 is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
198 mlx5e_rx_is_linear_skb(mdev, params) :
199 mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
200
201 return is_linear_skb ? linear_rq_headroom : 0;
202}
203
204void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,107void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
205 struct mlx5e_params *params)108 struct mlx5e_params *params)
206{109{
@@ -470,7 +373,6 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) · 263 unmodified lines
470}373}
471 374
472static int mlx5e_init_di_list(struct mlx5e_rq *rq,375static int mlx5e_init_di_list(struct mlx5e_rq *rq,
473 struct mlx5e_params *params,
474 int wq_sz, int cpu)376 int wq_sz, int cpu)
475{377{
476 int len = wq_sz << rq->wqe.info.log_num_frags;378 int len = wq_sz << rq->wqe.info.log_num_frags;
@@ -598,7 +500,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 121 unmodified lines
598 goto err_free;500 goto err_free;
599 }501 }
600 502
601 err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu);503 err = mlx5e_init_di_list(rq, wq_sz, c->cpu);
602 if (err)504 if (err)
603 goto err_free;505 goto err_free;
604 rq->post_wqes = mlx5e_post_rx_wqes;506 rq->post_wqes = mlx5e_post_rx_wqes;
@@ -616,7 +518,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 11 unmodified lines
616 goto err_free;518 goto err_free;
617 }519 }
618 520
619 rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ?521 rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params) ?
620 mlx5e_skb_from_cqe_linear :522 mlx5e_skb_from_cqe_linear :
621 mlx5e_skb_from_cqe_nonlinear;523 mlx5e_skb_from_cqe_nonlinear;
622 rq->mkey_be = c->mkey_be;524 rq->mkey_be = c->mkey_be;
@@ -903,10 +805,14 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) · 280 unmodified lines
903 805
904 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {806 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
905 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;807 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
808 u16 head = wq->head;
809 int i;
906 810
907 /* UMR WQE (if in progress) is always at wq->head */811 /* Outstanding UMR WQEs (in progress) start at wq->head */
908 if (rq->mpwqe.umr_in_progress)812 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
909 rq->dealloc_wqe(rq, wq->head);813 rq->dealloc_wqe(rq, head);
814 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
815 }
910 816
911 while (!mlx5_wq_ll_is_empty(wq)) {817 while (!mlx5_wq_ll_is_empty(wq)) {
912 struct mlx5e_rx_wqe_ll *wqe;818 struct mlx5e_rx_wqe_ll *wqe;
@@ -971,16 +877,8 @@ err_free_rq: · 58 unmodified lines
971 877
972static void mlx5e_activate_rq(struct mlx5e_rq *rq)878static void mlx5e_activate_rq(struct mlx5e_rq *rq)
973{879{
974 struct mlx5e_icosq *sq = &rq->channel->icosq;
975 struct mlx5_wq_cyc *wq = &sq->wq;
976 struct mlx5e_tx_wqe *nopwqe;
977
978 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
979
980 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);880 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
981 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;881 mlx5e_trigger_irq(&rq->channel->icosq);
982 nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
983 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
984}882}
985 883
986static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)884static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
@@ -1092,7 +990,7 @@ static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) · 105 unmodified lines
1092 990
1093static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)991static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1094{992{
1095 u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);993 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1096 994
1097 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz,995 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz,
1098 sizeof(*sq->db.ico_wqe)),996 sizeof(*sq->db.ico_wqe)),
@@ -1528,7 +1426,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 429 unmodified lines
1528 dseg->lkey = sq->mkey_be;1426 dseg->lkey = sq->mkey_be;
1529 1427
1530 wi->num_wqebbs = 1;1428 wi->num_wqebbs = 1;
1531 wi->num_ds = 1;1429 wi->num_pkts = 1;
1532 }1430 }
1533 }1431 }
1534 1432
@@ -2054,7 +1952,7 @@ static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, · 519 unmodified lines
2054 byte_count += MLX5E_METADATA_ETHER_LEN;1952 byte_count += MLX5E_METADATA_ETHER_LEN;
2055#endif1953#endif
2056 1954
2057 if (mlx5e_rx_is_linear_skb(mdev, params)) {1955 if (mlx5e_rx_is_linear_skb(params)) {
2058 int frag_stride;1956 int frag_stride;
2059 1957
2060 frag_stride = mlx5e_rx_get_linear_frag_sz(params);1958 frag_stride = mlx5e_rx_get_linear_frag_sz(params);
@@ -2108,6 +2006,13 @@ static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) · 47 unmodified lines
2108 return order_base_2(sz);2006 return order_base_2(sz);
2109}2007}
2110 2008
2009static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
2010{
2011 void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2012
2013 return MLX5_GET(wq, wq, log_wq_sz);
2014}
2015
2111static void mlx5e_build_rq_param(struct mlx5e_priv *priv,2016static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
2112 struct mlx5e_params *params,2017 struct mlx5e_params *params,
2113 struct mlx5e_rq_param *param)2018 struct mlx5e_rq_param *param)
@@ -2274,13 +2179,28 @@ static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, · 160 unmodified lines
2274 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);2179 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
2275}2180}
2276 2181
2182static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params,
2183 struct mlx5e_rq_param *rqp)
2184{
2185 switch (params->rq_wq_type) {
2186 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2187 return order_base_2(MLX5E_UMR_WQEBBS) +
2188 mlx5e_get_rq_log_wq_sz(rqp->rqc);
2189 default: /* MLX5_WQ_TYPE_CYCLIC */
2190 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2191 }
2192}
2193
2277static void mlx5e_build_channel_param(struct mlx5e_priv *priv,2194static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
2278 struct mlx5e_params *params,2195 struct mlx5e_params *params,
2279 struct mlx5e_channel_param *cparam)2196 struct mlx5e_channel_param *cparam)
2280{2197{
2281 u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;2198 u8 icosq_log_wq_sz;
2282 2199
2283 mlx5e_build_rq_param(priv, params, &cparam->rq);2200 mlx5e_build_rq_param(priv, params, &cparam->rq);
2201
2202 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);
2203
2284 mlx5e_build_sq_param(priv, params, &cparam->sq);2204 mlx5e_build_sq_param(priv, params, &cparam->sq);
2285 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);2205 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2286 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);2206 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
@@ -2336,14 +2256,18 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs) · 49 unmodified lines
2336 mlx5e_activate_channel(chs->c[i]);2256 mlx5e_activate_channel(chs->c[i]);
2337}2257}
2338 2258
2259#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2260
2339static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)2261static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2340{2262{
2341 int err = 0;2263 int err = 0;
2342 int i;2264 int i;
2343 2265
2344 for (i = 0; i < chs->num; i++)2266 for (i = 0; i < chs->num; i++) {
2345 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq,2267 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2346 err ? 0 : 20000);2268
2269 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
2270 }
2347 2271
2348 return err ? -ETIMEDOUT : 0;2272 return err ? -ETIMEDOUT : 0;
2349}2273}
@@ -3794,7 +3718,7 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, · 1444 unmodified lines
3794 new_channels.params.sw_mtu = new_mtu;3718 new_channels.params.sw_mtu = new_mtu;
3795 3719
3796 if (params->xdp_prog &&3720 if (params->xdp_prog &&
3797 !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {3721 !mlx5e_rx_is_linear_skb(&new_channels.params)) {
3798 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",3722 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3799 new_mtu, MLX5E_XDP_MAX_MTU);3723 new_mtu, MLX5E_XDP_MAX_MTU);
3800 err = -EINVAL;3724 err = -EINVAL;
@@ -4235,7 +4159,7 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) · 434 unmodified lines
4235 new_channels.params = priv->channels.params;4159 new_channels.params = priv->channels.params;
4236 new_channels.params.xdp_prog = prog;4160 new_channels.params.xdp_prog = prog;
4237 4161
4238 if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {4162 if (!mlx5e_rx_is_linear_skb(&new_channels.params)) {
4239 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",4163 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4240 new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);4164 new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
4241 return -EINVAL;4165 return -EINVAL;
@@ -4288,7 +4212,7 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) · 46 unmodified lines
4288 mlx5e_set_rq_type(priv->mdev, &priv->channels.params);4212 mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
4289 4213
4290 if (was_opened && reset)4214 if (was_opened && reset)
4291 mlx5e_open_locked(netdev);4215 err = mlx5e_open_locked(netdev);
4292 4216
4293 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)4217 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4294 goto unlock;4218 goto unlock;
@@ -4578,7 +4502,7 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, · 283 unmodified lines
4578 if (!slow_pci_heuristic(mdev) &&4502 if (!slow_pci_heuristic(mdev) &&
4579 mlx5e_striding_rq_possible(mdev, params) &&4503 mlx5e_striding_rq_possible(mdev, params) &&
4580 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ||4504 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ||
4581 !mlx5e_rx_is_linear_skb(mdev, params)))4505 !mlx5e_rx_is_linear_skb(params)))
4582 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);4506 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
4583 mlx5e_set_rq_type(mdev, params);4507 mlx5e_set_rq_type(mdev, params);
4584 mlx5e_init_rq_type_params(mdev, params);4508 mlx5e_init_rq_type_params(mdev, params);
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c+87−43
@@ -409,14 +409,15 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle · 408 unmodified lines
409 mlx5e_page_release(rq, &dma_info[i], recycle);409 mlx5e_page_release(rq, &dma_info[i], recycle);
410}410}
411 411
412static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)412static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
413{413{
414 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;414 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
415 struct mlx5e_rx_wqe_ll *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
416 415
417 rq->mpwqe.umr_in_progress = false;416 do {
417 u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head);
418 418
419 mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));419 mlx5_wq_ll_push(wq, next_wqe_index);
420 } while (--n);
420 421
421 /* ensure wqes are visible to device before updating doorbell record */422 /* ensure wqes are visible to device before updating doorbell record */
422 dma_wmb();423 dma_wmb();
@@ -426,7 +427,7 @@ static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq) · 3 unmodified lines
426 427
427static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)428static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)
428{429{
429 return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;430 return mlx5_wq_cyc_get_ctr_wrap_cnt(&sq->wq, sq->pc);
430}431}
431 432
432static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,433static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,
@@ -478,8 +479,6 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) · 45 unmodified lines
478 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);479 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
479 wi->consumed_strides = 0;480 wi->consumed_strides = 0;
480 481
481 rq->mpwqe.umr_in_progress = true;
482
483 umr_wqe->ctrl.opmod_idx_opcode =482 umr_wqe->ctrl.opmod_idx_opcode =
484 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |483 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
485 MLX5_OPCODE_UMR);484 MLX5_OPCODE_UMR);
@@ -487,7 +486,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) · 1 unmodified line
487 486
488 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;487 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
489 sq->pc += MLX5E_UMR_WQEBBS;488 sq->pc += MLX5E_UMR_WQEBBS;
490 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &umr_wqe->ctrl);489
490 sq->doorbell_cseg = &umr_wqe->ctrl;
491 491
492 return 0;492 return 0;
493 493
@@ -542,37 +542,13 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) · 48 unmodified lines
542 return !!err;542 return !!err;
543}543}
544 544
545static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
546 struct mlx5e_icosq *sq,
547 struct mlx5e_rq *rq,
548 struct mlx5_cqe64 *cqe)
549{
550 struct mlx5_wq_cyc *wq = &sq->wq;
551 u16 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
552 struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
553
554 mlx5_cqwq_pop(&cq->wq);
555
556 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
557 netdev_WARN_ONCE(cq->channel->netdev,
558 "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe));
559 return;
560 }
561
562 if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
563 mlx5e_post_rx_mpwqe(rq);
564 return;
565 }
566
567 if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
568 netdev_WARN_ONCE(cq->channel->netdev,
569 "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi->opcode);
570}
571
572static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)545static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
573{546{
574 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);547 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
575 struct mlx5_cqe64 *cqe;548 struct mlx5_cqe64 *cqe;
549 u8 completed_umr = 0;
550 u16 sqcc;
551 int i;
576 552
577 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))553 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
578 return;554 return;
@@ -581,28 +557,96 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) · 2 unmodified lines
581 if (likely(!cqe))557 if (likely(!cqe))
582 return;558 return;
583 559
584 /* by design, there's only a single cqe */560 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
585 mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe);561 * otherwise a cq overrun may occur
562 */
563 sqcc = sq->cc;
564
565 i = 0;
566 do {
567 u16 wqe_counter;
568 bool last_wqe;
569
570 mlx5_cqwq_pop(&cq->wq);
571
572 wqe_counter = be16_to_cpu(cqe->wqe_counter);
573
574 if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
575 netdev_WARN_ONCE(cq->channel->netdev,
576 "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe));
577 break;
578 }
579 do {
580 struct mlx5e_sq_wqe_info *wi;
581 u16 ci;
582
583 last_wqe = (sqcc == wqe_counter);
584
585 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
586 wi = &sq->db.ico_wqe[ci];
587
588 if (likely(wi->opcode == MLX5_OPCODE_UMR)) {
589 sqcc += MLX5E_UMR_WQEBBS;
590 completed_umr++;
591 } else if (likely(wi->opcode == MLX5_OPCODE_NOP)) {
592 sqcc++;
593 } else {
594 netdev_WARN_ONCE(cq->channel->netdev,
595 "Bad OPCODE in ICOSQ WQE info: 0x%x\n",
596 wi->opcode);
597 }
598
599 } while (!last_wqe);
600
601 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
602
603 sq->cc = sqcc;
586 604
587 mlx5_cqwq_update_db_record(&cq->wq);605 mlx5_cqwq_update_db_record(&cq->wq);
606
607 if (likely(completed_umr)) {
608 mlx5e_post_rx_mpwqe(rq, completed_umr);
609 rq->mpwqe.umr_in_progress -= completed_umr;
610 }
588}611}
589 612
590bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)613bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
591{614{
615 struct mlx5e_icosq *sq = &rq->channel->icosq;
592 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;616 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
617 u8 missing, i;
618 u16 head;
593 619
594 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))620 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
595 return false;621 return false;
596 622
597 mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);623 mlx5e_poll_ico_cq(&sq->cq, rq);
624
625 missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
598 626
599 if (mlx5_wq_ll_is_full(wq))627 if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk))
628 rq->stats->congst_umr++;
629
630#define UMR_WQE_BULK (2)
631 if (likely(missing < UMR_WQE_BULK))
600 return false;632 return false;
601 633
602 if (!rq->mpwqe.umr_in_progress)634 head = rq->mpwqe.actual_wq_head;
603 mlx5e_alloc_rx_mpwqe(rq, wq->head);635 i = missing;
604 else636 do {
605 rq->stats->congst_umr += mlx5_wq_ll_missing(wq) > 2;637 if (unlikely(mlx5e_alloc_rx_mpwqe(rq, head)))
638 break;
639 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
640 } while (--i);
641
642 rq->mpwqe.umr_last_bulk = missing - i;
643 if (sq->doorbell_cseg) {
644 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
645 sq->doorbell_cseg = NULL;
646 }
647
648 rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
649 rq->mpwqe.actual_wq_head = head;
606 650
607 return false;651 return false;
608}652}
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c+12−3
@@ -65,6 +65,8 @@ static const struct counter_desc sw_stats_desc[] = { · 64 unmodified lines
65 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },65 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
66 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },66 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },
67 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },67 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },
68 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_mpwqe) },
69 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_inlnw) },
68 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },70 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
69 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },71 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },
70 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },72 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
@@ -79,6 +81,8 @@ static const struct counter_desc sw_stats_desc[] = { · 8 unmodified lines
79 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },81 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
80 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },82 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },
81 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },83 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },
84 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_mpwqe) },
85 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_inlnw) },
82 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },86 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },
83 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },87 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },
84 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },88 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },
@@ -89,7 +93,6 @@ static const struct counter_desc sw_stats_desc[] = { · 4 unmodified lines
89 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },93 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
90 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },94 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
91 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },95 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
92 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_page_reuse) },
93 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },96 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },
94 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },97 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
95 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },98 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
@@ -160,6 +163,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 64 unmodified lines
160 s->rx_xdp_drop += rq_stats->xdp_drop;163 s->rx_xdp_drop += rq_stats->xdp_drop;
161 s->rx_xdp_redirect += rq_stats->xdp_redirect;164 s->rx_xdp_redirect += rq_stats->xdp_redirect;
162 s->rx_xdp_tx_xmit += xdpsq_stats->xmit;165 s->rx_xdp_tx_xmit += xdpsq_stats->xmit;
166 s->rx_xdp_tx_mpwqe += xdpsq_stats->mpwqe;
167 s->rx_xdp_tx_inlnw += xdpsq_stats->inlnw;
163 s->rx_xdp_tx_full += xdpsq_stats->full;168 s->rx_xdp_tx_full += xdpsq_stats->full;
164 s->rx_xdp_tx_err += xdpsq_stats->err;169 s->rx_xdp_tx_err += xdpsq_stats->err;
165 s->rx_xdp_tx_cqe += xdpsq_stats->cqes;170 s->rx_xdp_tx_cqe += xdpsq_stats->cqes;
@@ -170,7 +175,6 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 4 unmodified lines
170 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;175 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
171 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;176 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
172 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;177 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
173 s->rx_page_reuse += rq_stats->page_reuse;
174 s->rx_cache_reuse += rq_stats->cache_reuse;178 s->rx_cache_reuse += rq_stats->cache_reuse;
175 s->rx_cache_full += rq_stats->cache_full;179 s->rx_cache_full += rq_stats->cache_full;
176 s->rx_cache_empty += rq_stats->cache_empty;180 s->rx_cache_empty += rq_stats->cache_empty;
@@ -185,6 +189,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 8 unmodified lines
185 s->ch_eq_rearm += ch_stats->eq_rearm;189 s->ch_eq_rearm += ch_stats->eq_rearm;
186 /* xdp redirect */190 /* xdp redirect */
187 s->tx_xdp_xmit += xdpsq_red_stats->xmit;191 s->tx_xdp_xmit += xdpsq_red_stats->xmit;
192 s->tx_xdp_mpwqe += xdpsq_red_stats->mpwqe;
193 s->tx_xdp_inlnw += xdpsq_red_stats->inlnw;
188 s->tx_xdp_full += xdpsq_red_stats->full;194 s->tx_xdp_full += xdpsq_red_stats->full;
189 s->tx_xdp_err += xdpsq_red_stats->err;195 s->tx_xdp_err += xdpsq_red_stats->err;
190 s->tx_xdp_cqes += xdpsq_red_stats->cqes;196 s->tx_xdp_cqes += xdpsq_red_stats->cqes;
@@ -1212,7 +1218,6 @@ static const struct counter_desc rq_stats_desc[] = { · 1021 unmodified lines
1212 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },1218 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
1213 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },1219 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
1214 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },1220 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
1215 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, page_reuse) },
1216 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },1221 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },
1217 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },1222 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
1218 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },1223 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
@@ -1245,6 +1250,8 @@ static const struct counter_desc sq_stats_desc[] = { · 26 unmodified lines
1245 1250
1246static const struct counter_desc rq_xdpsq_stats_desc[] = {1251static const struct counter_desc rq_xdpsq_stats_desc[] = {
1247 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },1252 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1253 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1254 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
1248 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },1255 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1249 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },1256 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1250 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },1257 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
@@ -1252,6 +1259,8 @@ static const struct counter_desc rq_xdpsq_stats_desc[] = { · 1 unmodified line
1252 1259
1253static const struct counter_desc xdpsq_stats_desc[] = {1260static const struct counter_desc xdpsq_stats_desc[] = {
1254 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },1261 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1262 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1263 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
1255 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },1264 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1256 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },1265 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1257 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },1266 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h+6−2
@@ -77,6 +77,8 @@ struct mlx5e_sw_stats { · 76 unmodified lines
77 u64 rx_xdp_drop;77 u64 rx_xdp_drop;
78 u64 rx_xdp_redirect;78 u64 rx_xdp_redirect;
79 u64 rx_xdp_tx_xmit;79 u64 rx_xdp_tx_xmit;
80 u64 rx_xdp_tx_mpwqe;
81 u64 rx_xdp_tx_inlnw;
80 u64 rx_xdp_tx_full;82 u64 rx_xdp_tx_full;
81 u64 rx_xdp_tx_err;83 u64 rx_xdp_tx_err;
82 u64 rx_xdp_tx_cqe;84 u64 rx_xdp_tx_cqe;
@@ -91,6 +93,8 @@ struct mlx5e_sw_stats { · 8 unmodified lines
91 u64 tx_queue_wake;93 u64 tx_queue_wake;
92 u64 tx_cqe_err;94 u64 tx_cqe_err;
93 u64 tx_xdp_xmit;95 u64 tx_xdp_xmit;
96 u64 tx_xdp_mpwqe;
97 u64 tx_xdp_inlnw;
94 u64 tx_xdp_full;98 u64 tx_xdp_full;
95 u64 tx_xdp_err;99 u64 tx_xdp_err;
96 u64 tx_xdp_cqes;100 u64 tx_xdp_cqes;
@@ -101,7 +105,6 @@ struct mlx5e_sw_stats { · 4 unmodified lines
101 u64 rx_buff_alloc_err;105 u64 rx_buff_alloc_err;
102 u64 rx_cqe_compress_blks;106 u64 rx_cqe_compress_blks;
103 u64 rx_cqe_compress_pkts;107 u64 rx_cqe_compress_pkts;
104 u64 rx_page_reuse;
105 u64 rx_cache_reuse;108 u64 rx_cache_reuse;
106 u64 rx_cache_full;109 u64 rx_cache_full;
107 u64 rx_cache_empty;110 u64 rx_cache_empty;
@@ -201,7 +204,6 @@ struct mlx5e_rq_stats { · 93 unmodified lines
201 u64 buff_alloc_err;204 u64 buff_alloc_err;
202 u64 cqe_compress_blks;205 u64 cqe_compress_blks;
203 u64 cqe_compress_pkts;206 u64 cqe_compress_pkts;
204 u64 page_reuse;
205 u64 cache_reuse;207 u64 cache_reuse;
206 u64 cache_full;208 u64 cache_full;
207 u64 cache_empty;209 u64 cache_empty;
@@ -241,6 +243,8 @@ struct mlx5e_sq_stats { · 33 unmodified lines
241 243
242struct mlx5e_xdpsq_stats {244struct mlx5e_xdpsq_stats {
243 u64 xmit;245 u64 xmit;
246 u64 mpwqe;
247 u64 inlnw;
244 u64 full;248 u64 full;
245 u64 err;249 u64 err;
246 /* dirtied @completion */250 /* dirtied @completion */
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c+11−0
@@ -71,6 +71,17 @@ static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq) · 70 unmodified lines
71 net_dim(&rq->dim, dim_sample);71 net_dim(&rq->dim, dim_sample);
72}72}
73 73
74void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
75{
76 struct mlx5_wq_cyc *wq = &sq->wq;
77 struct mlx5e_tx_wqe *nopwqe;
78 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
79
80 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
81 nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
82 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
83}
84
74int mlx5e_napi_poll(struct napi_struct *napi, int budget)85int mlx5e_napi_poll(struct napi_struct *napi, int budget)
75{86{
76 struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,87 struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c+0−5
@@ -43,11 +43,6 @@ · 42 unmodified lines
43#include "ecpf.h"43#include "ecpf.h"
44#include "lib/eq.h"44#include "lib/eq.h"
45 45
46enum {
47 FDB_FAST_PATH = 0,
48 FDB_SLOW_PATH
49};
50
51/* There are two match-all miss flows, one for unicast dst mac and46/* There are two match-all miss flows, one for unicast dst mac and
52 * one for multicast.47 * one for multicast.
53 */48 */
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c+10−2
@@ -2516,8 +2516,16 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) · 2515 unmodified lines
2516 if (!steering->fdb_sub_ns)2516 if (!steering->fdb_sub_ns)
2517 return -ENOMEM;2517 return -ENOMEM;
2518 2518
2519 maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH,
2520 1);
2521 if (IS_ERR(maj_prio)) {
2522 err = PTR_ERR(maj_prio);
2523 goto out_err;
2524 }
2525
2519 levels = 2 * FDB_MAX_PRIO * (FDB_MAX_CHAIN + 1);2526 levels = 2 * FDB_MAX_PRIO * (FDB_MAX_CHAIN + 1);
2520 maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns, 0,2527 maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,
2528 FDB_FAST_PATH,
2521 levels);2529 levels);
2522 if (IS_ERR(maj_prio)) {2530 if (IS_ERR(maj_prio)) {
2523 err = PTR_ERR(maj_prio);2531 err = PTR_ERR(maj_prio);
@@ -2542,7 +2550,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) · 18 unmodified lines
2542 steering->fdb_sub_ns[chain] = ns;2550 steering->fdb_sub_ns[chain] = ns;
2543 }2551 }
2544 2552
2545 maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, 1, 1);2553 maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);
2546 if (IS_ERR(maj_prio)) {2554 if (IS_ERR(maj_prio)) {
2547 err = PTR_ERR(maj_prio);2555 err = PTR_ERR(maj_prio);
2548 goto out_err;2556 goto out_err;
drivers/net/ethernet/mellanox/mlx5/core/wq.h+12−0
@@ -134,6 +134,11 @@ static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) · 133 unmodified lines
134 *wq->db = cpu_to_be32(wq->wqe_ctr);134 *wq->db = cpu_to_be32(wq->wqe_ctr);
135}135}
136 136
137static inline u16 mlx5_wq_cyc_get_ctr_wrap_cnt(struct mlx5_wq_cyc *wq, u16 ctr)
138{
139 return ctr >> wq->fbc.log_sz;
140}
141
137static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)142static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
138{143{
139 return ctr & wq->fbc.sz_m1;144 return ctr & wq->fbc.sz_m1;
@@ -243,6 +248,13 @@ static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) · 103 unmodified lines
243 return mlx5_frag_buf_get_wqe(&wq->fbc, ix);248 return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
244}249}
245 250
251static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix)
252{
253 struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix);
254
255 return be16_to_cpu(wqe->next_wqe_index);
256}
257
246static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)258static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)
247{259{
248 wq->head = head_next;260 wq->head = head_next;
include/linux/mlx5/fs.h+6−0
@@ -75,6 +75,12 @@ enum mlx5_flow_namespace_type { · 74 unmodified lines
75 MLX5_FLOW_NAMESPACE_EGRESS,75 MLX5_FLOW_NAMESPACE_EGRESS,
76};76};
77 77
78enum {
79 FDB_BYPASS_PATH,
80 FDB_FAST_PATH,
81 FDB_SLOW_PATH,
82};
83
78struct mlx5_flow_table;84struct mlx5_flow_table;
79struct mlx5_flow_group;85struct mlx5_flow_group;
80struct mlx5_flow_namespace;86struct mlx5_flow_namespace;
include/linux/mlx5/qp.h+1−0
@@ -395,6 +395,7 @@ struct mlx5_wqe_signature_seg { · 394 unmodified lines
395 395
396struct mlx5_wqe_inline_seg {396struct mlx5_wqe_inline_seg {
397 __be32 byte_count;397 __be32 byte_count;
398 __be32 data[0];
398};399};
399 400
400enum mlx5_sig_type {401enum mlx5_sig_type {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t