History

Merge tag 'mlx5-updates-2019-04-22' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says: ==================== mlx5-updates-2019-04-22 This series includes updates to mlx5e driver RX data path and some significant XDP RX/TX improvements to overcome/mitigate HW and PCIE bottlenecks. From Tariq: 1) Some Enhancements in rq->flags 2) Stabilize RX packet rate (on Striding RQ) with multiple outstanding UMR posts In this patch, we add support for multiple outstanding UMR posts, to allow faster gap closure between consuming MPWQEs and reposting them back into the WQ. Performance test: As expected, huge improvement in large-scale (48 cores). xdp_redirect_map, 64B UDP multi-stream. Redirect from ConnectX-5 100Gbps to ConnectX-6 100Gbps. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz. Before: Unstable, 7 to 30 Mpps After: Stable, at 70.5 Mpps From Shay: 3) XDP, Inline small packets into the TX MPWQE in XDP xmit flow Upon high packet rate with multiple CPUs TX workloads, much of the HCA's resources are spent on prefetching TX descriptors, thus affecting transmission rates. This patch comes to mitigate this problem by moving some workload to the CPU and reducing the HW data prefetch overhead for small packets (<= 256B). When forwarding packets with XDP, a packet that is smaller than a certain size (set to ~256 bytes) would be sent inline within its WQE TX descrptor (mem-copied), when the hardware tx queue is congested beyond a pre-defined water-mark. Performance: Tested packet rate for UDP 64Byte multi-stream over two dual port ConnectX-5 100Gbps NICs. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz * Tested with hyper-threading disabled XDP_TX: | | before | after | | | 24 rings | 51Mpps | 116Mpps | +126% | | 1 ring | 12Mpps | 12Mpps | same | XDP_REDIRECT: ** Below is the transmit rate, not the redirection rate which might be larger, and is not affected by this patch. | | before | after | | | 32 rings | 64Mpps | 92Mpps | +43% | | 1 ring | 6.4Mpps | 6.4Mpps | same | As we can see, feature significantly improves scaling, without hurting single ring performance. From Maxim: 4) Some trivial refactoring and code improvements prior to a larger series to support AF_XDP. ====================

David S. Miller committed 7 years ago

Commit 20eb08b2 parents 539b593 f8ebecf16 files · +406 −202Browse files

Unified · Split

16 files
  1. drivers/net/ethernet/mellanox/mlx5/core/Makefile +2 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/en.h +13 −7
  3. drivers/net/ethernet/mellanox/mlx5/core/en/params.c +104 −0
  4. drivers/net/ethernet/mellanox/mlx5/core/en/params.h +22 −0
  5. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +18 −12
  6. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +54 −3
  7. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +48 −124
  8. drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +87 −43
  9. drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +12 −3
  10. drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +6 −2
  11. drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +11 −0
  12. drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +0 −5
  13. drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +10 −2
  14. drivers/net/ethernet/mellanox/mlx5/core/wq.h +12 −0
  15. include/linux/mlx5/fs.h +6 −0
  16. include/linux/mlx5/qp.h +1 −0
drivers/net/ethernet/mellanox/mlx5/core/Makefile+2−1
@@ -22,7 +22,8 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ · 21 unmodified lines
2222 #
2323 mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
2424 en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \
25− en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o
25+ en_selftest.o en/port.o en/monitor_stats.o en/reporter_tx.o \
26+ en/params.o
2627
2728 #
2829 # Netdev extra
drivers/net/ethernet/mellanox/mlx5/core/en.h+13−7
@@ -409,14 +409,17 @@ struct mlx5e_xdp_info_fifo { · 408 unmodified lines
409409
410410 struct mlx5e_xdp_wqe_info {
411411 u8 num_wqebbs;
412− u8 num_ds;
412+ u8 num_pkts;
413413 };
414414
415415 struct mlx5e_xdp_mpwqe {
416416 /* Current MPWQE session */
417417 struct mlx5e_tx_wqe *wqe;
418418 u8 ds_count;
419+ u8 pkt_count;
419420 u8 max_ds_count;
421+ u8 complete;
422+ u8 inline_on;
420423 };
421424
422425 struct mlx5e_xdpsq;
@@ -428,7 +431,6 @@ struct mlx5e_xdpsq { · 5 unmodified lines
428431 /* dirtied @completion */
429432 u32 xdpi_fifo_cc;
430433 u16 cc;
431− bool redirect_flush;
432434
433435 /* dirtied @xmit */
434436 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
@@ -461,10 +463,10 @@ struct mlx5e_xdpsq { · 26 unmodified lines
461463
462464 struct mlx5e_icosq {
463465 /* data path */
466+ u16 cc;
467+ u16 pc;
464468
465− /* dirtied @xmit */
466− u16 pc ____cacheline_aligned_in_smp;
467−
469+ struct mlx5_wqe_ctrl_seg *doorbell_cseg;
468470 struct mlx5e_cq cq;
469471
470472 /* write@xmit, read@completion */
@@ -531,7 +533,8 @@ typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); · 60 unmodified lines
531533 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
532534
533535 enum mlx5e_rq_flag {
534− MLX5E_RQ_FLAG_XDP_XMIT = BIT(0),
536+ MLX5E_RQ_FLAG_XDP_XMIT,
537+ MLX5E_RQ_FLAG_XDP_REDIRECT,
535538 };
536539
537540 struct mlx5e_rq_frag_info {
@@ -562,8 +565,10 @@ struct mlx5e_rq { · 24 unmodified lines
562565 struct mlx5e_mpw_info *info;
563566 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
564567 u16 num_strides;
568+ u16 actual_wq_head;
565569 u8 log_stride_sz;
566− bool umr_in_progress;
570+ u8 umr_in_progress;
571+ u8 umr_last_bulk;
567572 } mpwqe;
568573 };
569574 struct {
@@ -773,6 +778,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); · 203 unmodified lines
773778 netdev_tx_t mlx5e_sq_xmit(struct mlx5e_txqsq *sq, struct sk_buff *skb,
774779 struct mlx5e_tx_wqe *wqe, u16 pi, bool xmit_more);
775780
781+void mlx5e_trigger_irq(struct mlx5e_icosq *sq);
776782 void mlx5e_completion_event(struct mlx5_core_cq *mcq);
777783 void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
778784 int mlx5e_napi_poll(struct napi_struct *napi, int budget);
drivers/net/ethernet/mellanox/mlx5/core/en/params.c+104−0
@@ -0,0 +1,104 @@
1+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2+/* Copyright (c) 2019 Mellanox Technologies. */
3+
4+#include "en/params.h"
5+
6+u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
7+{
8+ u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
9+ u16 linear_rq_headroom = params->xdp_prog ?
10+ XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
11+ u32 frag_sz;
12+
13+ linear_rq_headroom += NET_IP_ALIGN;
14+
15+ frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
16+
17+ if (params->xdp_prog && frag_sz < PAGE_SIZE)
18+ frag_sz = PAGE_SIZE;
19+
20+ return frag_sz;
21+}
22+
23+u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
24+{
25+ u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
26+
27+ return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
28+}
29+
30+bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params)
31+{
32+ u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
33+
34+ return !params->lro_en && frag_sz <= PAGE_SIZE;
35+}
36+
37+#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \
38+ MLX5_MPWQE_LOG_STRIDE_SZ_BASE)
39+bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
40+ struct mlx5e_params *params)
41+{
42+ u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
43+ s8 signed_log_num_strides_param;
44+ u8 log_num_strides;
45+
46+ if (!mlx5e_rx_is_linear_skb(params))
47+ return false;
48+
49+ if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)
50+ return false;
51+
52+ if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
53+ return true;
54+
55+ log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
56+ signed_log_num_strides_param =
57+ (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
58+
59+ return signed_log_num_strides_param >= 0;
60+}
61+
62+u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
63+{
64+ u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params);
65+
66+ /* Numbers are unsigned, don't subtract to avoid underflow. */
67+ if (params->log_rq_mtu_frames <
68+ log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
69+ return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
70+
71+ return params->log_rq_mtu_frames - log_pkts_per_wqe;
72+}
73+
74+u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
75+ struct mlx5e_params *params)
76+{
77+ if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
78+ return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
79+
80+ return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
81+}
82+
83+u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
84+ struct mlx5e_params *params)
85+{
86+ return MLX5_MPWRQ_LOG_WQE_SZ -
87+ mlx5e_mpwqe_get_log_stride_size(mdev, params);
88+}
89+
90+u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
91+ struct mlx5e_params *params)
92+{
93+ u16 linear_rq_headroom = params->xdp_prog ?
94+ XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
95+ bool is_linear_skb;
96+
97+ linear_rq_headroom += NET_IP_ALIGN;
98+
99+ is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
100+ mlx5e_rx_is_linear_skb(params) :
101+ mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
102+
103+ return is_linear_skb ? linear_rq_headroom : 0;
104+}
drivers/net/ethernet/mellanox/mlx5/core/en/params.h+22−0
@@ -0,0 +1,22 @@
1+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2+/* Copyright (c) 2019 Mellanox Technologies. */
3+
4+#ifndef __MLX5_EN_PARAMS_H__
5+#define __MLX5_EN_PARAMS_H__
6+
7+#include "en.h"
8+
9+u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params);
10+u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params);
11+bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params);
12+bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
13+ struct mlx5e_params *params);
14+u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params);
15+u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
16+ struct mlx5e_params *params);
17+u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
18+ struct mlx5e_params *params);
19+u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
20+ struct mlx5e_params *params);
21+
22+#endif /* __MLX5_EN_PARAMS_H__ */
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+18−12
@@ -85,7 +85,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di, · 84 unmodified lines
8585 if (unlikely(err))
8686 goto xdp_abort;
8787 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
88− rq->xdpsq.redirect_flush = true;
88+ __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
8989 mlx5e_page_dma_unmap(rq, di);
9090 rq->stats->xdp_redirect++;
9191 return true;
@@ -105,6 +105,7 @@ xdp_abort: · 13 unmodified lines
105105 static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq)
106106 {
107107 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
108+ struct mlx5e_xdpsq_stats *stats = sq->stats;
108109 struct mlx5_wq_cyc *wq = &sq->wq;
109110 u8 wqebbs;
110111 u16 pi;
@@ -112,7 +113,9 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) · 1 unmodified line
112113 mlx5e_xdpsq_fetch_wqe(sq, &session->wqe);
113114
114115 prefetchw(session->wqe->data);
115− session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT;
116+ session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT;
117+ session->pkt_count = 0;
118+ session->complete = 0;
116119
117120 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
118121
@@ -131,6 +134,10 @@ static void mlx5e_xdp_mpwqe_session_start(struct mlx5e_xdpsq *sq) · 12 unmodified lines
131134 MLX5E_XDP_MPW_MAX_WQEBBS);
132135
133136 session->max_ds_count = MLX5_SEND_WQEBB_NUM_DS * wqebbs;
137+
138+ mlx5e_xdp_update_inline_state(sq);
139+
140+ stats->mpwqe++;
134141 }
135142
136143 static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq)
@@ -147,7 +154,7 @@ static void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq) · 10 unmodified lines
147154 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_count);
148155
149156 wi->num_wqebbs = DIV_ROUND_UP(ds_count, MLX5_SEND_WQEBB_NUM_DS);
150− wi->num_ds = ds_count - MLX5E_XDP_TX_EMPTY_DS_COUNT;
157+ wi->num_pkts = session->pkt_count;
151158
152159 sq->pc += wi->num_wqebbs;
153160
@@ -162,11 +169,9 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, · 8 unmodified lines
162169 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
163170 struct mlx5e_xdpsq_stats *stats = sq->stats;
164171
165− dma_addr_t dma_addr = xdpi->dma_addr;
166172 struct xdp_frame *xdpf = xdpi->xdpf;
167− unsigned int dma_len = xdpf->len;
168173
169− if (unlikely(sq->hw_mtu < dma_len)) {
174+ if (unlikely(sq->hw_mtu < xdpf->len)) {
170175 stats->err++;
171176 return false;
172177 }
@@ -183,9 +188,10 @@ static bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, · 10 unmodified lines
183188 mlx5e_xdp_mpwqe_session_start(sq);
184189 }
185190
186− mlx5e_xdp_mpwqe_add_dseg(sq, dma_addr, dma_len);
191+ mlx5e_xdp_mpwqe_add_dseg(sq, xdpi, stats);
187192
188− if (unlikely(session->ds_count == session->max_ds_count))
193+ if (unlikely(session->complete ||
194+ session->ds_count == session->max_ds_count))
189195 mlx5e_xdp_mpwqe_complete(sq);
190196
191197 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
@@ -299,7 +305,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) · 107 unmodified lines
299305
300306 sqcc += wi->num_wqebbs;
301307
302− for (j = 0; j < wi->num_ds; j++) {
308+ for (j = 0; j < wi->num_pkts; j++) {
303309 struct mlx5e_xdp_info xdpi =
304310 mlx5e_xdpi_fifo_pop(xdpi_fifo);
305311
@@ -340,7 +346,7 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) · 34 unmodified lines
340346
341347 sq->cc += wi->num_wqebbs;
342348
343− for (i = 0; i < wi->num_ds; i++) {
349+ for (i = 0; i < wi->num_pkts; i++) {
344350 struct mlx5e_xdp_info xdpi =
345351 mlx5e_xdpi_fifo_pop(xdpi_fifo);
346352
@@ -419,9 +425,9 @@ void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq) · 72 unmodified lines
419425
420426 mlx5e_xmit_xdp_doorbell(xdpsq);
421427
422− if (xdpsq->redirect_flush) {
428+ if (test_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags)) {
423429 xdp_do_flush_map();
424− xdpsq->redirect_flush = false;
430+ __clear_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
425431 }
426432 }
427433
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+54−3
@@ -75,16 +75,68 @@ static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq) · 74 unmodified lines
7575 }
7676 }
7777
78+/* Enable inline WQEs to shift some load from a congested HCA (HW) to
79+ * a less congested cpu (SW).
80+ */
81+static inline void mlx5e_xdp_update_inline_state(struct mlx5e_xdpsq *sq)
82+{
83+ u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc;
84+ struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
85+
86+#define MLX5E_XDP_INLINE_WATERMARK_LOW 10
87+#define MLX5E_XDP_INLINE_WATERMARK_HIGH 128
88+
89+ if (session->inline_on) {
90+ if (outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW)
91+ session->inline_on = 0;
92+ return;
93+ }
94+
95+ /* inline is false */
96+ if (outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH)
97+ session->inline_on = 1;
98+}
99+
78100 static inline void
79−mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, dma_addr_t dma_addr, u16 dma_len)
101+mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi,
102+ struct mlx5e_xdpsq_stats *stats)
80103 {
81104 struct mlx5e_xdp_mpwqe *session = &sq->mpwqe;
105+ dma_addr_t dma_addr = xdpi->dma_addr;
106+ struct xdp_frame *xdpf = xdpi->xdpf;
82107 struct mlx5_wqe_data_seg *dseg =
83− (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count++;
108+ (struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
109+ u16 dma_len = xdpf->len;
84110
111+ session->pkt_count++;
112+
113+#define MLX5E_XDP_INLINE_WQE_SZ_THRSD (256 - sizeof(struct mlx5_wqe_inline_seg))
114+
115+ if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) {
116+ struct mlx5_wqe_inline_seg *inline_dseg =
117+ (struct mlx5_wqe_inline_seg *)dseg;
118+ u16 ds_len = sizeof(*inline_dseg) + dma_len;
119+ u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS);
120+
121+ if (unlikely(session->ds_count + ds_cnt > session->max_ds_count)) {
122+ /* Not enough space for inline wqe, send with memory pointer */
123+ session->complete = true;
124+ goto no_inline;
125+ }
126+
127+ inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG);
128+ memcpy(inline_dseg->data, xdpf->data, dma_len);
129+
130+ session->ds_count += ds_cnt;
131+ stats->inlnw++;
132+ return;
133+ }
134+
135+no_inline:
85136 dseg->addr = cpu_to_be64(dma_addr);
86137 dseg->byte_count = cpu_to_be32(dma_len);
87138 dseg->lkey = sq->mkey_be;
139+ session->ds_count++;
88140 }
89141
90142 static inline void mlx5e_xdpsq_fetch_wqe(struct mlx5e_xdpsq *sq,
@@ -111,5 +163,4 @@ mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo) · 20 unmodified lines
111163 {
112164 return fifo->xi[(*fifo->cc)++ & fifo->mask];
113165 }
114−
115166 #endif
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+48−124
@@ -55,6 +55,7 @@ · 54 unmodified lines
5555 #include "lib/eq.h"
5656 #include "en/monitor_stats.h"
5757 #include "en/reporter.h"
58+#include "en/params.h"
5859
5960 struct mlx5e_rq_param {
6061 u32 rqc[MLX5_ST_SZ_DW(rqc)];
@@ -103,104 +104,6 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) · 42 unmodified lines
103104 return true;
104105 }
105106
106−static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
107−{
108− u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
109− u16 linear_rq_headroom = params->xdp_prog ?
110− XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
111− u32 frag_sz;
112−
113− linear_rq_headroom += NET_IP_ALIGN;
114−
115− frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
116−
117− if (params->xdp_prog && frag_sz < PAGE_SIZE)
118− frag_sz = PAGE_SIZE;
119−
120− return frag_sz;
121−}
122−
123−static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
124−{
125− u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
126−
127− return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
128−}
129−
130−static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
131− struct mlx5e_params *params)
132−{
133− u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
134−
135− return !params->lro_en && frag_sz <= PAGE_SIZE;
136−}
137−
138−#define MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ ((BIT(__mlx5_bit_sz(wq, log_wqe_stride_size)) - 1) + \
139− MLX5_MPWQE_LOG_STRIDE_SZ_BASE)
140−static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
141− struct mlx5e_params *params)
142−{
143− u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
144− s8 signed_log_num_strides_param;
145− u8 log_num_strides;
146−
147− if (!mlx5e_rx_is_linear_skb(mdev, params))
148− return false;
149−
150− if (order_base_2(frag_sz) > MLX5_MAX_MPWQE_LOG_WQE_STRIDE_SZ)
151− return false;
152−
153− if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
154− return true;
155−
156− log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
157− signed_log_num_strides_param =
158− (s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
159−
160− return signed_log_num_strides_param >= 0;
161−}
162−
163−static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
164−{
165− if (params->log_rq_mtu_frames <
166− mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
167− return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
168−
169− return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params);
170−}
171−
172−static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
173− struct mlx5e_params *params)
174−{
175− if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
176− return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
177−
178− return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
179−}
180−
181−static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
182− struct mlx5e_params *params)
183−{
184− return MLX5_MPWRQ_LOG_WQE_SZ -
185− mlx5e_mpwqe_get_log_stride_size(mdev, params);
186−}
187−
188−static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
189− struct mlx5e_params *params)
190−{
191− u16 linear_rq_headroom = params->xdp_prog ?
192− XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
193− bool is_linear_skb;
194−
195− linear_rq_headroom += NET_IP_ALIGN;
196−
197− is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
198− mlx5e_rx_is_linear_skb(mdev, params) :
199− mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
200−
201− return is_linear_skb ? linear_rq_headroom : 0;
202−}
203−
204107 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
205108 struct mlx5e_params *params)
206109 {
@@ -470,7 +373,6 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) · 263 unmodified lines
470373 }
471374
472375 static int mlx5e_init_di_list(struct mlx5e_rq *rq,
473− struct mlx5e_params *params,
474376 int wq_sz, int cpu)
475377 {
476378 int len = wq_sz << rq->wqe.info.log_num_frags;
@@ -598,7 +500,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 121 unmodified lines
598500 goto err_free;
599501 }
600502
601− err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu);
503+ err = mlx5e_init_di_list(rq, wq_sz, c->cpu);
602504 if (err)
603505 goto err_free;
604506 rq->post_wqes = mlx5e_post_rx_wqes;
@@ -616,7 +518,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 11 unmodified lines
616518 goto err_free;
617519 }
618520
619− rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ?
521+ rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params) ?
620522 mlx5e_skb_from_cqe_linear :
621523 mlx5e_skb_from_cqe_nonlinear;
622524 rq->mkey_be = c->mkey_be;
@@ -903,10 +805,14 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) · 280 unmodified lines
903805
904806 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
905807 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
808+ u16 head = wq->head;
809+ int i;
906810
907− /* UMR WQE (if in progress) is always at wq->head */
908− if (rq->mpwqe.umr_in_progress)
909− rq->dealloc_wqe(rq, wq->head);
811+ /* Outstanding UMR WQEs (in progress) start at wq->head */
812+ for (i = 0; i < rq->mpwqe.umr_in_progress; i++) {
813+ rq->dealloc_wqe(rq, head);
814+ head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
815+ }
910816
911817 while (!mlx5_wq_ll_is_empty(wq)) {
912818 struct mlx5e_rx_wqe_ll *wqe;
@@ -971,16 +877,8 @@ err_free_rq: · 58 unmodified lines
971877
972878 static void mlx5e_activate_rq(struct mlx5e_rq *rq)
973879 {
974− struct mlx5e_icosq *sq = &rq->channel->icosq;
975− struct mlx5_wq_cyc *wq = &sq->wq;
976− struct mlx5e_tx_wqe *nopwqe;
977−
978− u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
979−
980880 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
981− sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
982− nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
983− mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
881+ mlx5e_trigger_irq(&rq->channel->icosq);
984882 }
985883
986884 static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
@@ -1092,7 +990,7 @@ static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) · 105 unmodified lines
1092990
1093991 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1094992 {
1095− u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
993+ int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1096994
1097995 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz,
1098996 sizeof(*sq->db.ico_wqe)),
@@ -1528,7 +1426,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 429 unmodified lines
15281426 dseg->lkey = sq->mkey_be;
15291427
15301428 wi->num_wqebbs = 1;
1531− wi->num_ds = 1;
1429+ wi->num_pkts = 1;
15321430 }
15331431 }
15341432
@@ -2054,7 +1952,7 @@ static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, · 519 unmodified lines
20541952 byte_count += MLX5E_METADATA_ETHER_LEN;
20551953 #endif
20561954
2057− if (mlx5e_rx_is_linear_skb(mdev, params)) {
1955+ if (mlx5e_rx_is_linear_skb(params)) {
20581956 int frag_stride;
20591957
20601958 frag_stride = mlx5e_rx_get_linear_frag_sz(params);
@@ -2108,6 +2006,13 @@ static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) · 47 unmodified lines
21082006 return order_base_2(sz);
21092007 }
21102008
2009+static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
2010+{
2011+ void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2012+
2013+ return MLX5_GET(wq, wq, log_wq_sz);
2014+}
2015+
21112016 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
21122017 struct mlx5e_params *params,
21132018 struct mlx5e_rq_param *param)
@@ -2274,13 +2179,28 @@ static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, · 160 unmodified lines
22742179 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
22752180 }
22762181
2182+static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params,
2183+ struct mlx5e_rq_param *rqp)
2184+{
2185+ switch (params->rq_wq_type) {
2186+ case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2187+ return order_base_2(MLX5E_UMR_WQEBBS) +
2188+ mlx5e_get_rq_log_wq_sz(rqp->rqc);
2189+ default: /* MLX5_WQ_TYPE_CYCLIC */
2190+ return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2191+ }
2192+}
2193+
22772194 static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
22782195 struct mlx5e_params *params,
22792196 struct mlx5e_channel_param *cparam)
22802197 {
2281− u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2198+ u8 icosq_log_wq_sz;
22822199
22832200 mlx5e_build_rq_param(priv, params, &cparam->rq);
2201+
2202+ icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq);
2203+
22842204 mlx5e_build_sq_param(priv, params, &cparam->sq);
22852205 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
22862206 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
@@ -2336,14 +2256,18 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs) · 49 unmodified lines
23362256 mlx5e_activate_channel(chs->c[i]);
23372257 }
23382258
2259+#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2260+
23392261 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
23402262 {
23412263 int err = 0;
23422264 int i;
23432265
2344− for (i = 0; i < chs->num; i++)
2345− err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq,
2346− err ? 0 : 20000);
2266+ for (i = 0; i < chs->num; i++) {
2267+ int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
2268+
2269+ err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
2270+ }
23472271
23482272 return err ? -ETIMEDOUT : 0;
23492273 }
@@ -3794,7 +3718,7 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, · 1444 unmodified lines
37943718 new_channels.params.sw_mtu = new_mtu;
37953719
37963720 if (params->xdp_prog &&
3797− !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
3721+ !mlx5e_rx_is_linear_skb(&new_channels.params)) {
37983722 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
37993723 new_mtu, MLX5E_XDP_MAX_MTU);
38003724 err = -EINVAL;
@@ -4235,7 +4159,7 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) · 434 unmodified lines
42354159 new_channels.params = priv->channels.params;
42364160 new_channels.params.xdp_prog = prog;
42374161
4238− if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
4162+ if (!mlx5e_rx_is_linear_skb(&new_channels.params)) {
42394163 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
42404164 new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
42414165 return -EINVAL;
@@ -4288,7 +4212,7 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) · 46 unmodified lines
42884212 mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
42894213
42904214 if (was_opened && reset)
4291− mlx5e_open_locked(netdev);
4215+ err = mlx5e_open_locked(netdev);
42924216
42934217 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
42944218 goto unlock;
@@ -4578,7 +4502,7 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, · 283 unmodified lines
45784502 if (!slow_pci_heuristic(mdev) &&
45794503 mlx5e_striding_rq_possible(mdev, params) &&
45804504 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ||
4581− !mlx5e_rx_is_linear_skb(mdev, params)))
4505+ !mlx5e_rx_is_linear_skb(params)))
45824506 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
45834507 mlx5e_set_rq_type(mdev, params);
45844508 mlx5e_init_rq_type_params(mdev, params);
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c+87−43
@@ -409,14 +409,15 @@ mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle · 408 unmodified lines
409409 mlx5e_page_release(rq, &dma_info[i], recycle);
410410 }
411411
412−static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
412+static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
413413 {
414414 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
415− struct mlx5e_rx_wqe_ll *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
416415
417− rq->mpwqe.umr_in_progress = false;
416+ do {
417+ u16 next_wqe_index = mlx5_wq_ll_get_wqe_next_ix(wq, wq->head);
418418
419− mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
419+ mlx5_wq_ll_push(wq, next_wqe_index);
420+ } while (--n);
420421
421422 /* ensure wqes are visible to device before updating doorbell record */
422423 dma_wmb();
@@ -426,7 +427,7 @@ static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq) · 3 unmodified lines
426427
427428 static inline u16 mlx5e_icosq_wrap_cnt(struct mlx5e_icosq *sq)
428429 {
429− return sq->pc >> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
430+ return mlx5_wq_cyc_get_ctr_wrap_cnt(&sq->wq, sq->pc);
430431 }
431432
432433 static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq *sq,
@@ -478,8 +479,6 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) · 45 unmodified lines
478479 bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
479480 wi->consumed_strides = 0;
480481
481− rq->mpwqe.umr_in_progress = true;
482−
483482 umr_wqe->ctrl.opmod_idx_opcode =
484483 cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
485484 MLX5_OPCODE_UMR);
@@ -487,7 +486,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) · 1 unmodified line
487486
488487 sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
489488 sq->pc += MLX5E_UMR_WQEBBS;
490− mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &umr_wqe->ctrl);
489+
490+ sq->doorbell_cseg = &umr_wqe->ctrl;
491491
492492 return 0;
493493
@@ -542,37 +542,13 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) · 48 unmodified lines
542542 return !!err;
543543 }
544544
545−static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
546− struct mlx5e_icosq *sq,
547− struct mlx5e_rq *rq,
548− struct mlx5_cqe64 *cqe)
549−{
550− struct mlx5_wq_cyc *wq = &sq->wq;
551− u16 ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter));
552− struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
553−
554− mlx5_cqwq_pop(&cq->wq);
555−
556− if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
557− netdev_WARN_ONCE(cq->channel->netdev,
558− "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe));
559− return;
560− }
561−
562− if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
563− mlx5e_post_rx_mpwqe(rq);
564− return;
565− }
566−
567− if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
568− netdev_WARN_ONCE(cq->channel->netdev,
569− "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi->opcode);
570−}
571−
572545 static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
573546 {
574547 struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
575548 struct mlx5_cqe64 *cqe;
549+ u8 completed_umr = 0;
550+ u16 sqcc;
551+ int i;
576552
577553 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
578554 return;
@@ -581,28 +557,96 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq) · 2 unmodified lines
581557 if (likely(!cqe))
582558 return;
583559
584− /* by design, there's only a single cqe */
585− mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe);
560+ /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
561+ * otherwise a cq overrun may occur
562+ */
563+ sqcc = sq->cc;
564+
565+ i = 0;
566+ do {
567+ u16 wqe_counter;
568+ bool last_wqe;
569+
570+ mlx5_cqwq_pop(&cq->wq);
571+
572+ wqe_counter = be16_to_cpu(cqe->wqe_counter);
573+
574+ if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
575+ netdev_WARN_ONCE(cq->channel->netdev,
576+ "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe));
577+ break;
578+ }
579+ do {
580+ struct mlx5e_sq_wqe_info *wi;
581+ u16 ci;
582+
583+ last_wqe = (sqcc == wqe_counter);
584+
585+ ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
586+ wi = &sq->db.ico_wqe[ci];
587+
588+ if (likely(wi->opcode == MLX5_OPCODE_UMR)) {
589+ sqcc += MLX5E_UMR_WQEBBS;
590+ completed_umr++;
591+ } else if (likely(wi->opcode == MLX5_OPCODE_NOP)) {
592+ sqcc++;
593+ } else {
594+ netdev_WARN_ONCE(cq->channel->netdev,
595+ "Bad OPCODE in ICOSQ WQE info: 0x%x\n",
596+ wi->opcode);
597+ }
598+
599+ } while (!last_wqe);
600+
601+ } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
602+
603+ sq->cc = sqcc;
586604
587605 mlx5_cqwq_update_db_record(&cq->wq);
606+
607+ if (likely(completed_umr)) {
608+ mlx5e_post_rx_mpwqe(rq, completed_umr);
609+ rq->mpwqe.umr_in_progress -= completed_umr;
610+ }
588611 }
589612
590613 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
591614 {
615+ struct mlx5e_icosq *sq = &rq->channel->icosq;
592616 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
617+ u8 missing, i;
618+ u16 head;
593619
594620 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
595621 return false;
596622
597− mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);
623+ mlx5e_poll_ico_cq(&sq->cq, rq);
624+
625+ missing = mlx5_wq_ll_missing(wq) - rq->mpwqe.umr_in_progress;
598626
599− if (mlx5_wq_ll_is_full(wq))
627+ if (unlikely(rq->mpwqe.umr_in_progress > rq->mpwqe.umr_last_bulk))
628+ rq->stats->congst_umr++;
629+
630+#define UMR_WQE_BULK (2)
631+ if (likely(missing < UMR_WQE_BULK))
600632 return false;
601633
602− if (!rq->mpwqe.umr_in_progress)
603− mlx5e_alloc_rx_mpwqe(rq, wq->head);
604− else
605− rq->stats->congst_umr += mlx5_wq_ll_missing(wq) > 2;
634+ head = rq->mpwqe.actual_wq_head;
635+ i = missing;
636+ do {
637+ if (unlikely(mlx5e_alloc_rx_mpwqe(rq, head)))
638+ break;
639+ head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
640+ } while (--i);
641+
642+ rq->mpwqe.umr_last_bulk = missing - i;
643+ if (sq->doorbell_cseg) {
644+ mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
645+ sq->doorbell_cseg = NULL;
646+ }
647+
648+ rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
649+ rq->mpwqe.actual_wq_head = head;
606650
607651 return false;
608652 }
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c+12−3
@@ -65,6 +65,8 @@ static const struct counter_desc sw_stats_desc[] = { · 64 unmodified lines
6565 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
6666 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },
6767 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },
68+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_mpwqe) },
69+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_inlnw) },
6870 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
6971 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },
7072 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
@@ -79,6 +81,8 @@ static const struct counter_desc sw_stats_desc[] = { · 8 unmodified lines
7981 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
8082 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },
8183 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },
84+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_mpwqe) },
85+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_inlnw) },
8286 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },
8387 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },
8488 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },
@@ -89,7 +93,6 @@ static const struct counter_desc sw_stats_desc[] = { · 4 unmodified lines
8993 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
9094 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
9195 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
92− { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_page_reuse) },
9396 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_reuse) },
9497 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
9598 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
@@ -160,6 +163,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 64 unmodified lines
160163 s->rx_xdp_drop += rq_stats->xdp_drop;
161164 s->rx_xdp_redirect += rq_stats->xdp_redirect;
162165 s->rx_xdp_tx_xmit += xdpsq_stats->xmit;
166+ s->rx_xdp_tx_mpwqe += xdpsq_stats->mpwqe;
167+ s->rx_xdp_tx_inlnw += xdpsq_stats->inlnw;
163168 s->rx_xdp_tx_full += xdpsq_stats->full;
164169 s->rx_xdp_tx_err += xdpsq_stats->err;
165170 s->rx_xdp_tx_cqe += xdpsq_stats->cqes;
@@ -170,7 +175,6 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 4 unmodified lines
170175 s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
171176 s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
172177 s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
173− s->rx_page_reuse += rq_stats->page_reuse;
174178 s->rx_cache_reuse += rq_stats->cache_reuse;
175179 s->rx_cache_full += rq_stats->cache_full;
176180 s->rx_cache_empty += rq_stats->cache_empty;
@@ -185,6 +189,8 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 8 unmodified lines
185189 s->ch_eq_rearm += ch_stats->eq_rearm;
186190 /* xdp redirect */
187191 s->tx_xdp_xmit += xdpsq_red_stats->xmit;
192+ s->tx_xdp_mpwqe += xdpsq_red_stats->mpwqe;
193+ s->tx_xdp_inlnw += xdpsq_red_stats->inlnw;
188194 s->tx_xdp_full += xdpsq_red_stats->full;
189195 s->tx_xdp_err += xdpsq_red_stats->err;
190196 s->tx_xdp_cqes += xdpsq_red_stats->cqes;
@@ -1212,7 +1218,6 @@ static const struct counter_desc rq_stats_desc[] = { · 1021 unmodified lines
12121218 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
12131219 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
12141220 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
1215− { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, page_reuse) },
12161221 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_reuse) },
12171222 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
12181223 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
@@ -1245,6 +1250,8 @@ static const struct counter_desc sq_stats_desc[] = { · 26 unmodified lines
12451250
12461251 static const struct counter_desc rq_xdpsq_stats_desc[] = {
12471252 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1253+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1254+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
12481255 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
12491256 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
12501257 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
@@ -1252,6 +1259,8 @@ static const struct counter_desc rq_xdpsq_stats_desc[] = { · 1 unmodified line
12521259
12531260 static const struct counter_desc xdpsq_stats_desc[] = {
12541261 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1262+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, mpwqe) },
1263+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, inlnw) },
12551264 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
12561265 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
12571266 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h+6−2
@@ -77,6 +77,8 @@ struct mlx5e_sw_stats { · 76 unmodified lines
7777 u64 rx_xdp_drop;
7878 u64 rx_xdp_redirect;
7979 u64 rx_xdp_tx_xmit;
80+ u64 rx_xdp_tx_mpwqe;
81+ u64 rx_xdp_tx_inlnw;
8082 u64 rx_xdp_tx_full;
8183 u64 rx_xdp_tx_err;
8284 u64 rx_xdp_tx_cqe;
@@ -91,6 +93,8 @@ struct mlx5e_sw_stats { · 8 unmodified lines
9193 u64 tx_queue_wake;
9294 u64 tx_cqe_err;
9395 u64 tx_xdp_xmit;
96+ u64 tx_xdp_mpwqe;
97+ u64 tx_xdp_inlnw;
9498 u64 tx_xdp_full;
9599 u64 tx_xdp_err;
96100 u64 tx_xdp_cqes;
@@ -101,7 +105,6 @@ struct mlx5e_sw_stats { · 4 unmodified lines
101105 u64 rx_buff_alloc_err;
102106 u64 rx_cqe_compress_blks;
103107 u64 rx_cqe_compress_pkts;
104− u64 rx_page_reuse;
105108 u64 rx_cache_reuse;
106109 u64 rx_cache_full;
107110 u64 rx_cache_empty;
@@ -201,7 +204,6 @@ struct mlx5e_rq_stats { · 93 unmodified lines
201204 u64 buff_alloc_err;
202205 u64 cqe_compress_blks;
203206 u64 cqe_compress_pkts;
204− u64 page_reuse;
205207 u64 cache_reuse;
206208 u64 cache_full;
207209 u64 cache_empty;
@@ -241,6 +243,8 @@ struct mlx5e_sq_stats { · 33 unmodified lines
241243
242244 struct mlx5e_xdpsq_stats {
243245 u64 xmit;
246+ u64 mpwqe;
247+ u64 inlnw;
244248 u64 full;
245249 u64 err;
246250 /* dirtied @completion */
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c+11−0
@@ -71,6 +71,17 @@ static void mlx5e_handle_rx_dim(struct mlx5e_rq *rq) · 70 unmodified lines
7171 net_dim(&rq->dim, dim_sample);
7272 }
7373
74+void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
75+{
76+ struct mlx5_wq_cyc *wq = &sq->wq;
77+ struct mlx5e_tx_wqe *nopwqe;
78+ u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
79+
80+ sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
81+ nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
82+ mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
83+}
84+
7485 int mlx5e_napi_poll(struct napi_struct *napi, int budget)
7586 {
7687 struct mlx5e_channel *c = container_of(napi, struct mlx5e_channel,
drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c+0−5
@@ -43,11 +43,6 @@ · 42 unmodified lines
4343 #include "ecpf.h"
4444 #include "lib/eq.h"
4545
46−enum {
47− FDB_FAST_PATH = 0,
48− FDB_SLOW_PATH
49−};
50−
5146 /* There are two match-all miss flows, one for unicast dst mac and
5247 * one for multicast.
5348 */
drivers/net/ethernet/mellanox/mlx5/core/fs_core.c+10−2
@@ -2516,8 +2516,16 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) · 2515 unmodified lines
25162516 if (!steering->fdb_sub_ns)
25172517 return -ENOMEM;
25182518
2519+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BYPASS_PATH,
2520+ 1);
2521+ if (IS_ERR(maj_prio)) {
2522+ err = PTR_ERR(maj_prio);
2523+ goto out_err;
2524+ }
2525+
25192526 levels = 2 * FDB_MAX_PRIO * (FDB_MAX_CHAIN + 1);
2520− maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns, 0,
2527+ maj_prio = fs_create_prio_chained(&steering->fdb_root_ns->ns,
2528+ FDB_FAST_PATH,
25212529 levels);
25222530 if (IS_ERR(maj_prio)) {
25232531 err = PTR_ERR(maj_prio);
@@ -2542,7 +2550,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) · 18 unmodified lines
25422550 steering->fdb_sub_ns[chain] = ns;
25432551 }
25442552
2545− maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, 1, 1);
2553+ maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1);
25462554 if (IS_ERR(maj_prio)) {
25472555 err = PTR_ERR(maj_prio);
25482556 goto out_err;
drivers/net/ethernet/mellanox/mlx5/core/wq.h+12−0
@@ -134,6 +134,11 @@ static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) · 133 unmodified lines
134134 *wq->db = cpu_to_be32(wq->wqe_ctr);
135135 }
136136
137+static inline u16 mlx5_wq_cyc_get_ctr_wrap_cnt(struct mlx5_wq_cyc *wq, u16 ctr)
138+{
139+ return ctr >> wq->fbc.log_sz;
140+}
141+
137142 static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
138143 {
139144 return ctr & wq->fbc.sz_m1;
@@ -243,6 +248,13 @@ static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) · 103 unmodified lines
243248 return mlx5_frag_buf_get_wqe(&wq->fbc, ix);
244249 }
245250
251+static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix)
252+{
253+ struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix);
254+
255+ return be16_to_cpu(wqe->next_wqe_index);
256+}
257+
246258 static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)
247259 {
248260 wq->head = head_next;
include/linux/mlx5/fs.h+6−0
@@ -75,6 +75,12 @@ enum mlx5_flow_namespace_type { · 74 unmodified lines
7575 MLX5_FLOW_NAMESPACE_EGRESS,
7676 };
7777
78+enum {
79+ FDB_BYPASS_PATH,
80+ FDB_FAST_PATH,
81+ FDB_SLOW_PATH,
82+};
83+
7884 struct mlx5_flow_table;
7985 struct mlx5_flow_group;
8086 struct mlx5_flow_namespace;
include/linux/mlx5/qp.h+1−0
@@ -395,6 +395,7 @@ struct mlx5_wqe_signature_seg { · 394 unmodified lines
395395
396396 struct mlx5_wqe_inline_seg {
397397 __be32 byte_count;
398+ __be32 data[0];
398399 };
399400
400401 enum mlx5_sig_type {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t