History

Merge tag 'mlx5e-updates-2018-07-26' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says: ==================== mlx5e-updates-2018-07-26 (XDP redirect) This series from Tariq adds the support for device-out XDP redirect. Start with a simple RX and XDP cleanups: - Replace call to MPWQE free with dealloc in interface down flow - Do not recycle RX pages in interface down flow - Gather all XDP pre-requisite checks in a single function - Restrict the combination of large MTU and XDP Since now XDP logic is going to be called from TX side as well, generic XDP TX logic is not RX only anymore, for that Tariq creates a new xdp.c file and moves XDP related code into it, and generalizes the code to support XDP TX for XDP redirect, such as the xdp tx sq structures and xdp counters. XDP redirect support: Add implementation for the ndo_xdp_xmit callback. Dedicate a new set of XDP-SQ instances to satisfy the XDP_REDIRECT requests. These instances are totally separated from the existing XDP-SQ objects that satisfy local XDP_TX actions. Performance tests: xdp_redirect_map from ConnectX-5 to ConnectX-5. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz Packet-rate of 64B packets. Single queue: 7 Mpps. Multi queue: 55 Mpps. ====================

David S. Miller committed 8 years ago

Commit ecbcd682 parents f61b6db 8ee482310 files · +612 −313Browse files

Unified · Split

10 files
  1. drivers/net/ethernet/mellanox/mlx5/core/Makefile +1 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/en.h +33 −28
  3. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +302 −0
  4. drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +63 −0
  5. drivers/net/ethernet/mellanox/mlx5/core/en_main.c +88 −28
  6. drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +27 −227
  7. drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +66 −14
  8. drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +19 −5
  9. drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +9 −10
  10. drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +4 −0
drivers/net/ethernet/mellanox/mlx5/core/Makefile+1−1
@@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \ · 13 unmodified lines
14 fpga/ipsec.o fpga/tls.o14 fpga/ipsec.o fpga/tls.o
15 15
16mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \16mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
17 en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \17 en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o vxlan.o \
18 en_arfs.o en_fs_ethtool.o en_selftest.o en/port.o18 en_arfs.o en_fs_ethtool.o en_selftest.o en/port.o
19 19
20mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o20mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
drivers/net/ethernet/mellanox/mlx5/core/en.h+33−28
@@ -147,10 +147,6 @@ struct page_pool; · 146 unmodified lines
147 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))147 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
148#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS148#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS
149 149
150#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
151#define MLX5E_XDP_TX_DS_COUNT \
152 ((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
153
154#define MLX5E_NUM_MAIN_GROUPS 9150#define MLX5E_NUM_MAIN_GROUPS 9
155 151
156#define MLX5E_MSG_LEVEL NETIF_MSG_LINK152#define MLX5E_MSG_LEVEL NETIF_MSG_LINK
@@ -348,6 +344,7 @@ enum { · 191 unmodified lines
348 MLX5E_SQ_STATE_IPSEC,344 MLX5E_SQ_STATE_IPSEC,
349 MLX5E_SQ_STATE_AM,345 MLX5E_SQ_STATE_AM,
350 MLX5E_SQ_STATE_TLS,346 MLX5E_SQ_STATE_TLS,
347 MLX5E_SQ_STATE_REDIRECT,
351};348};
352 349
353struct mlx5e_sq_wqe_info {350struct mlx5e_sq_wqe_info {
@@ -368,16 +365,14 @@ struct mlx5e_txqsq { · 14 unmodified lines
368 365
369 struct mlx5e_cq cq;366 struct mlx5e_cq cq;
370 367
371 /* write@xmit, read@completion */
372 struct {
373 struct mlx5e_sq_dma *dma_fifo;
374 struct mlx5e_tx_wqe_info *wqe_info;
375 } db;
376
377 /* read only */368 /* read only */
378 struct mlx5_wq_cyc wq;369 struct mlx5_wq_cyc wq;
379 u32 dma_fifo_mask;370 u32 dma_fifo_mask;
380 struct mlx5e_sq_stats *stats;371 struct mlx5e_sq_stats *stats;
372 struct {
373 struct mlx5e_sq_dma *dma_fifo;
374 struct mlx5e_tx_wqe_info *wqe_info;
375 } db;
381 void __iomem *uar_map;376 void __iomem *uar_map;
382 struct netdev_queue *txq;377 struct netdev_queue *txq;
383 u32 sqn;378 u32 sqn;
@@ -399,30 +394,43 @@ struct mlx5e_txqsq { · 15 unmodified lines
399 } recover;394 } recover;
400} ____cacheline_aligned_in_smp;395} ____cacheline_aligned_in_smp;
401 396
397struct mlx5e_dma_info {
398 struct page *page;
399 dma_addr_t addr;
400};
401
402struct mlx5e_xdp_info {
403 struct xdp_frame *xdpf;
404 dma_addr_t dma_addr;
405 struct mlx5e_dma_info di;
406};
407
402struct mlx5e_xdpsq {408struct mlx5e_xdpsq {
403 /* data path */409 /* data path */
404 410
405 /* dirtied @rx completion */411 /* dirtied @completion */
406 u16 cc;412 u16 cc;
407 u16 pc;413 bool redirect_flush;
408 414
409 struct mlx5e_cq cq;415 /* dirtied @xmit */
416 u16 pc ____cacheline_aligned_in_smp;
417 bool doorbell;
410 418
411 /* write@xmit, read@completion */419 struct mlx5e_cq cq;
412 struct {
413 struct mlx5e_dma_info *di;
414 bool doorbell;
415 bool redirect_flush;
416 } db;
417 420
418 /* read only */421 /* read only */
419 struct mlx5_wq_cyc wq;422 struct mlx5_wq_cyc wq;
423 struct mlx5e_xdpsq_stats *stats;
424 struct {
425 struct mlx5e_xdp_info *xdpi;
426 } db;
420 void __iomem *uar_map;427 void __iomem *uar_map;
421 u32 sqn;428 u32 sqn;
422 struct device *pdev;429 struct device *pdev;
423 __be32 mkey_be;430 __be32 mkey_be;
424 u8 min_inline_mode;431 u8 min_inline_mode;
425 unsigned long state;432 unsigned long state;
433 unsigned int hw_mtu;
426 434
427 /* control path */435 /* control path */
428 struct mlx5_wq_ctrl wq_ctrl;436 struct mlx5_wq_ctrl wq_ctrl;
@@ -459,11 +467,6 @@ mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) · 30 unmodified lines
459 return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);467 return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) || (cc == pc);
460}468}
461 469
462struct mlx5e_dma_info {
463 struct page *page;
464 dma_addr_t addr;
465};
466
467struct mlx5e_wqe_frag_info {470struct mlx5e_wqe_frag_info {
468 struct mlx5e_dma_info *di;471 struct mlx5e_dma_info *di;
469 u32 offset;472 u32 offset;
@@ -566,7 +569,6 @@ struct mlx5e_rq { · 96 unmodified lines
566 569
567 /* XDP */570 /* XDP */
568 struct bpf_prog *xdp_prog;571 struct bpf_prog *xdp_prog;
569 unsigned int hw_mtu;
570 struct mlx5e_xdpsq xdpsq;572 struct mlx5e_xdpsq xdpsq;
571 DECLARE_BITMAP(flags, 8);573 DECLARE_BITMAP(flags, 8);
572 struct page_pool *page_pool;574 struct page_pool *page_pool;
@@ -595,6 +597,9 @@ struct mlx5e_channel { · 22 unmodified lines
595 __be32 mkey_be;597 __be32 mkey_be;
596 u8 num_tc;598 u8 num_tc;
597 599
600 /* XDP_REDIRECT */
601 struct mlx5e_xdpsq xdpsq;
602
598 /* data path - accessed per napi poll */603 /* data path - accessed per napi poll */
599 struct irq_desc *irq_desc;604 struct irq_desc *irq_desc;
600 struct mlx5e_ch_stats *stats;605 struct mlx5e_ch_stats *stats;
@@ -617,6 +622,8 @@ struct mlx5e_channel_stats { · 16 unmodified lines
617 struct mlx5e_ch_stats ch;622 struct mlx5e_ch_stats ch;
618 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];623 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
619 struct mlx5e_rq_stats rq;624 struct mlx5e_rq_stats rq;
625 struct mlx5e_xdpsq_stats rq_xdpsq;
626 struct mlx5e_xdpsq_stats xdpsq;
620} ____cacheline_aligned_in_smp;627} ____cacheline_aligned_in_smp;
621 628
622enum mlx5e_traffic_types {629enum mlx5e_traffic_types {
@@ -876,14 +883,13 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); · 253 unmodified lines
876int mlx5e_napi_poll(struct napi_struct *napi, int budget);883int mlx5e_napi_poll(struct napi_struct *napi, int budget);
877bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);884bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
878int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);885int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
879bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
880void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);886void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
881void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
882 887
883bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);888bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
884bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,889bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
885 struct mlx5e_params *params);890 struct mlx5e_params *params);
886 891
892void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info);
887void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,893void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
888 bool recycle);894 bool recycle);
889void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);895void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
@@ -892,7 +898,6 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq); · 2 unmodified lines
892bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);898bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
893void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);899void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
894void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);900void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
895void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
896struct sk_buff *901struct sk_buff *
897mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,902mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
898 u16 cqe_bcnt, u32 head_offset, u32 page_idx);903 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+302−0
@@ -0,0 +1,302 @@
1/*
2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/bpf_trace.h>
34#include "en/xdp.h"
35
36static inline bool
37mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_dma_info *di,
38 struct xdp_buff *xdp)
39{
40 struct mlx5e_xdp_info xdpi;
41
42 xdpi.xdpf = convert_to_xdp_frame(xdp);
43 if (unlikely(!xdpi.xdpf))
44 return false;
45 xdpi.dma_addr = di->addr + (xdpi.xdpf->data - (void *)xdpi.xdpf);
46 dma_sync_single_for_device(sq->pdev, xdpi.dma_addr,
47 xdpi.xdpf->len, PCI_DMA_TODEVICE);
48 xdpi.di = *di;
49
50 return mlx5e_xmit_xdp_frame(sq, &xdpi);
51}
52
53/* returns true if packet was consumed by xdp */
54bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
55 void *va, u16 *rx_headroom, u32 *len)
56{
57 struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
58 struct xdp_buff xdp;
59 u32 act;
60 int err;
61
62 if (!prog)
63 return false;
64
65 xdp.data = va + *rx_headroom;
66 xdp_set_data_meta_invalid(&xdp);
67 xdp.data_end = xdp.data + *len;
68 xdp.data_hard_start = va;
69 xdp.rxq = &rq->xdp_rxq;
70
71 act = bpf_prog_run_xdp(prog, &xdp);
72 switch (act) {
73 case XDP_PASS:
74 *rx_headroom = xdp.data - xdp.data_hard_start;
75 *len = xdp.data_end - xdp.data;
76 return false;
77 case XDP_TX:
78 if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp)))
79 goto xdp_abort;
80 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
81 return true;
82 case XDP_REDIRECT:
83 /* When XDP enabled then page-refcnt==1 here */
84 err = xdp_do_redirect(rq->netdev, &xdp, prog);
85 if (unlikely(err))
86 goto xdp_abort;
87 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
88 rq->xdpsq.redirect_flush = true;
89 mlx5e_page_dma_unmap(rq, di);
90 rq->stats->xdp_redirect++;
91 return true;
92 default:
93 bpf_warn_invalid_xdp_action(act);
94 case XDP_ABORTED:
95xdp_abort:
96 trace_xdp_exception(rq->netdev, prog, act);
97 case XDP_DROP:
98 rq->stats->xdp_drop++;
99 return true;
100 }
101}
102
103bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi)
104{
105 struct mlx5_wq_cyc *wq = &sq->wq;
106 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
107 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
108
109 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
110 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
111 struct mlx5_wqe_data_seg *dseg = wqe->data;
112
113 struct xdp_frame *xdpf = xdpi->xdpf;
114 dma_addr_t dma_addr = xdpi->dma_addr;
115 unsigned int dma_len = xdpf->len;
116
117 struct mlx5e_xdpsq_stats *stats = sq->stats;
118
119 prefetchw(wqe);
120
121 if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || sq->hw_mtu < dma_len)) {
122 stats->err++;
123 return false;
124 }
125
126 if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
127 if (sq->doorbell) {
128 /* SQ is full, ring doorbell */
129 mlx5e_xmit_xdp_doorbell(sq);
130 sq->doorbell = false;
131 }
132 stats->full++;
133 return false;
134 }
135
136 cseg->fm_ce_se = 0;
137
138 /* copy the inline part if required */
139 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
140 memcpy(eseg->inline_hdr.start, xdpf->data, MLX5E_XDP_MIN_INLINE);
141 eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
142 dma_len -= MLX5E_XDP_MIN_INLINE;
143 dma_addr += MLX5E_XDP_MIN_INLINE;
144 dseg++;
145 }
146
147 /* write the dma part */
148 dseg->addr = cpu_to_be64(dma_addr);
149 dseg->byte_count = cpu_to_be32(dma_len);
150
151 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
152
153 /* move page to reference to sq responsibility,
154 * and mark so it's not put back in page-cache.
155 */
156 sq->db.xdpi[pi] = *xdpi;
157 sq->pc++;
158
159 sq->doorbell = true;
160
161 stats->xmit++;
162 return true;
163}
164
165bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
166{
167 struct mlx5e_xdpsq *sq;
168 struct mlx5_cqe64 *cqe;
169 struct mlx5e_rq *rq;
170 bool is_redirect;
171 u16 sqcc;
172 int i;
173
174 sq = container_of(cq, struct mlx5e_xdpsq, cq);
175
176 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
177 return false;
178
179 cqe = mlx5_cqwq_get_cqe(&cq->wq);
180 if (!cqe)
181 return false;
182
183 is_redirect = test_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
184 rq = container_of(sq, struct mlx5e_rq, xdpsq);
185
186 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
187 * otherwise a cq overrun may occur
188 */
189 sqcc = sq->cc;
190
191 i = 0;
192 do {
193 u16 wqe_counter;
194 bool last_wqe;
195
196 mlx5_cqwq_pop(&cq->wq);
197
198 wqe_counter = be16_to_cpu(cqe->wqe_counter);
199
200 do {
201 u16 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
202 struct mlx5e_xdp_info *xdpi = &sq->db.xdpi[ci];
203
204 last_wqe = (sqcc == wqe_counter);
205 sqcc++;
206
207 if (is_redirect) {
208 xdp_return_frame(xdpi->xdpf);
209 dma_unmap_single(sq->pdev, xdpi->dma_addr,
210 xdpi->xdpf->len, DMA_TO_DEVICE);
211 } else {
212 /* Recycle RX page */
213 mlx5e_page_release(rq, &xdpi->di, true);
214 }
215 } while (!last_wqe);
216 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
217
218 sq->stats->cqes += i;
219
220 mlx5_cqwq_update_db_record(&cq->wq);
221
222 /* ensure cq space is freed before enabling more cqes */
223 wmb();
224
225 sq->cc = sqcc;
226 return (i == MLX5E_TX_CQ_POLL_BUDGET);
227}
228
229void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
230{
231 struct mlx5e_rq *rq;
232 bool is_redirect;
233
234 is_redirect = test_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
235 rq = is_redirect ? NULL : container_of(sq, struct mlx5e_rq, xdpsq);
236
237 while (sq->cc != sq->pc) {
238 u16 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
239 struct mlx5e_xdp_info *xdpi = &sq->db.xdpi[ci];
240
241 sq->cc++;
242
243 if (is_redirect) {
244 xdp_return_frame(xdpi->xdpf);
245 dma_unmap_single(sq->pdev, xdpi->dma_addr,
246 xdpi->xdpf->len, DMA_TO_DEVICE);
247 } else {
248 /* Recycle RX page */
249 mlx5e_page_release(rq, &xdpi->di, false);
250 }
251 }
252}
253
254int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
255 u32 flags)
256{
257 struct mlx5e_priv *priv = netdev_priv(dev);
258 struct mlx5e_xdpsq *sq;
259 int drops = 0;
260 int sq_num;
261 int i;
262
263 if (unlikely(!test_bit(MLX5E_STATE_OPENED, &priv->state)))
264 return -ENETDOWN;
265
266 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
267 return -EINVAL;
268
269 sq_num = smp_processor_id();
270
271 if (unlikely(sq_num >= priv->channels.num))
272 return -ENXIO;
273
274 sq = &priv->channels.c[sq_num]->xdpsq;
275
276 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
277 return -ENETDOWN;
278
279 for (i = 0; i < n; i++) {
280 struct xdp_frame *xdpf = frames[i];
281 struct mlx5e_xdp_info xdpi;
282
283 xdpi.dma_addr = dma_map_single(sq->pdev, xdpf->data, xdpf->len,
284 DMA_TO_DEVICE);
285 if (unlikely(dma_mapping_error(sq->pdev, xdpi.dma_addr))) {
286 drops++;
287 continue;
288 }
289
290 xdpi.xdpf = xdpf;
291
292 if (unlikely(!mlx5e_xmit_xdp_frame(sq, &xdpi))) {
293 xdp_return_frame_rx_napi(xdpf);
294 drops++;
295 }
296 }
297
298 if (flags & XDP_XMIT_FLUSH)
299 mlx5e_xmit_xdp_doorbell(sq);
300
301 return n - drops;
302}
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+63−0
@@ -0,0 +1,63 @@
1/*
2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#ifndef __MLX5_EN_XDP_H__
33#define __MLX5_EN_XDP_H__
34
35#include "en.h"
36
37#define MLX5E_XDP_MAX_MTU ((int)(PAGE_SIZE - \
38 MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM)))
39#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
40#define MLX5E_XDP_TX_DS_COUNT \
41 ((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
42
43bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
44 void *va, u16 *rx_headroom, u32 *len);
45bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
46void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
47
48bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xdp_info *xdpi);
49int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
50 u32 flags);
51
52static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
53{
54 struct mlx5_wq_cyc *wq = &sq->wq;
55 struct mlx5e_tx_wqe *wqe;
56 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
57
58 wqe = mlx5_wq_cyc_get_wqe(wq, pi);
59
60 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
61}
62
63#endif
drivers/net/ethernet/mellanox/mlx5/core/en_main.c+88−28
@@ -47,6 +47,7 @@ · 46 unmodified lines
47#include "accel/tls.h"47#include "accel/tls.h"
48#include "vxlan.h"48#include "vxlan.h"
49#include "en/port.h"49#include "en/port.h"
50#include "en/xdp.h"
50 51
51struct mlx5e_rq_param {52struct mlx5e_rq_param {
52 u32 rqc[MLX5_ST_SZ_DW(rqc)];53 u32 rqc[MLX5_ST_SZ_DW(rqc)];
@@ -96,14 +97,19 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) · 43 unmodified lines
96 97
97static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)98static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
98{99{
99 if (!params->xdp_prog) {100 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
100 u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);101 u16 linear_rq_headroom = params->xdp_prog ?
101 u16 rq_headroom = MLX5_RX_HEADROOM + NET_IP_ALIGN;102 XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
103 u32 frag_sz;
102 104
103 return MLX5_SKB_FRAG_SZ(rq_headroom + hw_mtu);105 linear_rq_headroom += NET_IP_ALIGN;
104 }
105 106
106 return PAGE_SIZE;107 frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
108
109 if (params->xdp_prog && frag_sz < PAGE_SIZE)
110 frag_sz = PAGE_SIZE;
111
112 return frag_sz;
107}113}
108 114
109static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)115static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
@@ -485,7 +491,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 375 unmodified lines
485 rq->channel = c;491 rq->channel = c;
486 rq->ix = c->ix;492 rq->ix = c->ix;
487 rq->mdev = mdev;493 rq->mdev = mdev;
488 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
489 rq->stats = &c->priv->channel_stats[c->ix].rq;494 rq->stats = &c->priv->channel_stats[c->ix].rq;
490 495
491 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;496 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
@@ -877,7 +882,7 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) · 385 unmodified lines
877 882
878 /* UMR WQE (if in progress) is always at wq->head */883 /* UMR WQE (if in progress) is always at wq->head */
879 if (rq->mpwqe.umr_in_progress)884 if (rq->mpwqe.umr_in_progress)
880 mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);885 rq->dealloc_wqe(rq, wq->head);
881 886
882 while (!mlx5_wq_ll_is_empty(wq)) {887 while (!mlx5_wq_ll_is_empty(wq)) {
883 struct mlx5e_rx_wqe_ll *wqe;888 struct mlx5e_rx_wqe_ll *wqe;
@@ -963,16 +968,16 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq) · 79 unmodified lines
963 968
964static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)969static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
965{970{
966 kvfree(sq->db.di);971 kvfree(sq->db.xdpi);
967}972}
968 973
969static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)974static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
970{975{
971 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);976 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
972 977
973 sq->db.di = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.di)),978 sq->db.xdpi = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.xdpi)),
974 GFP_KERNEL, numa);979 GFP_KERNEL, numa);
975 if (!sq->db.di) {980 if (!sq->db.xdpi) {
976 mlx5e_free_xdpsq_db(sq);981 mlx5e_free_xdpsq_db(sq);
977 return -ENOMEM;982 return -ENOMEM;
978 }983 }
@@ -983,7 +988,8 @@ static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) · 4 unmodified lines
983static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,988static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
984 struct mlx5e_params *params,989 struct mlx5e_params *params,
985 struct mlx5e_sq_param *param,990 struct mlx5e_sq_param *param,
986 struct mlx5e_xdpsq *sq)991 struct mlx5e_xdpsq *sq,
992 bool is_redirect)
987{993{
988 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);994 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
989 struct mlx5_core_dev *mdev = c->mdev;995 struct mlx5_core_dev *mdev = c->mdev;
@@ -995,6 +1001,10 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, · 5 unmodified lines
995 sq->channel = c;1001 sq->channel = c;
996 sq->uar_map = mdev->mlx5e_res.bfreg.map;1002 sq->uar_map = mdev->mlx5e_res.bfreg.map;
997 sq->min_inline_mode = params->tx_min_inline_mode;1003 sq->min_inline_mode = params->tx_min_inline_mode;
1004 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1005 sq->stats = is_redirect ?
1006 &c->priv->channel_stats[c->ix].xdpsq :
1007 &c->priv->channel_stats[c->ix].rq_xdpsq;
998 1008
999 param->wq.db_numa_node = cpu_to_node(c->cpu);1009 param->wq.db_numa_node = cpu_to_node(c->cpu);
1000 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);1010 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@@ -1524,7 +1534,8 @@ static void mlx5e_close_icosq(struct mlx5e_icosq *sq) · 523 unmodified lines
1524static int mlx5e_open_xdpsq(struct mlx5e_channel *c,1534static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
1525 struct mlx5e_params *params,1535 struct mlx5e_params *params,
1526 struct mlx5e_sq_param *param,1536 struct mlx5e_sq_param *param,
1527 struct mlx5e_xdpsq *sq)1537 struct mlx5e_xdpsq *sq,
1538 bool is_redirect)
1528{1539{
1529 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;1540 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1530 struct mlx5e_create_sq_param csp = {};1541 struct mlx5e_create_sq_param csp = {};
@@ -1532,7 +1543,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 1 unmodified line
1532 int err;1543 int err;
1533 int i;1544 int i;
1534 1545
1535 err = mlx5e_alloc_xdpsq(c, params, param, sq);1546 err = mlx5e_alloc_xdpsq(c, params, param, sq, is_redirect);
1536 if (err)1547 if (err)
1537 return err;1548 return err;
1538 1549
@@ -1541,6 +1552,8 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 2 unmodified lines
1541 csp.cqn = sq->cq.mcq.cqn;1552 csp.cqn = sq->cq.mcq.cqn;
1542 csp.wq_ctrl = &sq->wq_ctrl;1553 csp.wq_ctrl = &sq->wq_ctrl;
1543 csp.min_inline_mode = sq->min_inline_mode;1554 csp.min_inline_mode = sq->min_inline_mode;
1555 if (is_redirect)
1556 set_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
1544 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);1557 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1545 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);1558 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
1546 if (err)1559 if (err)
@@ -1923,10 +1936,14 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 376 unmodified lines
1923 if (err)1936 if (err)
1924 goto err_close_icosq_cq;1937 goto err_close_icosq_cq;
1925 1938
1926 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);1939 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq);
1927 if (err)1940 if (err)
1928 goto err_close_tx_cqs;1941 goto err_close_tx_cqs;
1929 1942
1943 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
1944 if (err)
1945 goto err_close_xdp_tx_cqs;
1946
1930 /* XDP SQ CQ params are same as normal TXQ sq CQ params */1947 /* XDP SQ CQ params are same as normal TXQ sq CQ params */
1931 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,1948 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1932 &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;1949 &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
@@ -1943,7 +1960,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 10 unmodified lines
1943 if (err)1960 if (err)
1944 goto err_close_icosq;1961 goto err_close_icosq;
1945 1962
1946 err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq) : 0;1963 err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0;
1947 if (err)1964 if (err)
1948 goto err_close_sqs;1965 goto err_close_sqs;
1949 1966
@@ -1951,9 +1968,17 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 1 unmodified line
1951 if (err)1968 if (err)
1952 goto err_close_xdp_sq;1969 goto err_close_xdp_sq;
1953 1970
1971 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->xdpsq, true);
1972 if (err)
1973 goto err_close_rq;
1974
1954 *cp = c;1975 *cp = c;
1955 1976
1956 return 0;1977 return 0;
1978
1979err_close_rq:
1980 mlx5e_close_rq(&c->rq);
1981
1957err_close_xdp_sq:1982err_close_xdp_sq:
1958 if (c->xdp)1983 if (c->xdp)
1959 mlx5e_close_xdpsq(&c->rq.xdpsq);1984 mlx5e_close_xdpsq(&c->rq.xdpsq);
@@ -1972,6 +1997,9 @@ err_disable_napi: · 12 unmodified lines
1972err_close_rx_cq:1997err_close_rx_cq:
1973 mlx5e_close_cq(&c->rq.cq);1998 mlx5e_close_cq(&c->rq.cq);
1974 1999
2000err_close_xdp_tx_cqs:
2001 mlx5e_close_cq(&c->xdpsq.cq);
2002
1975err_close_tx_cqs:2003err_close_tx_cqs:
1976 mlx5e_close_tx_cqs(c);2004 mlx5e_close_tx_cqs(c);
1977 2005
@@ -2006,6 +2034,7 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) · 28 unmodified lines
2006 2034
2007static void mlx5e_close_channel(struct mlx5e_channel *c)2035static void mlx5e_close_channel(struct mlx5e_channel *c)
2008{2036{
2037 mlx5e_close_xdpsq(&c->xdpsq);
2009 mlx5e_close_rq(&c->rq);2038 mlx5e_close_rq(&c->rq);
2010 if (c->xdp)2039 if (c->xdp)
2011 mlx5e_close_xdpsq(&c->rq.xdpsq);2040 mlx5e_close_xdpsq(&c->rq.xdpsq);
@@ -2015,6 +2044,7 @@ static void mlx5e_close_channel(struct mlx5e_channel *c) · 3 unmodified lines
2015 if (c->xdp)2044 if (c->xdp)
2016 mlx5e_close_cq(&c->rq.xdpsq.cq);2045 mlx5e_close_cq(&c->rq.xdpsq.cq);
2017 mlx5e_close_cq(&c->rq.cq);2046 mlx5e_close_cq(&c->rq.cq);
2047 mlx5e_close_cq(&c->xdpsq.cq);
2018 mlx5e_close_tx_cqs(c);2048 mlx5e_close_tx_cqs(c);
2019 mlx5e_close_cq(&c->icosq.cq);2049 mlx5e_close_cq(&c->icosq.cq);
2020 netif_napi_del(&c->napi);2050 netif_napi_del(&c->napi);
@@ -3707,6 +3737,14 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, · 1686 unmodified lines
3707 new_channels.params = *params;3737 new_channels.params = *params;
3708 new_channels.params.sw_mtu = new_mtu;3738 new_channels.params.sw_mtu = new_mtu;
3709 3739
3740 if (params->xdp_prog &&
3741 !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
3742 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3743 new_mtu, MLX5E_XDP_MAX_MTU);
3744 err = -EINVAL;
3745 goto out;
3746 }
3747
3710 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {3748 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3711 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);3749 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);
3712 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);3750 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);
@@ -4094,26 +4132,47 @@ static void mlx5e_tx_timeout(struct net_device *dev) · 381 unmodified lines
4094 queue_work(priv->wq, &priv->tx_timeout_work);4132 queue_work(priv->wq, &priv->tx_timeout_work);
4095}4133}
4096 4134
4135static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
4136{
4137 struct net_device *netdev = priv->netdev;
4138 struct mlx5e_channels new_channels = {};
4139
4140 if (priv->channels.params.lro_en) {
4141 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
4142 return -EINVAL;
4143 }
4144
4145 if (MLX5_IPSEC_DEV(priv->mdev)) {
4146 netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4147 return -EINVAL;
4148 }
4149
4150 new_channels.params = priv->channels.params;
4151 new_channels.params.xdp_prog = prog;
4152
4153 if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
4154 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4155 new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
4156 return -EINVAL;
4157 }
4158
4159 return 0;
4160}
4161
4097static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)4162static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4098{4163{
4099 struct mlx5e_priv *priv = netdev_priv(netdev);4164 struct mlx5e_priv *priv = netdev_priv(netdev);
4100 struct bpf_prog *old_prog;4165 struct bpf_prog *old_prog;
4101 int err = 0;
4102 bool reset, was_opened;4166 bool reset, was_opened;
4167 int err;
4103 int i;4168 int i;
4104 4169
4105 mutex_lock(&priv->state_lock);4170 mutex_lock(&priv->state_lock);
4106 4171
4107 if ((netdev->features & NETIF_F_LRO) && prog) {4172 if (prog) {
4108 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");4173 err = mlx5e_xdp_allowed(priv, prog);
4109 err = -EINVAL;4174 if (err)
4110 goto unlock;4175 goto unlock;
4111 }
4112
4113 if ((netdev->features & NETIF_F_HW_ESP) && prog) {
4114 netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4115 err = -EINVAL;
4116 goto unlock;
4117 }4176 }
4118 4177
4119 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);4178 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
@@ -4242,6 +4301,7 @@ static const struct net_device_ops mlx5e_netdev_ops = { · 122 unmodified lines
4242#endif4301#endif
4243 .ndo_tx_timeout = mlx5e_tx_timeout,4302 .ndo_tx_timeout = mlx5e_tx_timeout,
4244 .ndo_bpf = mlx5e_xdp,4303 .ndo_bpf = mlx5e_xdp,
4304 .ndo_xdp_xmit = mlx5e_xdp_xmit,
4245#ifdef CONFIG_NET_POLL_CONTROLLER4305#ifdef CONFIG_NET_POLL_CONTROLLER
4246 .ndo_poll_controller = mlx5e_netpoll,4306 .ndo_poll_controller = mlx5e_netpoll,
4247#endif4307#endif
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c+27−227
@@ -34,7 +34,6 @@ · 33 unmodified lines
34#include <linux/ip.h>34#include <linux/ip.h>
35#include <linux/ipv6.h>35#include <linux/ipv6.h>
36#include <linux/tcp.h>36#include <linux/tcp.h>
37#include <linux/bpf_trace.h>
38#include <net/busy_poll.h>37#include <net/busy_poll.h>
39#include <net/ip6_checksum.h>38#include <net/ip6_checksum.h>
40#include <net/page_pool.h>39#include <net/page_pool.h>
@@ -46,6 +45,7 @@ · 5 unmodified lines
46#include "en_accel/ipsec_rxtx.h"45#include "en_accel/ipsec_rxtx.h"
47#include "en_accel/tls_rxtx.h"46#include "en_accel/tls_rxtx.h"
48#include "lib/clock.h"47#include "lib/clock.h"
48#include "en/xdp.h"
49 49
50static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)50static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
51{51{
@@ -239,8 +239,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq, · 187 unmodified lines
239 return 0;239 return 0;
240}240}
241 241
242static void mlx5e_page_dma_unmap(struct mlx5e_rq *rq,242void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info)
243 struct mlx5e_dma_info *dma_info)
244{243{
245 dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);244 dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
246}245}
@@ -277,10 +276,11 @@ static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, · 30 unmodified lines
277}276}
278 277
279static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,278static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
280 struct mlx5e_wqe_frag_info *frag)279 struct mlx5e_wqe_frag_info *frag,
280 bool recycle)
281{281{
282 if (frag->last_in_page)282 if (frag->last_in_page)
283 mlx5e_page_release(rq, frag->di, true);283 mlx5e_page_release(rq, frag->di, recycle);
284}284}
285 285
286static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)286static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
@@ -308,25 +308,26 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, · 21 unmodified lines
308 308
309free_frags:309free_frags:
310 while (--i >= 0)310 while (--i >= 0)
311 mlx5e_put_rx_frag(rq, --frag);311 mlx5e_put_rx_frag(rq, --frag, true);
312 312
313 return err;313 return err;
314}314}
315 315
316static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,316static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
317 struct mlx5e_wqe_frag_info *wi)317 struct mlx5e_wqe_frag_info *wi,
318 bool recycle)
318{319{
319 int i;320 int i;
320 321
321 for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)322 for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
322 mlx5e_put_rx_frag(rq, wi);323 mlx5e_put_rx_frag(rq, wi, recycle);
323}324}
324 325
325void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)326void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
326{327{
327 struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);328 struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
328 329
329 mlx5e_free_rx_wqe(rq, wi);330 mlx5e_free_rx_wqe(rq, wi, false);
330}331}
331 332
332static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)333static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
@@ -396,7 +397,8 @@ mlx5e_copy_skb_header_mpwqe(struct device *pdev, · 63 unmodified lines
396 }397 }
397}398}
398 399
399void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)400static void
401mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
400{402{
401 const bool no_xdp_xmit =403 const bool no_xdp_xmit =
402 bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);404 bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
@@ -405,7 +407,7 @@ void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi) · 2 unmodified lines
405 407
406 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)408 for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
407 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))409 if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
408 mlx5e_page_release(rq, &dma_info[i], true);410 mlx5e_page_release(rq, &dma_info[i], recycle);
409}411}
410 412
411static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)413static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
@@ -505,8 +507,8 @@ err_unmap: · 93 unmodified lines
505void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)507void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
506{508{
507 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];509 struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
508 510 /* Don't recycle, this function is called on rq/netdev close */
509 mlx5e_free_rx_mpwqe(rq, wi);511 mlx5e_free_rx_mpwqe(rq, wi, false);
510}512}
511 513
512bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)514bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
@@ -847,135 +849,6 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, · 334 unmodified lines
847 mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);849 mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
848}850}
849 851
850static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
851{
852 struct mlx5_wq_cyc *wq = &sq->wq;
853 struct mlx5e_tx_wqe *wqe;
854 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
855
856 wqe = mlx5_wq_cyc_get_wqe(wq, pi);
857
858 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
859}
860
861static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
862 struct mlx5e_dma_info *di,
863 const struct xdp_buff *xdp)
864{
865 struct mlx5e_xdpsq *sq = &rq->xdpsq;
866 struct mlx5_wq_cyc *wq = &sq->wq;
867 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
868 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
869
870 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
871 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
872 struct mlx5_wqe_data_seg *dseg;
873
874 ptrdiff_t data_offset = xdp->data - xdp->data_hard_start;
875 dma_addr_t dma_addr = di->addr + data_offset;
876 unsigned int dma_len = xdp->data_end - xdp->data;
877
878 struct mlx5e_rq_stats *stats = rq->stats;
879
880 prefetchw(wqe);
881
882 if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || rq->hw_mtu < dma_len)) {
883 stats->xdp_drop++;
884 return false;
885 }
886
887 if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
888 if (sq->db.doorbell) {
889 /* SQ is full, ring doorbell */
890 mlx5e_xmit_xdp_doorbell(sq);
891 sq->db.doorbell = false;
892 }
893 stats->xdp_tx_full++;
894 return false;
895 }
896
897 dma_sync_single_for_device(sq->pdev, dma_addr, dma_len, PCI_DMA_TODEVICE);
898
899 cseg->fm_ce_se = 0;
900
901 dseg = (struct mlx5_wqe_data_seg *)eseg + 1;
902
903 /* copy the inline part if required */
904 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
905 memcpy(eseg->inline_hdr.start, xdp->data, MLX5E_XDP_MIN_INLINE);
906 eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
907 dma_len -= MLX5E_XDP_MIN_INLINE;
908 dma_addr += MLX5E_XDP_MIN_INLINE;
909 dseg++;
910 }
911
912 /* write the dma part */
913 dseg->addr = cpu_to_be64(dma_addr);
914 dseg->byte_count = cpu_to_be32(dma_len);
915
916 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_SEND);
917
918 /* move page to reference to sq responsibility,
919 * and mark so it's not put back in page-cache.
920 */
921 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
922 sq->db.di[pi] = *di;
923 sq->pc++;
924
925 sq->db.doorbell = true;
926
927 stats->xdp_tx++;
928 return true;
929}
930
931/* returns true if packet was consumed by xdp */
932static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
933 struct mlx5e_dma_info *di,
934 void *va, u16 *rx_headroom, u32 *len)
935{
936 struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
937 struct xdp_buff xdp;
938 u32 act;
939 int err;
940
941 if (!prog)
942 return false;
943
944 xdp.data = va + *rx_headroom;
945 xdp_set_data_meta_invalid(&xdp);
946 xdp.data_end = xdp.data + *len;
947 xdp.data_hard_start = va;
948 xdp.rxq = &rq->xdp_rxq;
949
950 act = bpf_prog_run_xdp(prog, &xdp);
951 switch (act) {
952 case XDP_PASS:
953 *rx_headroom = xdp.data - xdp.data_hard_start;
954 *len = xdp.data_end - xdp.data;
955 return false;
956 case XDP_TX:
957 if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
958 trace_xdp_exception(rq->netdev, prog, act);
959 return true;
960 case XDP_REDIRECT:
961 /* When XDP enabled then page-refcnt==1 here */
962 err = xdp_do_redirect(rq->netdev, &xdp, prog);
963 if (!err) {
964 __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
965 rq->xdpsq.db.redirect_flush = true;
966 mlx5e_page_dma_unmap(rq, di);
967 }
968 return true;
969 default:
970 bpf_warn_invalid_xdp_action(act);
971 case XDP_ABORTED:
972 trace_xdp_exception(rq->netdev, prog, act);
973 case XDP_DROP:
974 rq->stats->xdp_drop++;
975 return true;
976 }
977}
978
979static inline852static inline
980struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,853struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
981 u32 frag_size, u16 headroom,854 u32 frag_size, u16 headroom,
@@ -1113,7 +986,7 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) · 131 unmodified lines
1113 napi_gro_receive(rq->cq.napi, skb);986 napi_gro_receive(rq->cq.napi, skb);
1114 987
1115free_wqe:988free_wqe:
1116 mlx5e_free_rx_wqe(rq, wi);989 mlx5e_free_rx_wqe(rq, wi, true);
1117wq_cyc_pop:990wq_cyc_pop:
1118 mlx5_wq_cyc_pop(wq);991 mlx5_wq_cyc_pop(wq);
1119}992}
@@ -1155,7 +1028,7 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) · 35 unmodified lines
1155 napi_gro_receive(rq->cq.napi, skb);1028 napi_gro_receive(rq->cq.napi, skb);
1156 1029
1157free_wqe:1030free_wqe:
1158 mlx5e_free_rx_wqe(rq, wi);1031 mlx5e_free_rx_wqe(rq, wi, true);
1159wq_cyc_pop:1032wq_cyc_pop:
1160 mlx5_wq_cyc_pop(wq);1033 mlx5_wq_cyc_pop(wq);
1161}1034}
@@ -1226,6 +1099,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, · 64 unmodified lines
1226 1099
1227 dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,1100 dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1228 frag_size, DMA_FROM_DEVICE);1101 frag_size, DMA_FROM_DEVICE);
1102 prefetchw(va); /* xdp_frame data area */
1229 prefetch(data);1103 prefetch(data);
1230 1104
1231 rcu_read_lock();1105 rcu_read_lock();
@@ -1292,7 +1166,7 @@ mpwrq_cqe_out: · 60 unmodified lines
1292 1166
1293 wq = &rq->mpwqe.wq;1167 wq = &rq->mpwqe.wq;
1294 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);1168 wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1295 mlx5e_free_rx_mpwqe(rq, wi);1169 mlx5e_free_rx_mpwqe(rq, wi, true);
1296 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);1170 mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1297}1171}
1298 1172
@@ -1328,14 +1202,14 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) · 29 unmodified lines
1328 rq->handle_rx_cqe(rq, cqe);1202 rq->handle_rx_cqe(rq, cqe);
1329 } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));1203 } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
1330 1204
1331 if (xdpsq->db.doorbell) {1205 if (xdpsq->doorbell) {
1332 mlx5e_xmit_xdp_doorbell(xdpsq);1206 mlx5e_xmit_xdp_doorbell(xdpsq);
1333 xdpsq->db.doorbell = false;1207 xdpsq->doorbell = false;
1334 }1208 }
1335 1209
1336 if (xdpsq->db.redirect_flush) {1210 if (xdpsq->redirect_flush) {
1337 xdp_do_flush_map();1211 xdp_do_flush_map();
1338 xdpsq->db.redirect_flush = false;1212 xdpsq->redirect_flush = false;
1339 }1213 }
1340 1214
1341 mlx5_cqwq_update_db_record(&cq->wq);1215 mlx5_cqwq_update_db_record(&cq->wq);
@@ -1346,80 +1220,6 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) · 4 unmodified lines
1346 return work_done;1220 return work_done;
1347}1221}
1348 1222
1349bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
1350{
1351 struct mlx5e_xdpsq *sq;
1352 struct mlx5_cqe64 *cqe;
1353 struct mlx5e_rq *rq;
1354 u16 sqcc;
1355 int i;
1356
1357 sq = container_of(cq, struct mlx5e_xdpsq, cq);
1358
1359 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
1360 return false;
1361
1362 cqe = mlx5_cqwq_get_cqe(&cq->wq);
1363 if (!cqe)
1364 return false;
1365
1366 rq = container_of(sq, struct mlx5e_rq, xdpsq);
1367
1368 /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
1369 * otherwise a cq overrun may occur
1370 */
1371 sqcc = sq->cc;
1372
1373 i = 0;
1374 do {
1375 u16 wqe_counter;
1376 bool last_wqe;
1377
1378 mlx5_cqwq_pop(&cq->wq);
1379
1380 wqe_counter = be16_to_cpu(cqe->wqe_counter);
1381
1382 do {
1383 struct mlx5e_dma_info *di;
1384 u16 ci;
1385
1386 last_wqe = (sqcc == wqe_counter);
1387
1388 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
1389 di = &sq->db.di[ci];
1390
1391 sqcc++;
1392 /* Recycle RX page */
1393 mlx5e_page_release(rq, di, true);
1394 } while (!last_wqe);
1395 } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
1396
1397 rq->stats->xdp_tx_cqe += i;
1398
1399 mlx5_cqwq_update_db_record(&cq->wq);
1400
1401 /* ensure cq space is freed before enabling more cqes */
1402 wmb();
1403
1404 sq->cc = sqcc;
1405 return (i == MLX5E_TX_CQ_POLL_BUDGET);
1406}
1407
1408void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
1409{
1410 struct mlx5e_rq *rq = container_of(sq, struct mlx5e_rq, xdpsq);
1411 struct mlx5e_dma_info *di;
1412 u16 ci;
1413
1414 while (sq->cc != sq->pc) {
1415 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
1416 di = &sq->db.di[ci];
1417 sq->cc++;
1418
1419 mlx5e_page_release(rq, di, false);
1420 }
1421}
1422
1423#ifdef CONFIG_MLX5_CORE_IPOIB1223#ifdef CONFIG_MLX5_CORE_IPOIB
1424 1224
1425#define MLX5_IB_GRH_DGID_OFFSET 241225#define MLX5_IB_GRH_DGID_OFFSET 24
@@ -1521,7 +1321,7 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) · 95 unmodified lines
1521 napi_gro_receive(rq->cq.napi, skb);1321 napi_gro_receive(rq->cq.napi, skb);
1522 1322
1523wq_free_wqe:1323wq_free_wqe:
1524 mlx5e_free_rx_wqe(rq, wi);1324 mlx5e_free_rx_wqe(rq, wi, true);
1525 mlx5_wq_cyc_pop(wq);1325 mlx5_wq_cyc_pop(wq);
1526}1326}
1527 1327
@@ -1544,19 +1344,19 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) · 16 unmodified lines
1544 skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);1344 skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
1545 if (unlikely(!skb)) {1345 if (unlikely(!skb)) {
1546 /* a DROP, save the page-reuse checks */1346 /* a DROP, save the page-reuse checks */
1547 mlx5e_free_rx_wqe(rq, wi);1347 mlx5e_free_rx_wqe(rq, wi, true);
1548 goto wq_cyc_pop;1348 goto wq_cyc_pop;
1549 }1349 }
1550 skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);1350 skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
1551 if (unlikely(!skb)) {1351 if (unlikely(!skb)) {
1552 mlx5e_free_rx_wqe(rq, wi);1352 mlx5e_free_rx_wqe(rq, wi, true);
1553 goto wq_cyc_pop;1353 goto wq_cyc_pop;
1554 }1354 }
1555 1355
1556 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);1356 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1557 napi_gro_receive(rq->cq.napi, skb);1357 napi_gro_receive(rq->cq.napi, skb);
1558 1358
1559 mlx5e_free_rx_wqe(rq, wi);1359 mlx5e_free_rx_wqe(rq, wi, true);
1560wq_cyc_pop:1360wq_cyc_pop:
1561 mlx5_wq_cyc_pop(wq);1361 mlx5_wq_cyc_pop(wq);
1562}1362}
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c+66−14
@@ -59,9 +59,11 @@ static const struct counter_desc sw_stats_desc[] = { · 58 unmodified lines
59 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) },59 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) },
60 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) },60 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) },
61 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },61 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
62 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx) },62 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },
63 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },63 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },
64 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },64 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
65 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },
66 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
65 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_none) },67 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_none) },
66 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial) },68 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial) },
67 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial_inner) },69 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial_inner) },
@@ -73,6 +75,10 @@ static const struct counter_desc sw_stats_desc[] = { · 5 unmodified lines
73 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },75 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
74 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_udp_seg_rem) },76 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_udp_seg_rem) },
75 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },77 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },
78 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },
79 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },
80 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },
81 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },
76 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_wqe_err) },82 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_wqe_err) },
77 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_cqes) },83 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_cqes) },
78 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_strides) },84 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_strides) },
@@ -128,6 +134,8 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 49 unmodified lines
128 for (i = 0; i < priv->profile->max_nch(priv->mdev); i++) {134 for (i = 0; i < priv->profile->max_nch(priv->mdev); i++) {
129 struct mlx5e_channel_stats *channel_stats =135 struct mlx5e_channel_stats *channel_stats =
130 &priv->channel_stats[i];136 &priv->channel_stats[i];
137 struct mlx5e_xdpsq_stats *xdpsq_red_stats = &channel_stats->xdpsq;
138 struct mlx5e_xdpsq_stats *xdpsq_stats = &channel_stats->rq_xdpsq;
131 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;139 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
132 struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;140 struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;
133 int j;141 int j;
@@ -141,10 +149,12 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 7 unmodified lines
141 s->rx_csum_complete += rq_stats->csum_complete;149 s->rx_csum_complete += rq_stats->csum_complete;
142 s->rx_csum_unnecessary += rq_stats->csum_unnecessary;150 s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
143 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;151 s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
144 s->rx_xdp_drop += rq_stats->xdp_drop;152 s->rx_xdp_drop += rq_stats->xdp_drop;
145 s->rx_xdp_tx += rq_stats->xdp_tx;153 s->rx_xdp_redirect += rq_stats->xdp_redirect;
146 s->rx_xdp_tx_cqe += rq_stats->xdp_tx_cqe;154 s->rx_xdp_tx_xmit += xdpsq_stats->xmit;
147 s->rx_xdp_tx_full += rq_stats->xdp_tx_full;155 s->rx_xdp_tx_full += xdpsq_stats->full;
156 s->rx_xdp_tx_err += xdpsq_stats->err;
157 s->rx_xdp_tx_cqe += xdpsq_stats->cqes;
148 s->rx_wqe_err += rq_stats->wqe_err;158 s->rx_wqe_err += rq_stats->wqe_err;
149 s->rx_mpwqe_filler_cqes += rq_stats->mpwqe_filler_cqes;159 s->rx_mpwqe_filler_cqes += rq_stats->mpwqe_filler_cqes;
150 s->rx_mpwqe_filler_strides += rq_stats->mpwqe_filler_strides;160 s->rx_mpwqe_filler_strides += rq_stats->mpwqe_filler_strides;
@@ -162,7 +172,12 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 11 unmodified lines
162 s->ch_poll += ch_stats->poll;172 s->ch_poll += ch_stats->poll;
163 s->ch_arm += ch_stats->arm;173 s->ch_arm += ch_stats->arm;
164 s->ch_aff_change += ch_stats->aff_change;174 s->ch_aff_change += ch_stats->aff_change;
165 s->ch_eq_rearm += ch_stats->eq_rearm;175 s->ch_eq_rearm += ch_stats->eq_rearm;
176 /* xdp redirect */
177 s->tx_xdp_xmit += xdpsq_red_stats->xmit;
178 s->tx_xdp_full += xdpsq_red_stats->full;
179 s->tx_xdp_err += xdpsq_red_stats->err;
180 s->tx_xdp_cqes += xdpsq_red_stats->cqes;
166 181
167 for (j = 0; j < priv->max_opened_tc; j++) {182 for (j = 0; j < priv->max_opened_tc; j++) {
168 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];183 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
@@ -1126,9 +1141,7 @@ static const struct counter_desc rq_stats_desc[] = { · 957 unmodified lines
1126 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },1141 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
1127 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) },1142 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) },
1128 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_drop) },1143 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_drop) },
1129 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx) },1144 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_redirect) },
1130 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_cqe) },
1131 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_full) },
1132 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_packets) },1145 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_packets) },
1133 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },1146 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },
1134 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, removed_vlan_packets) },1147 { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, removed_vlan_packets) },
@@ -1168,6 +1181,20 @@ static const struct counter_desc sq_stats_desc[] = { · 33 unmodified lines
1168 { MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, cqe_err) },1181 { MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, cqe_err) },
1169};1182};
1170 1183
1184static const struct counter_desc rq_xdpsq_stats_desc[] = {
1185 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1186 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1187 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1188 { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
1189};
1190
1191static const struct counter_desc xdpsq_stats_desc[] = {
1192 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
1193 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
1194 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
1195 { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
1196};
1197
1171static const struct counter_desc ch_stats_desc[] = {1198static const struct counter_desc ch_stats_desc[] = {
1172 { MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, events) },1199 { MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, events) },
1173 { MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },1200 { MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },
@@ -1178,6 +1205,8 @@ static const struct counter_desc ch_stats_desc[] = { · 4 unmodified lines
1178 1205
1179#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc)1206#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc)
1180#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)1207#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)
1208#define NUM_XDPSQ_STATS ARRAY_SIZE(xdpsq_stats_desc)
1209#define NUM_RQ_XDPSQ_STATS ARRAY_SIZE(rq_xdpsq_stats_desc)
1181#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)1210#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)
1182 1211
1183static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)1212static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
@@ -1186,7 +1215,9 @@ static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv) · 2 unmodified lines
1186 1215
1187 return (NUM_RQ_STATS * max_nch) +1216 return (NUM_RQ_STATS * max_nch) +
1188 (NUM_CH_STATS * max_nch) +1217 (NUM_CH_STATS * max_nch) +
1189 (NUM_SQ_STATS * max_nch * priv->max_opened_tc);1218 (NUM_SQ_STATS * max_nch * priv->max_opened_tc) +
1219 (NUM_RQ_XDPSQ_STATS * max_nch) +
1220 (NUM_XDPSQ_STATS * max_nch);
1190}1221}
1191 1222
1192static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,1223static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
@@ -1200,9 +1231,14 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data, · 7 unmodified lines
1200 sprintf(data + (idx++) * ETH_GSTRING_LEN,1231 sprintf(data + (idx++) * ETH_GSTRING_LEN,
1201 ch_stats_desc[j].format, i);1232 ch_stats_desc[j].format, i);
1202 1233
1203 for (i = 0; i < max_nch; i++)1234 for (i = 0; i < max_nch; i++) {
1204 for (j = 0; j < NUM_RQ_STATS; j++)1235 for (j = 0; j < NUM_RQ_STATS; j++)
1205 sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i);1236 sprintf(data + (idx++) * ETH_GSTRING_LEN,
1237 rq_stats_desc[j].format, i);
1238 for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1239 sprintf(data + (idx++) * ETH_GSTRING_LEN,
1240 rq_xdpsq_stats_desc[j].format, i);
1241 }
1206 1242
1207 for (tc = 0; tc < priv->max_opened_tc; tc++)1243 for (tc = 0; tc < priv->max_opened_tc; tc++)
1208 for (i = 0; i < max_nch; i++)1244 for (i = 0; i < max_nch; i++)
@@ -1211,6 +1247,11 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data, · 2 unmodified lines
1211 sq_stats_desc[j].format,1247 sq_stats_desc[j].format,
1212 priv->channel_tc2txq[i][tc]);1248 priv->channel_tc2txq[i][tc]);
1213 1249
1250 for (i = 0; i < max_nch; i++)
1251 for (j = 0; j < NUM_XDPSQ_STATS; j++)
1252 sprintf(data + (idx++) * ETH_GSTRING_LEN,
1253 xdpsq_stats_desc[j].format, i);
1254
1214 return idx;1255 return idx;
1215}1256}
1216 1257
@@ -1226,11 +1267,16 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data, · 9 unmodified lines
1226 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].ch,1267 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].ch,
1227 ch_stats_desc, j);1268 ch_stats_desc, j);
1228 1269
1229 for (i = 0; i < max_nch; i++)1270 for (i = 0; i < max_nch; i++) {
1230 for (j = 0; j < NUM_RQ_STATS; j++)1271 for (j = 0; j < NUM_RQ_STATS; j++)
1231 data[idx++] =1272 data[idx++] =
1232 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,1273 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,
1233 rq_stats_desc, j);1274 rq_stats_desc, j);
1275 for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
1276 data[idx++] =
1277 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq_xdpsq,
1278 rq_xdpsq_stats_desc, j);
1279 }
1234 1280
1235 for (tc = 0; tc < priv->max_opened_tc; tc++)1281 for (tc = 0; tc < priv->max_opened_tc; tc++)
1236 for (i = 0; i < max_nch; i++)1282 for (i = 0; i < max_nch; i++)
@@ -1239,6 +1285,12 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data, · 2 unmodified lines
1239 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],1285 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],
1240 sq_stats_desc, j);1286 sq_stats_desc, j);
1241 1287
1288 for (i = 0; i < max_nch; i++)
1289 for (j = 0; j < NUM_XDPSQ_STATS; j++)
1290 data[idx++] =
1291 MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xdpsq,
1292 xdpsq_stats_desc, j);
1293
1242 return idx;1294 return idx;
1243}1295}
1244 1296
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h+19−5
@@ -44,6 +44,8 @@ · 43 unmodified lines
44#define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld)44#define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld)
45#define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld)45#define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld)
46#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)46#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)
47#define MLX5E_DECLARE_XDPSQ_STAT(type, fld) "tx%d_xdp_"#fld, offsetof(type, fld)
48#define MLX5E_DECLARE_RQ_XDPSQ_STAT(type, fld) "rx%d_xdp_tx_"#fld, offsetof(type, fld)
47#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)49#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)
48 50
49struct counter_desc {51struct counter_desc {
@@ -70,9 +72,11 @@ struct mlx5e_sw_stats { · 20 unmodified lines
70 u64 rx_csum_complete;72 u64 rx_csum_complete;
71 u64 rx_csum_unnecessary_inner;73 u64 rx_csum_unnecessary_inner;
72 u64 rx_xdp_drop;74 u64 rx_xdp_drop;
73 u64 rx_xdp_tx;75 u64 rx_xdp_redirect;
74 u64 rx_xdp_tx_cqe;76 u64 rx_xdp_tx_xmit;
75 u64 rx_xdp_tx_full;77 u64 rx_xdp_tx_full;
78 u64 rx_xdp_tx_err;
79 u64 rx_xdp_tx_cqe;
76 u64 tx_csum_none;80 u64 tx_csum_none;
77 u64 tx_csum_partial;81 u64 tx_csum_partial;
78 u64 tx_csum_partial_inner;82 u64 tx_csum_partial_inner;
@@ -84,6 +88,10 @@ struct mlx5e_sw_stats { · 5 unmodified lines
84 u64 tx_queue_wake;88 u64 tx_queue_wake;
85 u64 tx_udp_seg_rem;89 u64 tx_udp_seg_rem;
86 u64 tx_cqe_err;90 u64 tx_cqe_err;
91 u64 tx_xdp_xmit;
92 u64 tx_xdp_full;
93 u64 tx_xdp_err;
94 u64 tx_xdp_cqes;
87 u64 rx_wqe_err;95 u64 rx_wqe_err;
88 u64 rx_mpwqe_filler_cqes;96 u64 rx_mpwqe_filler_cqes;
89 u64 rx_mpwqe_filler_strides;97 u64 rx_mpwqe_filler_strides;
@@ -178,9 +186,7 @@ struct mlx5e_rq_stats { · 88 unmodified lines
178 u64 lro_bytes;186 u64 lro_bytes;
179 u64 removed_vlan_packets;187 u64 removed_vlan_packets;
180 u64 xdp_drop;188 u64 xdp_drop;
181 u64 xdp_tx;189 u64 xdp_redirect;
182 u64 xdp_tx_cqe;
183 u64 xdp_tx_full;
184 u64 wqe_err;190 u64 wqe_err;
185 u64 mpwqe_filler_cqes;191 u64 mpwqe_filler_cqes;
186 u64 mpwqe_filler_strides;192 u64 mpwqe_filler_strides;
@@ -225,6 +231,14 @@ struct mlx5e_sq_stats { · 38 unmodified lines
225 u64 cqe_err;231 u64 cqe_err;
226};232};
227 233
234struct mlx5e_xdpsq_stats {
235 u64 xmit;
236 u64 full;
237 u64 err;
238 /* dirtied @completion */
239 u64 cqes ____cacheline_aligned_in_smp;
240};
241
228struct mlx5e_ch_stats {242struct mlx5e_ch_stats {
229 u64 events;243 u64 events;
230 u64 poll;244 u64 poll;
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c+9−10
@@ -66,22 +66,21 @@ static inline void mlx5e_tx_dma_unmap(struct device *pdev, · 65 unmodified lines
66 }66 }
67}67}
68 68
69static inline struct mlx5e_sq_dma *mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
70{
71 return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
72}
73
69static inline void mlx5e_dma_push(struct mlx5e_txqsq *sq,74static inline void mlx5e_dma_push(struct mlx5e_txqsq *sq,
70 dma_addr_t addr,75 dma_addr_t addr,
71 u32 size,76 u32 size,
72 enum mlx5e_dma_map_type map_type)77 enum mlx5e_dma_map_type map_type)
73{78{
74 u32 i = sq->dma_fifo_pc & sq->dma_fifo_mask;79 struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
75 80
76 sq->db.dma_fifo[i].addr = addr;81 dma->addr = addr;
77 sq->db.dma_fifo[i].size = size;82 dma->size = size;
78 sq->db.dma_fifo[i].type = map_type;83 dma->type = map_type;
79 sq->dma_fifo_pc++;
80}
81
82static inline struct mlx5e_sq_dma *mlx5e_dma_get(struct mlx5e_txqsq *sq, u32 i)
83{
84 return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
85}84}
86 85
87static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)86static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c+4−0
@@ -32,6 +32,7 @@ · 31 unmodified lines
32 32
33#include <linux/irq.h>33#include <linux/irq.h>
34#include "en.h"34#include "en.h"
35#include "en/xdp.h"
35 36
36static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)37static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
37{38{
@@ -84,6 +85,8 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) · 46 unmodified lines
84 for (i = 0; i < c->num_tc; i++)85 for (i = 0; i < c->num_tc; i++)
85 busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);86 busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);
86 87
88 busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq);
89
87 if (c->xdp)90 if (c->xdp)
88 busy |= mlx5e_poll_xdpsq_cq(&c->rq.xdpsq.cq);91 busy |= mlx5e_poll_xdpsq_cq(&c->rq.xdpsq.cq);
89 92
@@ -116,6 +119,7 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) · 26 unmodified lines
116 119
117 mlx5e_cq_arm(&c->rq.cq);120 mlx5e_cq_arm(&c->rq.cq);
118 mlx5e_cq_arm(&c->icosq.cq);121 mlx5e_cq_arm(&c->icosq.cq);
122 mlx5e_cq_arm(&c->xdpsq.cq);
119 123
120 return work_done;124 return work_done;
121}125}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t