Merge tag 'mlx5e-updates-2018-07-26' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says: ==================== mlx5e-updates-2018-07-26 (XDP redirect) This series from Tariq adds the support for device-out XDP redirect. Start with a simple RX and XDP cleanups: - Replace call to MPWQE free with dealloc in interface down flow - Do not recycle RX pages in interface down flow - Gather all XDP pre-requisite checks in a single function - Restrict the combination of large MTU and XDP Since now XDP logic is going to be called from TX side as well, generic XDP TX logic is not RX only anymore, for that Tariq creates a new xdp.c file and moves XDP related code into it, and generalizes the code to support XDP TX for XDP redirect, such as the xdp tx sq structures and xdp counters. XDP redirect support: Add implementation for the ndo_xdp_xmit callback. Dedicate a new set of XDP-SQ instances to satisfy the XDP_REDIRECT requests. These instances are totally separated from the existing XDP-SQ objects that satisfy local XDP_TX actions. Performance tests: xdp_redirect_map from ConnectX-5 to ConnectX-5. CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz Packet-rate of 64B packets. Single queue: 7 Mpps. Multi queue: 55 Mpps. ====================

David S. Miller committed 8 years ago

Commit ecbcd682 parents f61b6db 8ee482310 files · +612 −313Browse files

Unified · Split

10 files

drivers/net/ethernet/mellanox/mlx5/core/Makefile +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en.h +33 −28
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +302 −0
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +63 −0
drivers/net/ethernet/mellanox/mlx5/core/en_main.c +88 −28
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +27 −227
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +66 −14
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +19 −5
drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +9 −10
drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +4 −0

drivers/net/ethernet/mellanox/mlx5/core/Makefile+1−1

@@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \ · 13 unmodified lines
14	14	fpga/ipsec.o fpga/tls.o
15	15
16	16	mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
17		− en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \
	17	+ en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o vxlan.o \
18	18	en_arfs.o en_fs_ethtool.o en_selftest.o en/port.o
19	19
20	20	mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o

drivers/net/ethernet/mellanox/mlx5/core/en.h+33−28

@@ -147,10 +147,6 @@ struct page_pool; · 146 unmodified lines
147	147	(DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
148	148	#define MLX5E_ICOSQ_MAX_WQEBBS MLX5E_UMR_WQEBBS
149	149
150		−#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
151		−#define MLX5E_XDP_TX_DS_COUNT \
152		− ((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
153		−
154	150	#define MLX5E_NUM_MAIN_GROUPS 9
155	151
156	152	#define MLX5E_MSG_LEVEL NETIF_MSG_LINK
@@ -348,6 +344,7 @@ enum { · 191 unmodified lines
348	344	MLX5E_SQ_STATE_IPSEC,
349	345	MLX5E_SQ_STATE_AM,
350	346	MLX5E_SQ_STATE_TLS,
	347	+ MLX5E_SQ_STATE_REDIRECT,
351	348	};
352	349
353	350	struct mlx5e_sq_wqe_info {
@@ -368,16 +365,14 @@ struct mlx5e_txqsq { · 14 unmodified lines
368	365
369	366	struct mlx5e_cq cq;
370	367
371		− /* write@xmit, read@completion */
372		− struct {
373		− struct mlx5e_sq_dma *dma_fifo;
374		− struct mlx5e_tx_wqe_info *wqe_info;
375		− } db;
376		−
377	368	/* read only */
378	369	struct mlx5_wq_cyc wq;
379	370	u32 dma_fifo_mask;
380	371	struct mlx5e_sq_stats *stats;
	372	+ struct {
	373	+ struct mlx5e_sq_dma *dma_fifo;
	374	+ struct mlx5e_tx_wqe_info *wqe_info;
	375	+ } db;
381	376	void __iomem *uar_map;
382	377	struct netdev_queue *txq;
383	378	u32 sqn;
@@ -399,30 +394,43 @@ struct mlx5e_txqsq { · 15 unmodified lines
399	394	} recover;
400	395	} ____cacheline_aligned_in_smp;
401	396
	397	+struct mlx5e_dma_info {
	398	+ struct page *page;
	399	+ dma_addr_t addr;
	400	+};
	401	+
	402	+struct mlx5e_xdp_info {
	403	+ struct xdp_frame *xdpf;
	404	+ dma_addr_t dma_addr;
	405	+ struct mlx5e_dma_info di;
	406	+};
	407	+
402	408	struct mlx5e_xdpsq {
403	409	/* data path */
404	410
405		− /* dirtied @rx completion */
	411	+ /* dirtied @completion */
406	412	u16 cc;
407		− u16 pc;
	413	+ bool redirect_flush;
408	414
409		− struct mlx5e_cq cq;
	415	+ /* dirtied @xmit */
	416	+ u16 pc ____cacheline_aligned_in_smp;
	417	+ bool doorbell;
410	418
411		− /* write@xmit, read@completion */
412		− struct {
413		− struct mlx5e_dma_info *di;
414		− bool doorbell;
415		− bool redirect_flush;
416		− } db;
	419	+ struct mlx5e_cq cq;
417	420
418	421	/* read only */
419	422	struct mlx5_wq_cyc wq;
	423	+ struct mlx5e_xdpsq_stats *stats;
	424	+ struct {
	425	+ struct mlx5e_xdp_info *xdpi;
	426	+ } db;
420	427	void __iomem *uar_map;
421	428	u32 sqn;
422	429	struct device *pdev;
423	430	__be32 mkey_be;
424	431	u8 min_inline_mode;
425	432	unsigned long state;
	433	+ unsigned int hw_mtu;
426	434
427	435	/* control path */
428	436	struct mlx5_wq_ctrl wq_ctrl;
@@ -459,11 +467,6 @@ mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) · 30 unmodified lines
459	467	return (mlx5_wq_cyc_ctr2ix(wq, cc - pc) >= n) \|\| (cc == pc);
460	468	}
461	469
462		−struct mlx5e_dma_info {
463		− struct page *page;
464		− dma_addr_t addr;
465		−};
466		−
467	470	struct mlx5e_wqe_frag_info {
468	471	struct mlx5e_dma_info *di;
469	472	u32 offset;
@@ -566,7 +569,6 @@ struct mlx5e_rq { · 96 unmodified lines
566	569
567	570	/* XDP */
568	571	struct bpf_prog *xdp_prog;
569		− unsigned int hw_mtu;
570	572	struct mlx5e_xdpsq xdpsq;
571	573	DECLARE_BITMAP(flags, 8);
572	574	struct page_pool *page_pool;
@@ -595,6 +597,9 @@ struct mlx5e_channel { · 22 unmodified lines
595	597	__be32 mkey_be;
596	598	u8 num_tc;
597	599
	600	+ /* XDP_REDIRECT */
	601	+ struct mlx5e_xdpsq xdpsq;
	602	+
598	603	/* data path - accessed per napi poll */
599	604	struct irq_desc *irq_desc;
600	605	struct mlx5e_ch_stats *stats;
@@ -617,6 +622,8 @@ struct mlx5e_channel_stats { · 16 unmodified lines
617	622	struct mlx5e_ch_stats ch;
618	623	struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
619	624	struct mlx5e_rq_stats rq;
	625	+ struct mlx5e_xdpsq_stats rq_xdpsq;
	626	+ struct mlx5e_xdpsq_stats xdpsq;
620	627	} ____cacheline_aligned_in_smp;
621	628
622	629	enum mlx5e_traffic_types {
@@ -876,14 +883,13 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event); · 253 unmodified lines
876	883	int mlx5e_napi_poll(struct napi_struct *napi, int budget);
877	884	bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
878	885	int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
879		−bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
880	886	void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq);
881		−void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
882	887
883	888	bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
884	889	bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
885	890	struct mlx5e_params *params);
886	891
	892	+void mlx5e_page_dma_unmap(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info);
887	893	void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
888	894	bool recycle);
889	895	void mlx5e_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe);
@@ -892,7 +898,6 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq); · 2 unmodified lines
892	898	bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
893	899	void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
894	900	void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
895		−void mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi);
896	901	struct sk_buff *
897	902	mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq rq, struct mlx5e_mpw_info wi,
898	903	u16 cqe_bcnt, u32 head_offset, u32 page_idx);

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c+302−0

@@ -0,0 +1,302 @@
	1	+/*
	2	+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
	3	+ *
	4	+ * This software is available to you under a choice of one of two
	5	+ * licenses. You may choose to be licensed under the terms of the GNU
	6	+ * General Public License (GPL) Version 2, available from the file
	7	+ * COPYING in the main directory of this source tree, or the
	8	+ * OpenIB.org BSD license below:
	9	+ *
	10	+ * Redistribution and use in source and binary forms, with or
	11	+ * without modification, are permitted provided that the following
	12	+ * conditions are met:
	13	+ *
	14	+ * - Redistributions of source code must retain the above
	15	+ * copyright notice, this list of conditions and the following
	16	+ * disclaimer.
	17	+ *
	18	+ * - Redistributions in binary form must reproduce the above
	19	+ * copyright notice, this list of conditions and the following
	20	+ * disclaimer in the documentation and/or other materials
	21	+ * provided with the distribution.
	22	+ *
	23	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	+ * SOFTWARE.
	31	+ */
	32	+
	33	+#include <linux/bpf_trace.h>
	34	+#include "en/xdp.h"
	35	+
	36	+static inline bool
	37	+mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq sq, struct mlx5e_dma_info di,
	38	+ struct xdp_buff *xdp)
	39	+{
	40	+ struct mlx5e_xdp_info xdpi;
	41	+
	42	+ xdpi.xdpf = convert_to_xdp_frame(xdp);
	43	+ if (unlikely(!xdpi.xdpf))
	44	+ return false;
	45	+ xdpi.dma_addr = di->addr + (xdpi.xdpf->data - (void *)xdpi.xdpf);
	46	+ dma_sync_single_for_device(sq->pdev, xdpi.dma_addr,
	47	+ xdpi.xdpf->len, PCI_DMA_TODEVICE);
	48	+ xdpi.di = *di;
	49	+
	50	+ return mlx5e_xmit_xdp_frame(sq, &xdpi);
	51	+}
	52	+
	53	+/* returns true if packet was consumed by xdp */
	54	+bool mlx5e_xdp_handle(struct mlx5e_rq rq, struct mlx5e_dma_info di,
	55	+ void va, u16 rx_headroom, u32 *len)
	56	+{
	57	+ struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
	58	+ struct xdp_buff xdp;
	59	+ u32 act;
	60	+ int err;
	61	+
	62	+ if (!prog)
	63	+ return false;
	64	+
	65	+ xdp.data = va + *rx_headroom;
	66	+ xdp_set_data_meta_invalid(&xdp);
	67	+ xdp.data_end = xdp.data + *len;
	68	+ xdp.data_hard_start = va;
	69	+ xdp.rxq = &rq->xdp_rxq;
	70	+
	71	+ act = bpf_prog_run_xdp(prog, &xdp);
	72	+ switch (act) {
	73	+ case XDP_PASS:
	74	+ *rx_headroom = xdp.data - xdp.data_hard_start;
	75	+ *len = xdp.data_end - xdp.data;
	76	+ return false;
	77	+ case XDP_TX:
	78	+ if (unlikely(!mlx5e_xmit_xdp_buff(&rq->xdpsq, di, &xdp)))
	79	+ goto xdp_abort;
	80	+ __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
	81	+ return true;
	82	+ case XDP_REDIRECT:
	83	+ /* When XDP enabled then page-refcnt==1 here */
	84	+ err = xdp_do_redirect(rq->netdev, &xdp, prog);
	85	+ if (unlikely(err))
	86	+ goto xdp_abort;
	87	+ __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
	88	+ rq->xdpsq.redirect_flush = true;
	89	+ mlx5e_page_dma_unmap(rq, di);
	90	+ rq->stats->xdp_redirect++;
	91	+ return true;
	92	+ default:
	93	+ bpf_warn_invalid_xdp_action(act);
	94	+ case XDP_ABORTED:
	95	+xdp_abort:
	96	+ trace_xdp_exception(rq->netdev, prog, act);
	97	+ case XDP_DROP:
	98	+ rq->stats->xdp_drop++;
	99	+ return true;
	100	+ }
	101	+}
	102	+
	103	+bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq sq, struct mlx5e_xdp_info xdpi)
	104	+{
	105	+ struct mlx5_wq_cyc *wq = &sq->wq;
	106	+ u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
	107	+ struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
	108	+
	109	+ struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
	110	+ struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
	111	+ struct mlx5_wqe_data_seg *dseg = wqe->data;
	112	+
	113	+ struct xdp_frame *xdpf = xdpi->xdpf;
	114	+ dma_addr_t dma_addr = xdpi->dma_addr;
	115	+ unsigned int dma_len = xdpf->len;
	116	+
	117	+ struct mlx5e_xdpsq_stats *stats = sq->stats;
	118	+
	119	+ prefetchw(wqe);
	120	+
	121	+ if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE \|\| sq->hw_mtu < dma_len)) {
	122	+ stats->err++;
	123	+ return false;
	124	+ }
	125	+
	126	+ if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
	127	+ if (sq->doorbell) {
	128	+ /* SQ is full, ring doorbell */
	129	+ mlx5e_xmit_xdp_doorbell(sq);
	130	+ sq->doorbell = false;
	131	+ }
	132	+ stats->full++;
	133	+ return false;
	134	+ }
	135	+
	136	+ cseg->fm_ce_se = 0;
	137	+
	138	+ /* copy the inline part if required */
	139	+ if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
	140	+ memcpy(eseg->inline_hdr.start, xdpf->data, MLX5E_XDP_MIN_INLINE);
	141	+ eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
	142	+ dma_len -= MLX5E_XDP_MIN_INLINE;
	143	+ dma_addr += MLX5E_XDP_MIN_INLINE;
	144	+ dseg++;
	145	+ }
	146	+
	147	+ /* write the dma part */
	148	+ dseg->addr = cpu_to_be64(dma_addr);
	149	+ dseg->byte_count = cpu_to_be32(dma_len);
	150	+
	151	+ cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_SEND);
	152	+
	153	+ /* move page to reference to sq responsibility,
	154	+ * and mark so it's not put back in page-cache.
	155	+ */
	156	+ sq->db.xdpi[pi] = *xdpi;
	157	+ sq->pc++;
	158	+
	159	+ sq->doorbell = true;
	160	+
	161	+ stats->xmit++;
	162	+ return true;
	163	+}
	164	+
	165	+bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
	166	+{
	167	+ struct mlx5e_xdpsq *sq;
	168	+ struct mlx5_cqe64 *cqe;
	169	+ struct mlx5e_rq *rq;
	170	+ bool is_redirect;
	171	+ u16 sqcc;
	172	+ int i;
	173	+
	174	+ sq = container_of(cq, struct mlx5e_xdpsq, cq);
	175	+
	176	+ if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
	177	+ return false;
	178	+
	179	+ cqe = mlx5_cqwq_get_cqe(&cq->wq);
	180	+ if (!cqe)
	181	+ return false;
	182	+
	183	+ is_redirect = test_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
	184	+ rq = container_of(sq, struct mlx5e_rq, xdpsq);
	185	+
	186	+ /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	187	+ * otherwise a cq overrun may occur
	188	+ */
	189	+ sqcc = sq->cc;
	190	+
	191	+ i = 0;
	192	+ do {
	193	+ u16 wqe_counter;
	194	+ bool last_wqe;
	195	+
	196	+ mlx5_cqwq_pop(&cq->wq);
	197	+
	198	+ wqe_counter = be16_to_cpu(cqe->wqe_counter);
	199	+
	200	+ do {
	201	+ u16 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
	202	+ struct mlx5e_xdp_info *xdpi = &sq->db.xdpi[ci];
	203	+
	204	+ last_wqe = (sqcc == wqe_counter);
	205	+ sqcc++;
	206	+
	207	+ if (is_redirect) {
	208	+ xdp_return_frame(xdpi->xdpf);
	209	+ dma_unmap_single(sq->pdev, xdpi->dma_addr,
	210	+ xdpi->xdpf->len, DMA_TO_DEVICE);
	211	+ } else {
	212	+ /* Recycle RX page */
	213	+ mlx5e_page_release(rq, &xdpi->di, true);
	214	+ }
	215	+ } while (!last_wqe);
	216	+ } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
	217	+
	218	+ sq->stats->cqes += i;
	219	+
	220	+ mlx5_cqwq_update_db_record(&cq->wq);
	221	+
	222	+ /* ensure cq space is freed before enabling more cqes */
	223	+ wmb();
	224	+
	225	+ sq->cc = sqcc;
	226	+ return (i == MLX5E_TX_CQ_POLL_BUDGET);
	227	+}
	228	+
	229	+void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
	230	+{
	231	+ struct mlx5e_rq *rq;
	232	+ bool is_redirect;
	233	+
	234	+ is_redirect = test_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
	235	+ rq = is_redirect ? NULL : container_of(sq, struct mlx5e_rq, xdpsq);
	236	+
	237	+ while (sq->cc != sq->pc) {
	238	+ u16 ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
	239	+ struct mlx5e_xdp_info *xdpi = &sq->db.xdpi[ci];
	240	+
	241	+ sq->cc++;
	242	+
	243	+ if (is_redirect) {
	244	+ xdp_return_frame(xdpi->xdpf);
	245	+ dma_unmap_single(sq->pdev, xdpi->dma_addr,
	246	+ xdpi->xdpf->len, DMA_TO_DEVICE);
	247	+ } else {
	248	+ /* Recycle RX page */
	249	+ mlx5e_page_release(rq, &xdpi->di, false);
	250	+ }
	251	+ }
	252	+}
	253	+
	254	+int mlx5e_xdp_xmit(struct net_device dev, int n, struct xdp_frame *frames,
	255	+ u32 flags)
	256	+{
	257	+ struct mlx5e_priv *priv = netdev_priv(dev);
	258	+ struct mlx5e_xdpsq *sq;
	259	+ int drops = 0;
	260	+ int sq_num;
	261	+ int i;
	262	+
	263	+ if (unlikely(!test_bit(MLX5E_STATE_OPENED, &priv->state)))
	264	+ return -ENETDOWN;
	265	+
	266	+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
	267	+ return -EINVAL;
	268	+
	269	+ sq_num = smp_processor_id();
	270	+
	271	+ if (unlikely(sq_num >= priv->channels.num))
	272	+ return -ENXIO;
	273	+
	274	+ sq = &priv->channels.c[sq_num]->xdpsq;
	275	+
	276	+ if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
	277	+ return -ENETDOWN;
	278	+
	279	+ for (i = 0; i < n; i++) {
	280	+ struct xdp_frame *xdpf = frames[i];
	281	+ struct mlx5e_xdp_info xdpi;
	282	+
	283	+ xdpi.dma_addr = dma_map_single(sq->pdev, xdpf->data, xdpf->len,
	284	+ DMA_TO_DEVICE);
	285	+ if (unlikely(dma_mapping_error(sq->pdev, xdpi.dma_addr))) {
	286	+ drops++;
	287	+ continue;
	288	+ }
	289	+
	290	+ xdpi.xdpf = xdpf;
	291	+
	292	+ if (unlikely(!mlx5e_xmit_xdp_frame(sq, &xdpi))) {
	293	+ xdp_return_frame_rx_napi(xdpf);
	294	+ drops++;
	295	+ }
	296	+ }
	297	+
	298	+ if (flags & XDP_XMIT_FLUSH)
	299	+ mlx5e_xmit_xdp_doorbell(sq);
	300	+
	301	+ return n - drops;
	302	+}

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h+63−0

@@ -0,0 +1,63 @@
	1	+/*
	2	+ * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
	3	+ *
	4	+ * This software is available to you under a choice of one of two
	5	+ * licenses. You may choose to be licensed under the terms of the GNU
	6	+ * General Public License (GPL) Version 2, available from the file
	7	+ * COPYING in the main directory of this source tree, or the
	8	+ * OpenIB.org BSD license below:
	9	+ *
	10	+ * Redistribution and use in source and binary forms, with or
	11	+ * without modification, are permitted provided that the following
	12	+ * conditions are met:
	13	+ *
	14	+ * - Redistributions of source code must retain the above
	15	+ * copyright notice, this list of conditions and the following
	16	+ * disclaimer.
	17	+ *
	18	+ * - Redistributions in binary form must reproduce the above
	19	+ * copyright notice, this list of conditions and the following
	20	+ * disclaimer in the documentation and/or other materials
	21	+ * provided with the distribution.
	22	+ *
	23	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	+ * SOFTWARE.
	31	+ */
	32	+#ifndef __MLX5_EN_XDP_H__
	33	+#define __MLX5_EN_XDP_H__
	34	+
	35	+#include "en.h"
	36	+
	37	+#define MLX5E_XDP_MAX_MTU ((int)(PAGE_SIZE - \
	38	+ MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM)))
	39	+#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
	40	+#define MLX5E_XDP_TX_DS_COUNT \
	41	+ ((sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
	42	+
	43	+bool mlx5e_xdp_handle(struct mlx5e_rq rq, struct mlx5e_dma_info di,
	44	+ void va, u16 rx_headroom, u32 *len);
	45	+bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
	46	+void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
	47	+
	48	+bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq sq, struct mlx5e_xdp_info xdpi);
	49	+int mlx5e_xdp_xmit(struct net_device dev, int n, struct xdp_frame *frames,
	50	+ u32 flags);
	51	+
	52	+static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
	53	+{
	54	+ struct mlx5_wq_cyc *wq = &sq->wq;
	55	+ struct mlx5e_tx_wqe *wqe;
	56	+ u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
	57	+
	58	+ wqe = mlx5_wq_cyc_get_wqe(wq, pi);
	59	+
	60	+ mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
	61	+}
	62	+
	63	+#endif

drivers/net/ethernet/mellanox/mlx5/core/en_main.c+88−28

@@ -47,6 +47,7 @@ · 46 unmodified lines
47	47	#include "accel/tls.h"
48	48	#include "vxlan.h"
49	49	#include "en/port.h"
	50	+#include "en/xdp.h"
50	51
51	52	struct mlx5e_rq_param {
52	53	u32 rqc[MLX5_ST_SZ_DW(rqc)];
@@ -96,14 +97,19 @@ bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) · 43 unmodified lines
96	97
97	98	static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
98	99	{
99		− if (!params->xdp_prog) {
100		− u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
101		− u16 rq_headroom = MLX5_RX_HEADROOM + NET_IP_ALIGN;
	100	+ u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
	101	+ u16 linear_rq_headroom = params->xdp_prog ?
	102	+ XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
	103	+ u32 frag_sz;
102	104
103		− return MLX5_SKB_FRAG_SZ(rq_headroom + hw_mtu);
104		− }
	105	+ linear_rq_headroom += NET_IP_ALIGN;
105	106
106		− return PAGE_SIZE;
	107	+ frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
	108	+
	109	+ if (params->xdp_prog && frag_sz < PAGE_SIZE)
	110	+ frag_sz = PAGE_SIZE;
	111	+
	112	+ return frag_sz;
107	113	}
108	114
109	115	static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
@@ -485,7 +491,6 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, · 375 unmodified lines
485	491	rq->channel = c;
486	492	rq->ix = c->ix;
487	493	rq->mdev = mdev;
488		− rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
489	494	rq->stats = &c->priv->channel_stats[c->ix].rq;
490	495
491	496	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
@@ -877,7 +882,7 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) · 385 unmodified lines
877	882
878	883	/* UMR WQE (if in progress) is always at wq->head */
879	884	if (rq->mpwqe.umr_in_progress)
880		− mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
	885	+ rq->dealloc_wqe(rq, wq->head);
881	886
882	887	while (!mlx5_wq_ll_is_empty(wq)) {
883	888	struct mlx5e_rx_wqe_ll *wqe;
@@ -963,16 +968,16 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq) · 79 unmodified lines
963	968
964	969	static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
965	970	{
966		− kvfree(sq->db.di);
	971	+ kvfree(sq->db.xdpi);
967	972	}
968	973
969	974	static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
970	975	{
971	976	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
972	977
973		− sq->db.di = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.di)),
974		− GFP_KERNEL, numa);
975		− if (!sq->db.di) {
	978	+ sq->db.xdpi = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.xdpi)),
	979	+ GFP_KERNEL, numa);
	980	+ if (!sq->db.xdpi) {
976	981	mlx5e_free_xdpsq_db(sq);
977	982	return -ENOMEM;
978	983	}
@@ -983,7 +988,8 @@ static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) · 4 unmodified lines
983	988	static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
984	989	struct mlx5e_params *params,
985	990	struct mlx5e_sq_param *param,
986		− struct mlx5e_xdpsq *sq)
	991	+ struct mlx5e_xdpsq *sq,
	992	+ bool is_redirect)
987	993	{
988	994	void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
989	995	struct mlx5_core_dev *mdev = c->mdev;
@@ -995,6 +1001,10 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, · 5 unmodified lines
995	1001	sq->channel = c;
996	1002	sq->uar_map = mdev->mlx5e_res.bfreg.map;
997	1003	sq->min_inline_mode = params->tx_min_inline_mode;
	1004	+ sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
	1005	+ sq->stats = is_redirect ?
	1006	+ &c->priv->channel_stats[c->ix].xdpsq :
	1007	+ &c->priv->channel_stats[c->ix].rq_xdpsq;
998	1008
999	1009	param->wq.db_numa_node = cpu_to_node(c->cpu);
1000	1010	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@@ -1524,7 +1534,8 @@ static void mlx5e_close_icosq(struct mlx5e_icosq *sq) · 523 unmodified lines
1524	1534	static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
1525	1535	struct mlx5e_params *params,
1526	1536	struct mlx5e_sq_param *param,
1527		− struct mlx5e_xdpsq *sq)
	1537	+ struct mlx5e_xdpsq *sq,
	1538	+ bool is_redirect)
1528	1539	{
1529	1540	unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1530	1541	struct mlx5e_create_sq_param csp = {};
@@ -1532,7 +1543,7 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 1 unmodified line
1532	1543	int err;
1533	1544	int i;
1534	1545
1535		− err = mlx5e_alloc_xdpsq(c, params, param, sq);
	1546	+ err = mlx5e_alloc_xdpsq(c, params, param, sq, is_redirect);
1536	1547	if (err)
1537	1548	return err;
1538	1549
@@ -1541,6 +1552,8 @@ static int mlx5e_open_xdpsq(struct mlx5e_channel *c, · 2 unmodified lines
1541	1552	csp.cqn = sq->cq.mcq.cqn;
1542	1553	csp.wq_ctrl = &sq->wq_ctrl;
1543	1554	csp.min_inline_mode = sq->min_inline_mode;
	1555	+ if (is_redirect)
	1556	+ set_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
1544	1557	set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1545	1558	err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
1546	1559	if (err)
@@ -1923,10 +1936,14 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 376 unmodified lines
1923	1936	if (err)
1924	1937	goto err_close_icosq_cq;
1925	1938
1926		− err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
	1939	+ err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq);
1927	1940	if (err)
1928	1941	goto err_close_tx_cqs;
1929	1942
	1943	+ err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
	1944	+ if (err)
	1945	+ goto err_close_xdp_tx_cqs;
	1946	+
1930	1947	/* XDP SQ CQ params are same as normal TXQ sq CQ params */
1931	1948	err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1932	1949	&cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
@@ -1943,7 +1960,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 10 unmodified lines
1943	1960	if (err)
1944	1961	goto err_close_icosq;
1945	1962
1946		− err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq) : 0;
	1963	+ err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0;
1947	1964	if (err)
1948	1965	goto err_close_sqs;
1949	1966
@@ -1951,9 +1968,17 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, · 1 unmodified line
1951	1968	if (err)
1952	1969	goto err_close_xdp_sq;
1953	1970
	1971	+ err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->xdpsq, true);
	1972	+ if (err)
	1973	+ goto err_close_rq;
	1974	+
1954	1975	*cp = c;
1955	1976
1956	1977	return 0;
	1978	+
	1979	+err_close_rq:
	1980	+ mlx5e_close_rq(&c->rq);
	1981	+
1957	1982	err_close_xdp_sq:
1958	1983	if (c->xdp)
1959	1984	mlx5e_close_xdpsq(&c->rq.xdpsq);
@@ -1972,6 +1997,9 @@ err_disable_napi: · 12 unmodified lines
1972	1997	err_close_rx_cq:
1973	1998	mlx5e_close_cq(&c->rq.cq);
1974	1999
	2000	+err_close_xdp_tx_cqs:
	2001	+ mlx5e_close_cq(&c->xdpsq.cq);
	2002	+
1975	2003	err_close_tx_cqs:
1976	2004	mlx5e_close_tx_cqs(c);
1977	2005
@@ -2006,6 +2034,7 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) · 28 unmodified lines
2006	2034
2007	2035	static void mlx5e_close_channel(struct mlx5e_channel *c)
2008	2036	{
	2037	+ mlx5e_close_xdpsq(&c->xdpsq);
2009	2038	mlx5e_close_rq(&c->rq);
2010	2039	if (c->xdp)
2011	2040	mlx5e_close_xdpsq(&c->rq.xdpsq);
@@ -2015,6 +2044,7 @@ static void mlx5e_close_channel(struct mlx5e_channel *c) · 3 unmodified lines
2015	2044	if (c->xdp)
2016	2045	mlx5e_close_cq(&c->rq.xdpsq.cq);
2017	2046	mlx5e_close_cq(&c->rq.cq);
	2047	+ mlx5e_close_cq(&c->xdpsq.cq);
2018	2048	mlx5e_close_tx_cqs(c);
2019	2049	mlx5e_close_cq(&c->icosq.cq);
2020	2050	netif_napi_del(&c->napi);
@@ -3707,6 +3737,14 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, · 1686 unmodified lines
3707	3737	new_channels.params = *params;
3708	3738	new_channels.params.sw_mtu = new_mtu;
3709	3739
	3740	+ if (params->xdp_prog &&
	3741	+ !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
	3742	+ netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
	3743	+ new_mtu, MLX5E_XDP_MAX_MTU);
	3744	+ err = -EINVAL;
	3745	+ goto out;
	3746	+ }
	3747	+
3710	3748	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3711	3749	u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);
3712	3750	u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);
@@ -4094,26 +4132,47 @@ static void mlx5e_tx_timeout(struct net_device *dev) · 381 unmodified lines
4094	4132	queue_work(priv->wq, &priv->tx_timeout_work);
4095	4133	}
4096	4134
	4135	+static int mlx5e_xdp_allowed(struct mlx5e_priv priv, struct bpf_prog prog)
	4136	+{
	4137	+ struct net_device *netdev = priv->netdev;
	4138	+ struct mlx5e_channels new_channels = {};
	4139	+
	4140	+ if (priv->channels.params.lro_en) {
	4141	+ netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
	4142	+ return -EINVAL;
	4143	+ }
	4144	+
	4145	+ if (MLX5_IPSEC_DEV(priv->mdev)) {
	4146	+ netdev_warn(netdev, "can't set XDP with IPSec offload\n");
	4147	+ return -EINVAL;
	4148	+ }
	4149	+
	4150	+ new_channels.params = priv->channels.params;
	4151	+ new_channels.params.xdp_prog = prog;
	4152	+
	4153	+ if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
	4154	+ netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
	4155	+ new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
	4156	+ return -EINVAL;
	4157	+ }
	4158	+
	4159	+ return 0;
	4160	+}
	4161	+
4097	4162	static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog)
4098	4163	{
4099	4164	struct mlx5e_priv *priv = netdev_priv(netdev);
4100	4165	struct bpf_prog *old_prog;
4101		− int err = 0;
4102	4166	bool reset, was_opened;
	4167	+ int err;
4103	4168	int i;
4104	4169
4105	4170	mutex_lock(&priv->state_lock);
4106	4171
4107		− if ((netdev->features & NETIF_F_LRO) && prog) {
4108		− netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
4109		− err = -EINVAL;
4110		− goto unlock;
4111		− }
4112		−
4113		− if ((netdev->features & NETIF_F_HW_ESP) && prog) {
4114		− netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4115		− err = -EINVAL;
4116		− goto unlock;
	4172	+ if (prog) {
	4173	+ err = mlx5e_xdp_allowed(priv, prog);
	4174	+ if (err)
	4175	+ goto unlock;
4117	4176	}
4118	4177
4119	4178	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
@@ -4242,6 +4301,7 @@ static const struct net_device_ops mlx5e_netdev_ops = { · 122 unmodified lines
4242	4301	#endif
4243	4302	.ndo_tx_timeout = mlx5e_tx_timeout,
4244	4303	.ndo_bpf = mlx5e_xdp,
	4304	+ .ndo_xdp_xmit = mlx5e_xdp_xmit,
4245	4305	#ifdef CONFIG_NET_POLL_CONTROLLER
4246	4306	.ndo_poll_controller = mlx5e_netpoll,
4247	4307	#endif

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c+27−227

@@ -34,7 +34,6 @@ · 33 unmodified lines
34	34	#include <linux/ip.h>
35	35	#include <linux/ipv6.h>
36	36	#include <linux/tcp.h>
37		−#include <linux/bpf_trace.h>
38	37	#include <net/busy_poll.h>
39	38	#include <net/ip6_checksum.h>
40	39	#include <net/page_pool.h>
@@ -46,6 +45,7 @@ · 5 unmodified lines
46	45	#include "en_accel/ipsec_rxtx.h"
47	46	#include "en_accel/tls_rxtx.h"
48	47	#include "lib/clock.h"
	48	+#include "en/xdp.h"
49	49
50	50	static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
51	51	{
@@ -239,8 +239,7 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq, · 187 unmodified lines
239	239	return 0;
240	240	}
241	241
242		−static void mlx5e_page_dma_unmap(struct mlx5e_rq *rq,
243		− struct mlx5e_dma_info *dma_info)
	242	+void mlx5e_page_dma_unmap(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info)
244	243	{
245	244	dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, rq->buff.map_dir);
246	245	}
@@ -277,10 +276,11 @@ static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, · 30 unmodified lines
277	276	}
278	277
279	278	static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
280		− struct mlx5e_wqe_frag_info *frag)
	279	+ struct mlx5e_wqe_frag_info *frag,
	280	+ bool recycle)
281	281	{
282	282	if (frag->last_in_page)
283		− mlx5e_page_release(rq, frag->di, true);
	283	+ mlx5e_page_release(rq, frag->di, recycle);
284	284	}
285	285
286	286	static inline struct mlx5e_wqe_frag_info get_frag(struct mlx5e_rq rq, u16 ix)
@@ -308,25 +308,26 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe_cyc wqe, · 21 unmodified lines
308	308
309	309	free_frags:
310	310	while (--i >= 0)
311		− mlx5e_put_rx_frag(rq, --frag);
	311	+ mlx5e_put_rx_frag(rq, --frag, true);
312	312
313	313	return err;
314	314	}
315	315
316	316	static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
317		− struct mlx5e_wqe_frag_info *wi)
	317	+ struct mlx5e_wqe_frag_info *wi,
	318	+ bool recycle)
318	319	{
319	320	int i;
320	321
321	322	for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
322		− mlx5e_put_rx_frag(rq, wi);
	323	+ mlx5e_put_rx_frag(rq, wi, recycle);
323	324	}
324	325
325	326	void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
326	327	{
327	328	struct mlx5e_wqe_frag_info *wi = get_frag(rq, ix);
328	329
329		− mlx5e_free_rx_wqe(rq, wi);
	330	+ mlx5e_free_rx_wqe(rq, wi, false);
330	331	}
331	332
332	333	static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
@@ -396,7 +397,8 @@ mlx5e_copy_skb_header_mpwqe(struct device *pdev, · 63 unmodified lines
396	397	}
397	398	}
398	399
399		−void mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi)
	400	+static void
	401	+mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi, bool recycle)
400	402	{
401	403	const bool no_xdp_xmit =
402	404	bitmap_empty(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
@@ -405,7 +407,7 @@ void mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi) · 2 unmodified lines
405	407
406	408	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
407	409	if (no_xdp_xmit \|\| !test_bit(i, wi->xdp_xmit_bitmap))
408		− mlx5e_page_release(rq, &dma_info[i], true);
	410	+ mlx5e_page_release(rq, &dma_info[i], recycle);
409	411	}
410	412
411	413	static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
@@ -505,8 +507,8 @@ err_unmap: · 93 unmodified lines
505	507	void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
506	508	{
507	509	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
508		−
509		− mlx5e_free_rx_mpwqe(rq, wi);
	510	+ /* Don't recycle, this function is called on rq/netdev close */
	511	+ mlx5e_free_rx_mpwqe(rq, wi, false);
510	512	}
511	513
512	514	bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
@@ -847,135 +849,6 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq, · 334 unmodified lines
847	849	mlx5e_build_rx_skb(cqe, cqe_bcnt, rq, skb);
848	850	}
849	851
850		−static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
851		−{
852		− struct mlx5_wq_cyc *wq = &sq->wq;
853		− struct mlx5e_tx_wqe *wqe;
854		− u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc - 1); /* last pi */
855		−
856		− wqe = mlx5_wq_cyc_get_wqe(wq, pi);
857		−
858		− mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &wqe->ctrl);
859		−}
860		−
861		−static inline bool mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
862		− struct mlx5e_dma_info *di,
863		− const struct xdp_buff *xdp)
864		−{
865		− struct mlx5e_xdpsq *sq = &rq->xdpsq;
866		− struct mlx5_wq_cyc *wq = &sq->wq;
867		− u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
868		− struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
869		−
870		− struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
871		− struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
872		− struct mlx5_wqe_data_seg *dseg;
873		−
874		− ptrdiff_t data_offset = xdp->data - xdp->data_hard_start;
875		− dma_addr_t dma_addr = di->addr + data_offset;
876		− unsigned int dma_len = xdp->data_end - xdp->data;
877		−
878		− struct mlx5e_rq_stats *stats = rq->stats;
879		−
880		− prefetchw(wqe);
881		−
882		− if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE \|\| rq->hw_mtu < dma_len)) {
883		− stats->xdp_drop++;
884		− return false;
885		− }
886		−
887		− if (unlikely(!mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1))) {
888		− if (sq->db.doorbell) {
889		− /* SQ is full, ring doorbell */
890		− mlx5e_xmit_xdp_doorbell(sq);
891		− sq->db.doorbell = false;
892		− }
893		− stats->xdp_tx_full++;
894		− return false;
895		− }
896		−
897		− dma_sync_single_for_device(sq->pdev, dma_addr, dma_len, PCI_DMA_TODEVICE);
898		−
899		− cseg->fm_ce_se = 0;
900		−
901		− dseg = (struct mlx5_wqe_data_seg *)eseg + 1;
902		−
903		− /* copy the inline part if required */
904		− if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
905		− memcpy(eseg->inline_hdr.start, xdp->data, MLX5E_XDP_MIN_INLINE);
906		− eseg->inline_hdr.sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);
907		− dma_len -= MLX5E_XDP_MIN_INLINE;
908		− dma_addr += MLX5E_XDP_MIN_INLINE;
909		− dseg++;
910		− }
911		−
912		− /* write the dma part */
913		− dseg->addr = cpu_to_be64(dma_addr);
914		− dseg->byte_count = cpu_to_be32(dma_len);
915		−
916		− cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_SEND);
917		−
918		− /* move page to reference to sq responsibility,
919		− * and mark so it's not put back in page-cache.
920		− */
921		− __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
922		− sq->db.di[pi] = *di;
923		− sq->pc++;
924		−
925		− sq->db.doorbell = true;
926		−
927		− stats->xdp_tx++;
928		− return true;
929		−}
930		−
931		−/* returns true if packet was consumed by xdp */
932		−static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
933		− struct mlx5e_dma_info *di,
934		− void va, u16 rx_headroom, u32 *len)
935		−{
936		− struct bpf_prog *prog = READ_ONCE(rq->xdp_prog);
937		− struct xdp_buff xdp;
938		− u32 act;
939		− int err;
940		−
941		− if (!prog)
942		− return false;
943		−
944		− xdp.data = va + *rx_headroom;
945		− xdp_set_data_meta_invalid(&xdp);
946		− xdp.data_end = xdp.data + *len;
947		− xdp.data_hard_start = va;
948		− xdp.rxq = &rq->xdp_rxq;
949		−
950		− act = bpf_prog_run_xdp(prog, &xdp);
951		− switch (act) {
952		− case XDP_PASS:
953		− *rx_headroom = xdp.data - xdp.data_hard_start;
954		− *len = xdp.data_end - xdp.data;
955		− return false;
956		− case XDP_TX:
957		− if (unlikely(!mlx5e_xmit_xdp_frame(rq, di, &xdp)))
958		− trace_xdp_exception(rq->netdev, prog, act);
959		− return true;
960		− case XDP_REDIRECT:
961		− /* When XDP enabled then page-refcnt==1 here */
962		− err = xdp_do_redirect(rq->netdev, &xdp, prog);
963		− if (!err) {
964		− __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
965		− rq->xdpsq.db.redirect_flush = true;
966		− mlx5e_page_dma_unmap(rq, di);
967		− }
968		− return true;
969		− default:
970		− bpf_warn_invalid_xdp_action(act);
971		− case XDP_ABORTED:
972		− trace_xdp_exception(rq->netdev, prog, act);
973		− case XDP_DROP:
974		− rq->stats->xdp_drop++;
975		− return true;
976		− }
977		−}
978		−
979	852	static inline
980	853	struct sk_buff mlx5e_build_linear_skb(struct mlx5e_rq rq, void *va,
981	854	u32 frag_size, u16 headroom,
@@ -1113,7 +986,7 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe) · 131 unmodified lines
1113	986	napi_gro_receive(rq->cq.napi, skb);
1114	987
1115	988	free_wqe:
1116		− mlx5e_free_rx_wqe(rq, wi);
	989	+ mlx5e_free_rx_wqe(rq, wi, true);
1117	990	wq_cyc_pop:
1118	991	mlx5_wq_cyc_pop(wq);
1119	992	}
@@ -1155,7 +1028,7 @@ void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq rq, struct mlx5_cqe64 cqe) · 35 unmodified lines
1155	1028	napi_gro_receive(rq->cq.napi, skb);
1156	1029
1157	1030	free_wqe:
1158		− mlx5e_free_rx_wqe(rq, wi);
	1031	+ mlx5e_free_rx_wqe(rq, wi, true);
1159	1032	wq_cyc_pop:
1160	1033	mlx5_wq_cyc_pop(wq);
1161	1034	}
@@ -1226,6 +1099,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq rq, struct mlx5e_mpw_info wi, · 64 unmodified lines
1226	1099
1227	1100	dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
1228	1101	frag_size, DMA_FROM_DEVICE);
	1102	+ prefetchw(va); /* xdp_frame data area */
1229	1103	prefetch(data);
1230	1104
1231	1105	rcu_read_lock();
@@ -1292,7 +1166,7 @@ mpwrq_cqe_out: · 60 unmodified lines
1292	1166
1293	1167	wq = &rq->mpwqe.wq;
1294	1168	wqe = mlx5_wq_ll_get_wqe(wq, wqe_id);
1295		− mlx5e_free_rx_mpwqe(rq, wi);
	1169	+ mlx5e_free_rx_mpwqe(rq, wi, true);
1296	1170	mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index);
1297	1171	}
1298	1172
@@ -1328,14 +1202,14 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) · 29 unmodified lines
1328	1202	rq->handle_rx_cqe(rq, cqe);
1329	1203	} while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
1330	1204
1331		− if (xdpsq->db.doorbell) {
	1205	+ if (xdpsq->doorbell) {
1332	1206	mlx5e_xmit_xdp_doorbell(xdpsq);
1333		− xdpsq->db.doorbell = false;
	1207	+ xdpsq->doorbell = false;
1334	1208	}
1335	1209
1336		− if (xdpsq->db.redirect_flush) {
	1210	+ if (xdpsq->redirect_flush) {
1337	1211	xdp_do_flush_map();
1338		− xdpsq->db.redirect_flush = false;
	1212	+ xdpsq->redirect_flush = false;
1339	1213	}
1340	1214
1341	1215	mlx5_cqwq_update_db_record(&cq->wq);
@@ -1346,80 +1220,6 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) · 4 unmodified lines
1346	1220	return work_done;
1347	1221	}
1348	1222
1349		−bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
1350		−{
1351		− struct mlx5e_xdpsq *sq;
1352		− struct mlx5_cqe64 *cqe;
1353		− struct mlx5e_rq *rq;
1354		− u16 sqcc;
1355		− int i;
1356		−
1357		− sq = container_of(cq, struct mlx5e_xdpsq, cq);
1358		−
1359		− if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
1360		− return false;
1361		−
1362		− cqe = mlx5_cqwq_get_cqe(&cq->wq);
1363		− if (!cqe)
1364		− return false;
1365		−
1366		− rq = container_of(sq, struct mlx5e_rq, xdpsq);
1367		−
1368		− /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
1369		− * otherwise a cq overrun may occur
1370		− */
1371		− sqcc = sq->cc;
1372		−
1373		− i = 0;
1374		− do {
1375		− u16 wqe_counter;
1376		− bool last_wqe;
1377		−
1378		− mlx5_cqwq_pop(&cq->wq);
1379		−
1380		− wqe_counter = be16_to_cpu(cqe->wqe_counter);
1381		−
1382		− do {
1383		− struct mlx5e_dma_info *di;
1384		− u16 ci;
1385		−
1386		− last_wqe = (sqcc == wqe_counter);
1387		−
1388		− ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sqcc);
1389		− di = &sq->db.di[ci];
1390		−
1391		− sqcc++;
1392		− /* Recycle RX page */
1393		− mlx5e_page_release(rq, di, true);
1394		− } while (!last_wqe);
1395		− } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
1396		−
1397		− rq->stats->xdp_tx_cqe += i;
1398		−
1399		− mlx5_cqwq_update_db_record(&cq->wq);
1400		−
1401		− /* ensure cq space is freed before enabling more cqes */
1402		− wmb();
1403		−
1404		− sq->cc = sqcc;
1405		− return (i == MLX5E_TX_CQ_POLL_BUDGET);
1406		−}
1407		−
1408		−void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
1409		−{
1410		− struct mlx5e_rq *rq = container_of(sq, struct mlx5e_rq, xdpsq);
1411		− struct mlx5e_dma_info *di;
1412		− u16 ci;
1413		−
1414		− while (sq->cc != sq->pc) {
1415		− ci = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->cc);
1416		− di = &sq->db.di[ci];
1417		− sq->cc++;
1418		−
1419		− mlx5e_page_release(rq, di, false);
1420		− }
1421		−}
1422		−
1423	1223	#ifdef CONFIG_MLX5_CORE_IPOIB
1424	1224
1425	1225	#define MLX5_IB_GRH_DGID_OFFSET 24
@@ -1521,7 +1321,7 @@ void mlx5i_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe) · 95 unmodified lines
1521	1321	napi_gro_receive(rq->cq.napi, skb);
1522	1322
1523	1323	wq_free_wqe:
1524		− mlx5e_free_rx_wqe(rq, wi);
	1324	+ mlx5e_free_rx_wqe(rq, wi, true);
1525	1325	mlx5_wq_cyc_pop(wq);
1526	1326	}
1527	1327
@@ -1544,19 +1344,19 @@ void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe) · 16 unmodified lines
1544	1344	skb = rq->wqe.skb_from_cqe(rq, cqe, wi, cqe_bcnt);
1545	1345	if (unlikely(!skb)) {
1546	1346	/* a DROP, save the page-reuse checks */
1547		− mlx5e_free_rx_wqe(rq, wi);
	1347	+ mlx5e_free_rx_wqe(rq, wi, true);
1548	1348	goto wq_cyc_pop;
1549	1349	}
1550	1350	skb = mlx5e_ipsec_handle_rx_skb(rq->netdev, skb, &cqe_bcnt);
1551	1351	if (unlikely(!skb)) {
1552		− mlx5e_free_rx_wqe(rq, wi);
	1352	+ mlx5e_free_rx_wqe(rq, wi, true);
1553	1353	goto wq_cyc_pop;
1554	1354	}
1555	1355
1556	1356	mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
1557	1357	napi_gro_receive(rq->cq.napi, skb);
1558	1358
1559		− mlx5e_free_rx_wqe(rq, wi);
	1359	+ mlx5e_free_rx_wqe(rq, wi, true);
1560	1360	wq_cyc_pop:
1561	1361	mlx5_wq_cyc_pop(wq);
1562	1362	}

drivers/net/ethernet/mellanox/mlx5/core/en_stats.c+66−14

@@ -59,9 +59,11 @@ static const struct counter_desc sw_stats_desc[] = { · 58 unmodified lines
59	59	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) },
60	60	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) },
61	61	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
62		− { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx) },
63		− { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
	62	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_redirect) },
	63	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_xmit) },
64	64	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
	65	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_err) },
	66	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_cqe) },
65	67	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_none) },
66	68	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial) },
67	69	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial_inner) },
@@ -73,6 +75,10 @@ static const struct counter_desc sw_stats_desc[] = { · 5 unmodified lines
73	75	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_wake) },
74	76	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_udp_seg_rem) },
75	77	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_cqe_err) },
	78	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_xmit) },
	79	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_full) },
	80	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_err) },
	81	+ { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xdp_cqes) },
76	82	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_wqe_err) },
77	83	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_cqes) },
78	84	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler_strides) },
@@ -128,6 +134,8 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 49 unmodified lines
128	134	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++) {
129	135	struct mlx5e_channel_stats *channel_stats =
130	136	&priv->channel_stats[i];
	137	+ struct mlx5e_xdpsq_stats *xdpsq_red_stats = &channel_stats->xdpsq;
	138	+ struct mlx5e_xdpsq_stats *xdpsq_stats = &channel_stats->rq_xdpsq;
131	139	struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
132	140	struct mlx5e_ch_stats *ch_stats = &channel_stats->ch;
133	141	int j;
@@ -141,10 +149,12 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 7 unmodified lines
141	149	s->rx_csum_complete += rq_stats->csum_complete;
142	150	s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
143	151	s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
144		− s->rx_xdp_drop += rq_stats->xdp_drop;
145		− s->rx_xdp_tx += rq_stats->xdp_tx;
146		− s->rx_xdp_tx_cqe += rq_stats->xdp_tx_cqe;
147		− s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
	152	+ s->rx_xdp_drop += rq_stats->xdp_drop;
	153	+ s->rx_xdp_redirect += rq_stats->xdp_redirect;
	154	+ s->rx_xdp_tx_xmit += xdpsq_stats->xmit;
	155	+ s->rx_xdp_tx_full += xdpsq_stats->full;
	156	+ s->rx_xdp_tx_err += xdpsq_stats->err;
	157	+ s->rx_xdp_tx_cqe += xdpsq_stats->cqes;
148	158	s->rx_wqe_err += rq_stats->wqe_err;
149	159	s->rx_mpwqe_filler_cqes += rq_stats->mpwqe_filler_cqes;
150	160	s->rx_mpwqe_filler_strides += rq_stats->mpwqe_filler_strides;
@@ -162,7 +172,12 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv) · 11 unmodified lines
162	172	s->ch_poll += ch_stats->poll;
163	173	s->ch_arm += ch_stats->arm;
164	174	s->ch_aff_change += ch_stats->aff_change;
165		− s->ch_eq_rearm += ch_stats->eq_rearm;
	175	+ s->ch_eq_rearm += ch_stats->eq_rearm;
	176	+ /* xdp redirect */
	177	+ s->tx_xdp_xmit += xdpsq_red_stats->xmit;
	178	+ s->tx_xdp_full += xdpsq_red_stats->full;
	179	+ s->tx_xdp_err += xdpsq_red_stats->err;
	180	+ s->tx_xdp_cqes += xdpsq_red_stats->cqes;
166	181
167	182	for (j = 0; j < priv->max_opened_tc; j++) {
168	183	struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
@@ -1126,9 +1141,7 @@ static const struct counter_desc rq_stats_desc[] = { · 957 unmodified lines
1126	1141	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
1127	1142	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) },
1128	1143	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_drop) },
1129		− { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx) },
1130		− { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_cqe) },
1131		− { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_full) },
	1144	+ { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_redirect) },
1132	1145	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_packets) },
1133	1146	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },
1134	1147	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, removed_vlan_packets) },
@@ -1168,6 +1181,20 @@ static const struct counter_desc sq_stats_desc[] = { · 33 unmodified lines
1168	1181	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, cqe_err) },
1169	1182	};
1170	1183
	1184	+static const struct counter_desc rq_xdpsq_stats_desc[] = {
	1185	+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
	1186	+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
	1187	+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
	1188	+ { MLX5E_DECLARE_RQ_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
	1189	+};
	1190	+
	1191	+static const struct counter_desc xdpsq_stats_desc[] = {
	1192	+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, xmit) },
	1193	+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, full) },
	1194	+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, err) },
	1195	+ { MLX5E_DECLARE_XDPSQ_STAT(struct mlx5e_xdpsq_stats, cqes) },
	1196	+};
	1197	+
1171	1198	static const struct counter_desc ch_stats_desc[] = {
1172	1199	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, events) },
1173	1200	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, poll) },
@@ -1178,6 +1205,8 @@ static const struct counter_desc ch_stats_desc[] = { · 4 unmodified lines
1178	1205
1179	1206	#define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc)
1180	1207	#define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc)
	1208	+#define NUM_XDPSQ_STATS ARRAY_SIZE(xdpsq_stats_desc)
	1209	+#define NUM_RQ_XDPSQ_STATS ARRAY_SIZE(rq_xdpsq_stats_desc)
1181	1210	#define NUM_CH_STATS ARRAY_SIZE(ch_stats_desc)
1182	1211
1183	1212	static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
@@ -1186,7 +1215,9 @@ static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv) · 2 unmodified lines
1186	1215
1187	1216	return (NUM_RQ_STATS * max_nch) +
1188	1217	(NUM_CH_STATS * max_nch) +
1189		− (NUM_SQ_STATS * max_nch * priv->max_opened_tc);
	1218	+ (NUM_SQ_STATS * max_nch * priv->max_opened_tc) +
	1219	+ (NUM_RQ_XDPSQ_STATS * max_nch) +
	1220	+ (NUM_XDPSQ_STATS * max_nch);
1190	1221	}
1191	1222
1192	1223	static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data,
@@ -1200,9 +1231,14 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data, · 7 unmodified lines
1200	1231	sprintf(data + (idx++) * ETH_GSTRING_LEN,
1201	1232	ch_stats_desc[j].format, i);
1202	1233
1203		− for (i = 0; i < max_nch; i++)
	1234	+ for (i = 0; i < max_nch; i++) {
1204	1235	for (j = 0; j < NUM_RQ_STATS; j++)
1205		− sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i);
	1236	+ sprintf(data + (idx++) * ETH_GSTRING_LEN,
	1237	+ rq_stats_desc[j].format, i);
	1238	+ for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
	1239	+ sprintf(data + (idx++) * ETH_GSTRING_LEN,
	1240	+ rq_xdpsq_stats_desc[j].format, i);
	1241	+ }
1206	1242
1207	1243	for (tc = 0; tc < priv->max_opened_tc; tc++)
1208	1244	for (i = 0; i < max_nch; i++)
@@ -1211,6 +1247,11 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv priv, u8 data, · 2 unmodified lines
1211	1247	sq_stats_desc[j].format,
1212	1248	priv->channel_tc2txq[i][tc]);
1213	1249
	1250	+ for (i = 0; i < max_nch; i++)
	1251	+ for (j = 0; j < NUM_XDPSQ_STATS; j++)
	1252	+ sprintf(data + (idx++) * ETH_GSTRING_LEN,
	1253	+ xdpsq_stats_desc[j].format, i);
	1254	+
1214	1255	return idx;
1215	1256	}
1216	1257
@@ -1226,11 +1267,16 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv priv, u64 data, · 9 unmodified lines
1226	1267	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].ch,
1227	1268	ch_stats_desc, j);
1228	1269
1229		− for (i = 0; i < max_nch; i++)
	1270	+ for (i = 0; i < max_nch; i++) {
1230	1271	for (j = 0; j < NUM_RQ_STATS; j++)
1231	1272	data[idx++] =
1232	1273	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq,
1233	1274	rq_stats_desc, j);
	1275	+ for (j = 0; j < NUM_RQ_XDPSQ_STATS; j++)
	1276	+ data[idx++] =
	1277	+ MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].rq_xdpsq,
	1278	+ rq_xdpsq_stats_desc, j);
	1279	+ }
1234	1280
1235	1281	for (tc = 0; tc < priv->max_opened_tc; tc++)
1236	1282	for (i = 0; i < max_nch; i++)
@@ -1239,6 +1285,12 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv priv, u64 data, · 2 unmodified lines
1239	1285	MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].sq[tc],
1240	1286	sq_stats_desc, j);
1241	1287
	1288	+ for (i = 0; i < max_nch; i++)
	1289	+ for (j = 0; j < NUM_XDPSQ_STATS; j++)
	1290	+ data[idx++] =
	1291	+ MLX5E_READ_CTR64_CPU(&priv->channel_stats[i].xdpsq,
	1292	+ xdpsq_stats_desc, j);
	1293	+
1242	1294	return idx;
1243	1295	}
1244	1296

drivers/net/ethernet/mellanox/mlx5/core/en_stats.h+19−5

@@ -44,6 +44,8 @@ · 43 unmodified lines
44	44	#define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld)
45	45	#define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld)
46	46	#define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)
	47	+#define MLX5E_DECLARE_XDPSQ_STAT(type, fld) "tx%d_xdp_"#fld, offsetof(type, fld)
	48	+#define MLX5E_DECLARE_RQ_XDPSQ_STAT(type, fld) "rx%d_xdp_tx_"#fld, offsetof(type, fld)
47	49	#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)
48	50
49	51	struct counter_desc {
@@ -70,9 +72,11 @@ struct mlx5e_sw_stats { · 20 unmodified lines
70	72	u64 rx_csum_complete;
71	73	u64 rx_csum_unnecessary_inner;
72	74	u64 rx_xdp_drop;
73		− u64 rx_xdp_tx;
74		− u64 rx_xdp_tx_cqe;
	75	+ u64 rx_xdp_redirect;
	76	+ u64 rx_xdp_tx_xmit;
75	77	u64 rx_xdp_tx_full;
	78	+ u64 rx_xdp_tx_err;
	79	+ u64 rx_xdp_tx_cqe;
76	80	u64 tx_csum_none;
77	81	u64 tx_csum_partial;
78	82	u64 tx_csum_partial_inner;
@@ -84,6 +88,10 @@ struct mlx5e_sw_stats { · 5 unmodified lines
84	88	u64 tx_queue_wake;
85	89	u64 tx_udp_seg_rem;
86	90	u64 tx_cqe_err;
	91	+ u64 tx_xdp_xmit;
	92	+ u64 tx_xdp_full;
	93	+ u64 tx_xdp_err;
	94	+ u64 tx_xdp_cqes;
87	95	u64 rx_wqe_err;
88	96	u64 rx_mpwqe_filler_cqes;
89	97	u64 rx_mpwqe_filler_strides;
@@ -178,9 +186,7 @@ struct mlx5e_rq_stats { · 88 unmodified lines
178	186	u64 lro_bytes;
179	187	u64 removed_vlan_packets;
180	188	u64 xdp_drop;
181		− u64 xdp_tx;
182		− u64 xdp_tx_cqe;
183		− u64 xdp_tx_full;
	189	+ u64 xdp_redirect;
184	190	u64 wqe_err;
185	191	u64 mpwqe_filler_cqes;
186	192	u64 mpwqe_filler_strides;
@@ -225,6 +231,14 @@ struct mlx5e_sq_stats { · 38 unmodified lines
225	231	u64 cqe_err;
226	232	};
227	233
	234	+struct mlx5e_xdpsq_stats {
	235	+ u64 xmit;
	236	+ u64 full;
	237	+ u64 err;
	238	+ /* dirtied @completion */
	239	+ u64 cqes ____cacheline_aligned_in_smp;
	240	+};
	241	+
228	242	struct mlx5e_ch_stats {
229	243	u64 events;
230	244	u64 poll;

drivers/net/ethernet/mellanox/mlx5/core/en_tx.c+9−10

@@ -66,22 +66,21 @@ static inline void mlx5e_tx_dma_unmap(struct device *pdev, · 65 unmodified lines
66	66	}
67	67	}
68	68
	69	+static inline struct mlx5e_sq_dma mlx5e_dma_get(struct mlx5e_txqsq sq, u32 i)
	70	+{
	71	+ return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
	72	+}
	73	+
69	74	static inline void mlx5e_dma_push(struct mlx5e_txqsq *sq,
70	75	dma_addr_t addr,
71	76	u32 size,
72	77	enum mlx5e_dma_map_type map_type)
73	78	{
74		− u32 i = sq->dma_fifo_pc & sq->dma_fifo_mask;
	79	+ struct mlx5e_sq_dma *dma = mlx5e_dma_get(sq, sq->dma_fifo_pc++);
75	80
76		− sq->db.dma_fifo[i].addr = addr;
77		− sq->db.dma_fifo[i].size = size;
78		− sq->db.dma_fifo[i].type = map_type;
79		− sq->dma_fifo_pc++;
80		−}
81		−
82		−static inline struct mlx5e_sq_dma mlx5e_dma_get(struct mlx5e_txqsq sq, u32 i)
83		−{
84		− return &sq->db.dma_fifo[i & sq->dma_fifo_mask];
	81	+ dma->addr = addr;
	82	+ dma->size = size;
	83	+ dma->type = map_type;
85	84	}
86	85
87	86	static void mlx5e_dma_unmap_wqe_err(struct mlx5e_txqsq *sq, u8 num_dma)

drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c+4−0

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32
33	33	#include <linux/irq.h>
34	34	#include "en.h"
	35	+#include "en/xdp.h"
35	36
36	37	static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
37	38	{
@@ -84,6 +85,8 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) · 46 unmodified lines
84	85	for (i = 0; i < c->num_tc; i++)
85	86	busy \|= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);
86	87
	88	+ busy \|= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq);
	89	+
87	90	if (c->xdp)
88	91	busy \|= mlx5e_poll_xdpsq_cq(&c->rq.xdpsq.cq);
89	92
@@ -116,6 +119,7 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) · 26 unmodified lines
116	119
117	120	mlx5e_cq_arm(&c->rq.cq);
118	121	mlx5e_cq_arm(&c->icosq.cq);
	122	+ mlx5e_cq_arm(&c->xdpsq.cq);
119	123
120	124	return work_done;
121	125	}

Merge tag 'mlx5e-updates-2018-07-26' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code