// SPDX-License-Identifier: GPL-2.0/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ #include <linux/ip.h>#include <linux/ipv6.h>#include <linux/if_vlan.h>#include <net/ip6_checksum.h>#include <net/netdev_queues.h>#include <net/page_pool/helpers.h> #include "ionic.h"#include "ionic_lif.h"#include "ionic_txrx.h" static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,				      void *data, size_t len); static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,				    const skb_frag_t *frag,				    size_t offset, size_t len); static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,				     struct ionic_tx_desc_info *desc_info); static void ionic_tx_clean(struct ionic_queue *q,			   struct ionic_tx_desc_info *desc_info,			   struct ionic_txq_comp *comp,			   bool in_napi); static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell){	/* Ensure TX descriptor writes reach memory before NIC reads them.	 * Prevents device from fetching stale descriptors.	 */	dma_wmb();	ionic_q_post(q, ring_dbell);} static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell){	ionic_q_post(q, ring_dbell);} bool ionic_txq_poke_doorbell(struct ionic_queue *q){	struct netdev_queue *netdev_txq;	unsigned long now, then, dif;	struct net_device *netdev; 	netdev = q->lif->netdev;	netdev_txq = netdev_get_tx_queue(netdev, q->index); 	HARD_TX_LOCK(netdev, netdev_txq, smp_processor_id()); 	if (q->tail_idx == q->head_idx) {		HARD_TX_UNLOCK(netdev, netdev_txq);		return false;	} 	now = READ_ONCE(jiffies);	then = q->dbell_jiffies;	dif = now - then; 	if (dif > q->dbell_deadline) {		ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,				 q->dbval | q->head_idx); 		q->dbell_jiffies = now;	} 	HARD_TX_UNLOCK(netdev, netdev_txq); 	return true;} bool ionic_rxq_poke_doorbell(struct ionic_queue *q){	unsigned long now, then, dif; 	/* no lock, called from rx napi or txrx napi, nothing else can fill */ 	if (q->tail_idx == q->head_idx)		return false; 	now = READ_ONCE(jiffies);	then = q->dbell_jiffies;	dif = now - then; 	if (dif > q->dbell_deadline) {		ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,				 q->dbval | q->head_idx); 		q->dbell_jiffies = now; 		dif = 2 * q->dbell_deadline;		if (dif > IONIC_RX_MAX_DOORBELL_DEADLINE)			dif = IONIC_RX_MAX_DOORBELL_DEADLINE; 		q->dbell_deadline = dif;	} 	return true;} static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q){	if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1)))		return q->txq_sgl_v1[q->head_idx].elems;	else		return q->txq_sgl[q->head_idx].elems;} static inline struct netdev_queue *q_to_ndq(struct net_device *netdev,					    struct ionic_queue *q){	return netdev_get_tx_queue(netdev, q->index);} static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info){	return page_address(buf_info->page) + buf_info->page_offset;} static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info){	return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset;} static void __ionic_rx_put_buf(struct ionic_queue *q,			       struct ionic_buf_info *buf_info,			       bool recycle_direct){	if (!buf_info->page)		return; 	page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct);	buf_info->page = NULL;	buf_info->len = 0;	buf_info->page_offset = 0;}  static void ionic_rx_put_buf(struct ionic_queue *q,			     struct ionic_buf_info *buf_info){	__ionic_rx_put_buf(q, buf_info, false);} static void ionic_rx_put_buf_direct(struct ionic_queue *q,				    struct ionic_buf_info *buf_info){	__ionic_rx_put_buf(q, buf_info, true);} static void ionic_rx_add_skb_frag(struct ionic_queue *q,				  struct sk_buff *skb,				  struct ionic_buf_info *buf_info,				  u32 headroom, u32 len,				  bool synced){	if (!synced)		page_pool_dma_sync_for_cpu(q->page_pool,					   buf_info->page,					   buf_info->page_offset + headroom,					   len); 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,			buf_info->page, buf_info->page_offset + headroom,			len, buf_info->len); 	/* napi_gro_frags() will release/recycle the	 * page_pool buffers from the frags list	 */	buf_info->page = NULL;	buf_info->len = 0;	buf_info->page_offset = 0;} static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,					  struct ionic_rx_desc_info *desc_info,					  unsigned int headroom,					  unsigned int len,					  unsigned int num_sg_elems,					  bool synced){	struct ionic_buf_info *buf_info;	struct sk_buff *skb;	unsigned int i;	u16 frag_len; 	buf_info = &desc_info->bufs[0];	prefetchw(buf_info->page); 	skb = napi_get_frags(&q_to_qcq(q)->napi);	if (unlikely(!skb)) {		net_warn_ratelimited("%s: SKB alloc failed on %s!\n",				     dev_name(q->dev), q->name);		q_to_rx_stats(q)->alloc_err++;		return NULL;	}	skb_mark_for_recycle(skb); 	if (headroom)		frag_len = min_t(u16, len,				 IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);	else		frag_len = min_t(u16, len, IONIC_PAGE_SIZE); 	if (unlikely(!buf_info->page))		goto err_bad_buf_page;	ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced);	len -= frag_len;	buf_info++; 	for (i = 0; i < num_sg_elems; i++, buf_info++) {		if (unlikely(!buf_info->page))			goto err_bad_buf_page;		frag_len = min_t(u16, len, buf_info->len);		ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced);		len -= frag_len;	} 	return skb; err_bad_buf_page:	dev_kfree_skb(skb);	return NULL;} static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,					  struct ionic_queue *q,					  struct ionic_rx_desc_info *desc_info,					  unsigned int headroom,					  unsigned int len,					  unsigned int num_sg_elems,					  bool synced){	struct ionic_buf_info *buf_info;	struct device *dev = q->dev;	struct sk_buff *skb;	int i; 	buf_info = &desc_info->bufs[0]; 	skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);	if (unlikely(!skb)) {		net_warn_ratelimited("%s: SKB alloc failed on %s!\n",				     dev_name(dev), q->name);		q_to_rx_stats(q)->alloc_err++;		return NULL;	}	skb_mark_for_recycle(skb); 	if (!synced)		page_pool_dma_sync_for_cpu(q->page_pool,					   buf_info->page,					   buf_info->page_offset + headroom,					   len); 	skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); 	skb_put(skb, len);	skb->protocol = eth_type_trans(skb, netdev); 	/* recycle the Rx buffer now that we're done with it */	ionic_rx_put_buf_direct(q, buf_info);	buf_info++;	for (i = 0; i < num_sg_elems; i++, buf_info++)		ionic_rx_put_buf_direct(q, buf_info); 	return skb;} static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,				    struct ionic_tx_desc_info *desc_info,				    bool in_napi){	struct xdp_frame_bulk bq; 	if (!desc_info->nbufs)		return; 	xdp_frame_bulk_init(&bq);	rcu_read_lock(); /* need for xdp_return_frame_bulk */ 	if (desc_info->act == XDP_TX) {		if (likely(in_napi))			xdp_return_frame_rx_napi(desc_info->xdpf);		else			xdp_return_frame(desc_info->xdpf);	} else if (desc_info->act == XDP_REDIRECT) {		ionic_tx_desc_unmap_bufs(q, desc_info);		xdp_return_frame_bulk(desc_info->xdpf, &bq);	} 	xdp_flush_frame_bulk(&bq);	rcu_read_unlock(); 	desc_info->nbufs = 0;	desc_info->xdpf = NULL;	desc_info->act = 0;} static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,				enum xdp_action act, struct page *page, int off,				bool ring_doorbell){	struct ionic_tx_desc_info *desc_info;	struct ionic_buf_info *buf_info;	struct ionic_tx_stats *stats;	struct ionic_txq_desc *desc;	size_t len = frame->len;	dma_addr_t dma_addr;	u64 cmd; 	desc_info = &q->tx_info[q->head_idx];	desc = &q->txq[q->head_idx];	buf_info = desc_info->bufs;	stats = q_to_tx_stats(q); 	if (act == XDP_TX) {		dma_addr = page_pool_get_dma_addr(page) +			   off + XDP_PACKET_HEADROOM;		dma_sync_single_for_device(q->dev, dma_addr,					   len, DMA_TO_DEVICE);	} else /* XDP_REDIRECT */ {		dma_addr = ionic_tx_map_single(q, frame->data, len);		if (dma_addr == DMA_MAPPING_ERROR)			return -EIO;	} 	buf_info->dma_addr = dma_addr;	buf_info->len = len;	buf_info->page = page;	buf_info->page_offset = off; 	desc_info->nbufs = 1;	desc_info->xdpf = frame;	desc_info->act = act; 	if (xdp_frame_has_frags(frame)) {		struct ionic_txq_sg_elem *elem;		struct skb_shared_info *sinfo;		struct ionic_buf_info *bi;		skb_frag_t *frag;		int i; 		bi = &buf_info[1];		sinfo = xdp_get_shared_info_from_frame(frame);		frag = sinfo->frags;		elem = ionic_tx_sg_elems(q);		for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {			if (act == XDP_TX) {				struct page *pg = skb_frag_page(frag); 				dma_addr = page_pool_get_dma_addr(pg) +					   skb_frag_off(frag);				dma_sync_single_for_device(q->dev, dma_addr,							   skb_frag_size(frag),							   DMA_TO_DEVICE);			} else {				dma_addr = ionic_tx_map_frag(q, frag, 0,							     skb_frag_size(frag));				if (dma_addr == DMA_MAPPING_ERROR) {					ionic_tx_desc_unmap_bufs(q, desc_info);					return -EIO;				}			}			bi->dma_addr = dma_addr;			bi->len = skb_frag_size(frag);			bi->page = skb_frag_page(frag); 			elem->addr = cpu_to_le64(bi->dma_addr);			elem->len = cpu_to_le16(bi->len);			elem++; 			desc_info->nbufs++;		}	} 	cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,				  0, (desc_info->nbufs - 1), buf_info->dma_addr);	desc->cmd = cpu_to_le64(cmd);	desc->len = cpu_to_le16(len);	desc->csum_start = 0;	desc->csum_offset = 0; 	stats->xdp_frames++;	stats->pkts++;	stats->bytes += len; 	ionic_txq_post(q, ring_doorbell); 	return 0;} int ionic_xdp_xmit(struct net_device *netdev, int n,		   struct xdp_frame **xdp_frames, u32 flags){	struct ionic_lif *lif = netdev_priv(netdev);	struct ionic_queue *txq;	struct netdev_queue *nq;	int nxmit;	int space;	int cpu;	int qi; 	if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))		return -ENETDOWN; 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))		return -EINVAL; 	/* AdminQ is assumed on cpu 0, while we attempt to affinitize the	 * TxRx queue pairs 0..n-1 on cpus 1..n.  We try to keep with that	 * affinitization here, but of course irqbalance and friends might	 * have juggled things anyway, so we have to check for the 0 case.	 */	cpu = smp_processor_id();	qi = cpu ? (cpu - 1) % lif->nxqs : cpu; 	txq = &lif->txqcqs[qi]->q;	nq = netdev_get_tx_queue(netdev, txq->index);	__netif_tx_lock(nq, cpu);	txq_trans_cond_update(nq); 	if (netif_tx_queue_stopped(nq) ||	    !netif_txq_maybe_stop(q_to_ndq(netdev, txq),				  ionic_q_space_avail(txq),				  1, 1)) {		__netif_tx_unlock(nq);		return -EIO;	} 	space = min_t(int, n, ionic_q_space_avail(txq));	for (nxmit = 0; nxmit < space ; nxmit++) {		if (ionic_xdp_post_frame(txq, xdp_frames[nxmit],					 XDP_REDIRECT,					 virt_to_page(xdp_frames[nxmit]->data),					 0, false)) {			nxmit--;			break;		}	} 	if (flags & XDP_XMIT_FLUSH)		ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,				 txq->dbval | txq->head_idx); 	netif_txq_maybe_stop(q_to_ndq(netdev, txq),			     ionic_q_space_avail(txq),			     4, 4);	__netif_tx_unlock(nq); 	return nxmit;} static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q,				     struct ionic_buf_info *buf_info,				     int nbufs){	int i; 	for (i = 0; i < nbufs; i++) {		buf_info->page = NULL;		buf_info++;	}} static bool ionic_run_xdp(struct ionic_rx_stats *stats,			  struct net_device *netdev,			  struct bpf_prog *xdp_prog,			  struct ionic_queue *rxq,			  struct ionic_buf_info *buf_info,			  int len){	u32 xdp_action = XDP_ABORTED;	struct xdp_buff xdp_buf;	struct ionic_queue *txq;	struct netdev_queue *nq;	struct xdp_frame *xdpf;	int remain_len;	int nbufs = 1;	int frag_len;	int err = 0; 	xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);	frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);	xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),			 XDP_PACKET_HEADROOM, frag_len, false);	page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page,				   buf_info->page_offset + XDP_PACKET_HEADROOM,				   frag_len);	prefetchw(&xdp_buf.data_hard_start); 	/*  We limit MTU size to one buffer if !xdp_has_frags, so	 *  if the recv len is bigger than one buffer	 *     then we know we have frag info to gather	 */	remain_len = len - frag_len;

Lines 1–500 of 1864. Load more