i40e: xsk: Move tmp desc array from driver to pool

Move desc_array from the driver to the pool. The reason behind this is that we can then reuse this array as a temporary storage for descriptors in all zero-copy drivers that use the batched interface. This will make it easier to add batching to more drivers. i40e is the only driver that has a batched Tx zero-copy implementation, so no need to touch any other driver.

Magnus Karlsson committed 4 years ago

Commit d1bc5321 parent 3dd411e8 files · +24 −30Browse files

Unified · Split

8 files

drivers/net/ethernet/intel/i40e/i40e_txrx.c +0 −11
drivers/net/ethernet/intel/i40e/i40e_txrx.h +0 −1
drivers/net/ethernet/intel/i40e/i40e_xsk.c +2 −2
include/net/xdp_sock_drv.h +2 −3
include/net/xsk_buff_pool.h +1 −0
net/xdp/xsk.c +6 −7
net/xdp/xsk_buff_pool.c +7 −0
net/xdp/xsk_queue.h +6 −6

drivers/net/ethernet/intel/i40e/i40e_txrx.c+0−11

 	i40e_clean_tx_ring(tx_ring);
 	kfree(tx_ring->tx_bi);
 	tx_ring->tx_bi = NULL;
-	kfree(tx_ring->xsk_descs);
-	tx_ring->xsk_descs = NULL;
 	if (tx_ring->desc) {
 		dma_free_coherent(tx_ring->dev, tx_ring->size,
 	if (!tx_ring->tx_bi)
 		goto err;
-	if (ring_is_xdp(tx_ring)) {
-		tx_ring->xsk_descs = kcalloc(I40E_MAX_NUM_DESCRIPTORS, sizeof(*tx_ring->xsk_descs),
-					     GFP_KERNEL);
-		if (!tx_ring->xsk_descs)
-			goto err;
-	}
 	u64_stats_init(&tx_ring->syncp);
 	/* round up to nearest 4K */
 	return 0;
 err:
-	kfree(tx_ring->xsk_descs);
-	tx_ring->xsk_descs = NULL;
 	kfree(tx_ring->tx_bi);
 	tx_ring->tx_bi = NULL;
 	return -ENOMEM;

drivers/net/ethernet/intel/i40e/i40e_txrx.h+0−1

@@ -390,7 +390,6 @@ struct i40e_ring { · 389 unmodified lines
390	u16 rx_offset;	390	u16 rx_offset;
391	struct xdp_rxq_info xdp_rxq;	391	struct xdp_rxq_info xdp_rxq;
392	struct xsk_buff_pool *xsk_pool;	392	struct xsk_buff_pool *xsk_pool;
393	struct xdp_desc xsk_descs; / For storing descriptors in the AF_XDP ZC path */
394	} ____cacheline_internodealigned_in_smp;	393	} ____cacheline_internodealigned_in_smp;
395		394
396	static inline bool ring_uses_build_skb(struct i40e_ring *ring)	395	static inline bool ring_uses_build_skb(struct i40e_ring *ring)

drivers/net/ethernet/intel/i40e/i40e_xsk.c+2−2

@@ -467,11 +467,11 @@ static void i40e_set_rs_bit(struct i40e_ring *xdp_ring) · 466 unmodified lines
467	**/	467	**/
468	static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)	468	static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
469	{	469	{
470	struct xdp_desc *descs = xdp_ring->xsk_descs;	470	struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
471	u32 nb_pkts, nb_processed = 0;	471	u32 nb_pkts, nb_processed = 0;
472	unsigned int total_bytes = 0;	472	unsigned int total_bytes = 0;
473		473
474	nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, descs, budget);	474	nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
475	if (!nb_pkts)	475	if (!nb_pkts)
476	return true;	476	return true;
477		477

include/net/xdp_sock_drv.h+2−3

 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
 bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
-u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, u32 max);
+u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
 void xsk_tx_release(struct xsk_buff_pool *pool);
 struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
 					    u16 queue_id);
 	return false;
 }
-static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc,
+static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
-						 u32 max)
 {
 	return 0;
 }

include/net/xsk_buff_pool.h+1−0

@@ -60,6 +60,7 @@ struct xsk_buff_pool { · 59 unmodified lines
60	*/	60	*/
61	dma_addr_t *dma_pages;	61	dma_addr_t *dma_pages;
62	struct xdp_buff_xsk *heads;	62	struct xdp_buff_xsk *heads;
		63	struct xdp_desc *tx_descs;
63	u64 chunk_mask;	64	u64 chunk_mask;
64	u64 addrs_cnt;	65	u64 addrs_cnt;
65	u32 free_list_cnt;	66	u32 free_list_cnt;

net/xdp/xsk.c+6−7

 }
 EXPORT_SYMBOL(xsk_tx_peek_desc);
-static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_desc *descs,
+static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
-					u32 max_entries)
 {
+	struct xdp_desc *descs = pool->tx_descs;
 	u32 nb_pkts = 0;
 	while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
 	return nb_pkts;
 }
-u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *descs,
+u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
-				   u32 max_entries)
 {
 	struct xdp_sock *xs;
 	u32 nb_pkts;
 	if (!list_is_singular(&pool->xsk_tx_list)) {
 		/* Fallback to the non-batched version */
 		rcu_read_unlock();
-		return xsk_tx_peek_release_fallback(pool, descs, max_entries);
+		return xsk_tx_peek_release_fallback(pool, max_entries);
 	}
 	xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
 		goto out;
 	}
-	nb_pkts = xskq_cons_peek_desc_batch(xs->tx, descs, pool, max_entries);
+	nb_pkts = xskq_cons_peek_desc_batch(xs->tx, pool, max_entries);
 	if (!nb_pkts) {
 		xs->tx->queue_empty_descs++;
 		goto out;
 	 * packets. This avoids having to implement any buffering in
 	 * the Tx path.
 	 */
-	nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, descs, nb_pkts);
+	nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
 	if (!nb_pkts)
 		goto out;

net/xdp/xsk_buff_pool.c+7−0

 	if (!pool)
 		return;
+	kvfree(pool->tx_descs);
 	kvfree(pool->heads);
 	kvfree(pool);
 }
 	if (!pool->heads)
 		goto out;
+	if (xs->tx) {
+		pool->tx_descs = kcalloc(xs->tx->nentries, sizeof(*pool->tx_descs), GFP_KERNEL);
+		if (!pool->tx_descs)
+			goto out;
+	}
 	pool->chunk_mask = ~((u64)umem->chunk_size - 1);
 	pool->addrs_cnt = umem->size;
 	pool->heads_cnt = umem->chunks;

net/xdp/xsk_queue.h+6−6

 	return false;
 }
-static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q,
+static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
-					    struct xdp_desc *descs,
+					    u32 max)
-					    struct xsk_buff_pool *pool, u32 max)
 {
 	u32 cached_cons = q->cached_cons, nb_entries = 0;
+	struct xdp_desc *descs = pool->tx_descs;
 	while (cached_cons != q->cached_prod && nb_entries < max) {
 		struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
 	return xskq_cons_read_desc(q, desc, pool);
 }
-static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xdp_desc *descs,
+static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
-					    struct xsk_buff_pool *pool, u32 max)
+					    u32 max)
 {
 	u32 entries = xskq_cons_nb_entries(q, max);
-	return xskq_cons_read_desc_batch(q, descs, pool, entries);
+	return xskq_cons_read_desc_batch(q, pool, entries);
 }
 /* To improve performance in the xskq_cons_release functions, only update local state here.

i40e: xsk: Move tmp desc array from driver to pool

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code