Move desc_array from the driver to the pool. The reason behind this is that we can then reuse this array as a temporary storage for descriptors in all zero-copy drivers that use the batched interface. This will make it easier to add batching to more drivers. i40e is the only driver that has a batched Tx zero-copy implementation, so no need to touch any other driver.
Unified · Split
| @@ -830,8 +830,6 @@ void i40e_free_tx_resources(struct i40e_ring *tx_ring) · 829 unmodified lines | ||
| 830 | 830 | i40e_clean_tx_ring(tx_ring); |
| 831 | 831 | kfree(tx_ring->tx_bi); |
| 832 | 832 | tx_ring->tx_bi = NULL; |
| 833 | − kfree(tx_ring->xsk_descs); | |
| 834 | − tx_ring->xsk_descs = NULL; | |
| 835 | 833 | |
| 836 | 834 | if (tx_ring->desc) { |
| 837 | 835 | dma_free_coherent(tx_ring->dev, tx_ring->size, |
| @@ -1433,13 +1431,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) · 595 unmodified lines | ||
| 1433 | 1431 | if (!tx_ring->tx_bi) |
| 1434 | 1432 | goto err; |
| 1435 | 1433 | |
| 1436 | − if (ring_is_xdp(tx_ring)) { | |
| 1437 | − tx_ring->xsk_descs = kcalloc(I40E_MAX_NUM_DESCRIPTORS, sizeof(*tx_ring->xsk_descs), | |
| 1438 | − GFP_KERNEL); | |
| 1439 | − if (!tx_ring->xsk_descs) | |
| 1440 | − goto err; | |
| 1441 | − } | |
| 1442 | − | |
| 1443 | 1434 | u64_stats_init(&tx_ring->syncp); |
| 1444 | 1435 | |
| 1445 | 1436 | /* round up to nearest 4K */ |
| @@ -1463,8 +1454,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) · 17 unmodified lines | ||
| 1463 | 1454 | return 0; |
| 1464 | 1455 | |
| 1465 | 1456 | err: |
| 1466 | − kfree(tx_ring->xsk_descs); | |
| 1467 | − tx_ring->xsk_descs = NULL; | |
| 1468 | 1457 | kfree(tx_ring->tx_bi); |
| 1469 | 1458 | tx_ring->tx_bi = NULL; |
| 1470 | 1459 | return -ENOMEM; |
| @@ -390,7 +390,6 @@ struct i40e_ring { · 389 unmodified lines | ||
| 390 | 390 | u16 rx_offset; |
| 391 | 391 | struct xdp_rxq_info xdp_rxq; |
| 392 | 392 | struct xsk_buff_pool *xsk_pool; |
| 393 | − struct xdp_desc *xsk_descs; /* For storing descriptors in the AF_XDP ZC path */ | |
| 394 | 393 | } ____cacheline_internodealigned_in_smp; |
| 395 | 394 | |
| 396 | 395 | static inline bool ring_uses_build_skb(struct i40e_ring *ring) |
| @@ -467,11 +467,11 @@ static void i40e_set_rs_bit(struct i40e_ring *xdp_ring) · 466 unmodified lines | ||
| 467 | 467 | **/ |
| 468 | 468 | static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) |
| 469 | 469 | { |
| 470 | − struct xdp_desc *descs = xdp_ring->xsk_descs; | |
| 470 | + struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs; | |
| 471 | 471 | u32 nb_pkts, nb_processed = 0; |
| 472 | 472 | unsigned int total_bytes = 0; |
| 473 | 473 | |
| 474 | − nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, descs, budget); | |
| 474 | + nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget); | |
| 475 | 475 | if (!nb_pkts) |
| 476 | 476 | return true; |
| 477 | 477 | |
| @@ -13,7 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | |
| 14 | 14 | void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); |
| 15 | 15 | bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc); |
| 16 | −u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, u32 max); | |
| 16 | +u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max); | |
| 17 | 17 | void xsk_tx_release(struct xsk_buff_pool *pool); |
| 18 | 18 | struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev, |
| 19 | 19 | u16 queue_id); |
| @@ -142,8 +142,7 @@ static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, · 122 unmodified lines | ||
| 142 | 142 | return false; |
| 143 | 143 | } |
| 144 | 144 | |
| 145 | −static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, | |
| 146 | − u32 max) | |
| 145 | +static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max) | |
| 147 | 146 | { |
| 148 | 147 | return 0; |
| 149 | 148 | } |
| @@ -60,6 +60,7 @@ struct xsk_buff_pool { · 59 unmodified lines | ||
| 60 | 60 | */ |
| 61 | 61 | dma_addr_t *dma_pages; |
| 62 | 62 | struct xdp_buff_xsk *heads; |
| 63 | + struct xdp_desc *tx_descs; | |
| 63 | 64 | u64 chunk_mask; |
| 64 | 65 | u64 addrs_cnt; |
| 65 | 66 | u32 free_list_cnt; |
| @@ -343,9 +343,9 @@ out: · 342 unmodified lines | ||
| 343 | 343 | } |
| 344 | 344 | EXPORT_SYMBOL(xsk_tx_peek_desc); |
| 345 | 345 | |
| 346 | −static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_desc *descs, | |
| 347 | − u32 max_entries) | |
| 346 | +static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries) | |
| 348 | 347 | { |
| 348 | + struct xdp_desc *descs = pool->tx_descs; | |
| 349 | 349 | u32 nb_pkts = 0; |
| 350 | 350 | |
| 351 | 351 | while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts])) |
| @@ -355,8 +355,7 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_d · 3 unmodified lines | ||
| 355 | 355 | return nb_pkts; |
| 356 | 356 | } |
| 357 | 357 | |
| 358 | −u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *descs, | |
| 359 | − u32 max_entries) | |
| 358 | +u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries) | |
| 360 | 359 | { |
| 361 | 360 | struct xdp_sock *xs; |
| 362 | 361 | u32 nb_pkts; |
| @@ -365,7 +364,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc * · 2 unmodified lines | ||
| 365 | 364 | if (!list_is_singular(&pool->xsk_tx_list)) { |
| 366 | 365 | /* Fallback to the non-batched version */ |
| 367 | 366 | rcu_read_unlock(); |
| 368 | − return xsk_tx_peek_release_fallback(pool, descs, max_entries); | |
| 367 | + return xsk_tx_peek_release_fallback(pool, max_entries); | |
| 369 | 368 | } |
| 370 | 369 | |
| 371 | 370 | xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list); |
| @@ -374,7 +373,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc * · 2 unmodified lines | ||
| 374 | 373 | goto out; |
| 375 | 374 | } |
| 376 | 375 | |
| 377 | − nb_pkts = xskq_cons_peek_desc_batch(xs->tx, descs, pool, max_entries); | |
| 376 | + nb_pkts = xskq_cons_peek_desc_batch(xs->tx, pool, max_entries); | |
| 378 | 377 | if (!nb_pkts) { |
| 379 | 378 | xs->tx->queue_empty_descs++; |
| 380 | 379 | goto out; |
| @@ -386,7 +385,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc * · 5 unmodified lines | ||
| 386 | 385 | * packets. This avoids having to implement any buffering in |
| 387 | 386 | * the Tx path. |
| 388 | 387 | */ |
| 389 | − nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, descs, nb_pkts); | |
| 388 | + nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts); | |
| 390 | 389 | if (!nb_pkts) |
| 391 | 390 | goto out; |
| 392 | 391 | |
| @@ -37,6 +37,7 @@ void xp_destroy(struct xsk_buff_pool *pool) · 36 unmodified lines | ||
| 37 | 37 | if (!pool) |
| 38 | 38 | return; |
| 39 | 39 | |
| 40 | + kvfree(pool->tx_descs); | |
| 40 | 41 | kvfree(pool->heads); |
| 41 | 42 | kvfree(pool); |
| 42 | 43 | } |
| @@ -58,6 +59,12 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, · 15 unmodified lines | ||
| 58 | 59 | if (!pool->heads) |
| 59 | 60 | goto out; |
| 60 | 61 | |
| 62 | + if (xs->tx) { | |
| 63 | + pool->tx_descs = kcalloc(xs->tx->nentries, sizeof(*pool->tx_descs), GFP_KERNEL); | |
| 64 | + if (!pool->tx_descs) | |
| 65 | + goto out; | |
| 66 | + } | |
| 67 | + | |
| 61 | 68 | pool->chunk_mask = ~((u64)umem->chunk_size - 1); |
| 62 | 69 | pool->addrs_cnt = umem->size; |
| 63 | 70 | pool->heads_cnt = umem->chunks; |
| @@ -205,11 +205,11 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q, · 204 unmodified lines | ||
| 205 | 205 | return false; |
| 206 | 206 | } |
| 207 | 207 | |
| 208 | −static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, | |
| 209 | − struct xdp_desc *descs, | |
| 210 | − struct xsk_buff_pool *pool, u32 max) | |
| 208 | +static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, | |
| 209 | + u32 max) | |
| 211 | 210 | { |
| 212 | 211 | u32 cached_cons = q->cached_cons, nb_entries = 0; |
| 212 | + struct xdp_desc *descs = pool->tx_descs; | |
| 213 | 213 | |
| 214 | 214 | while (cached_cons != q->cached_prod && nb_entries < max) { |
| 215 | 215 | struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; |
| @@ -282,12 +282,12 @@ static inline bool xskq_cons_peek_desc(struct xsk_queue *q, · 66 unmodified lines | ||
| 282 | 282 | return xskq_cons_read_desc(q, desc, pool); |
| 283 | 283 | } |
| 284 | 284 | |
| 285 | −static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xdp_desc *descs, | |
| 286 | − struct xsk_buff_pool *pool, u32 max) | |
| 285 | +static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, | |
| 286 | + u32 max) | |
| 287 | 287 | { |
| 288 | 288 | u32 entries = xskq_cons_nb_entries(q, max); |
| 289 | 289 | |
| 290 | − return xskq_cons_read_desc_batch(q, descs, pool, entries); | |
| 290 | + return xskq_cons_read_desc_batch(q, pool, entries); | |
| 291 | 291 | } |
| 292 | 292 | |
| 293 | 293 | /* To improve performance in the xskq_cons_release functions, only update local state here. |