History

Merge branch 'rework-mvneta-napi_poll-loop-for-XDP-multi-buffers'

Lorenzo Bianconi says: ==================== rework mvneta napi_poll loop for XDP multi-buffers Rework mvneta_rx_swbm routine in order to process all rx descriptors before building the skb or run the xdp program attached to the interface. Introduce xdp_get_shared_info_from_{buff,frame} utility routines to get the skb_shared_info pointer from xdp_buff or xdp_frame. This is a preliminary series to enable multi-buffers and jumbo frames for XDP according to [1] [1] https://github.com/xdp-project/xdp-project/blob/master/areas/core/xdp-multi-buffer01-design.org Changes since v1: - rely on skb_frag_* utility routines to access page/offset/len of the xdp multi-buffer ====================

David S. Miller committed 6 years ago

Commit 930bc4c2 parents bed1ce7 c7a3a8c2 files · +138 −98Browse files

Unified · Split

2 files
  1. drivers/net/ethernet/marvell/mvneta.c +123 −98
  2. include/net/xdp.h +15 −0
drivers/net/ethernet/marvell/mvneta.c+123−98
@@ -698,10 +698,6 @@ struct mvneta_rx_queue { · 697 unmodified lines
698698 /* Index of first RX DMA descriptor to refill */
699699 int first_to_refill;
700700 u32 refill_num;
701−
702− /* pointer to uncomplete skb buffer */
703− struct sk_buff *skb;
704− int left_size;
705701 };
706702
707703 static enum cpuhp_state online_hpstate;
@@ -2026,6 +2022,20 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) · 1318 unmodified lines
20262022 return i;
20272023 }
20282024
2025+static void
2026+mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2027+ struct xdp_buff *xdp, int sync_len, bool napi)
2028+{
2029+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2030+ int i;
2031+
2032+ page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),
2033+ sync_len, napi);
2034+ for (i = 0; i < sinfo->nr_frags; i++)
2035+ page_pool_put_full_page(rxq->page_pool,
2036+ skb_frag_page(&sinfo->frags[i]), napi);
2037+}
2038+
20292039 static int
20302040 mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
20312041 struct xdp_frame *xdpf, bool dma_map)
@@ -2158,13 +2168,13 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, · 126 unmodified lines
21582168 static int
21592169 mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
21602170 struct bpf_prog *prog, struct xdp_buff *xdp,
2161− struct mvneta_stats *stats)
2171+ u32 frame_sz, struct mvneta_stats *stats)
21622172 {
2163− unsigned int len, sync;
2164− struct page *page;
2173+ unsigned int len, data_len, sync;
21652174 u32 ret, act;
21662175
21672176 len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
2177+ data_len = xdp->data_end - xdp->data;
21682178 act = bpf_prog_run_xdp(prog, xdp);
21692179
21702180 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
@@ -2180,9 +2190,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 9 unmodified lines
21802190
21812191 err = xdp_do_redirect(pp->dev, xdp, prog);
21822192 if (unlikely(err)) {
2193+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
21832194 ret = MVNETA_XDP_DROPPED;
2184− page = virt_to_head_page(xdp->data);
2185− page_pool_put_page(rxq->page_pool, page, sync, true);
21862195 } else {
21872196 ret = MVNETA_XDP_REDIR;
21882197 stats->xdp_redirect++;
@@ -2191,10 +2200,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines
21912200 }
21922201 case XDP_TX:
21932202 ret = mvneta_xdp_xmit_back(pp, xdp);
2194− if (ret != MVNETA_XDP_TX) {
2195− page = virt_to_head_page(xdp->data);
2196− page_pool_put_page(rxq->page_pool, page, sync, true);
2197− }
2203+ if (ret != MVNETA_XDP_TX)
2204+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
21982205 break;
21992206 default:
22002207 bpf_warn_invalid_xdp_action(act);
@@ -2203,25 +2210,23 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines
22032210 trace_xdp_exception(pp->dev, prog, act);
22042211 /* fall through */
22052212 case XDP_DROP:
2206− page = virt_to_head_page(xdp->data);
2207− page_pool_put_page(rxq->page_pool, page, sync, true);
2213+ mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
22082214 ret = MVNETA_XDP_DROPPED;
22092215 stats->xdp_drop++;
22102216 break;
22112217 }
22122218
2213− stats->rx_bytes += xdp->data_end - xdp->data;
2219+ stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len;
22142220 stats->rx_packets++;
22152221
22162222 return ret;
22172223 }
22182224
2219−static int
2225+static void
22202226 mvneta_swbm_rx_frame(struct mvneta_port *pp,
22212227 struct mvneta_rx_desc *rx_desc,
22222228 struct mvneta_rx_queue *rxq,
2223− struct xdp_buff *xdp,
2224− struct bpf_prog *xdp_prog,
2229+ struct xdp_buff *xdp, int *size,
22252230 struct page *page,
22262231 struct mvneta_stats *stats)
22272232 {
@@ -2229,7 +2234,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 1 unmodified line
22292234 int data_len = -MVNETA_MH_SIZE, len;
22302235 struct net_device *dev = pp->dev;
22312236 enum dma_data_direction dma_dir;
2232− int ret = 0;
2237+ struct skb_shared_info *sinfo;
22332238
22342239 if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {
22352240 len = MVNETA_MAX_RX_BUF_SIZE;
@@ -2252,71 +2257,81 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 16 unmodified lines
22522257 xdp->data_end = xdp->data + data_len;
22532258 xdp_set_data_meta_invalid(xdp);
22542259
2255− if (xdp_prog) {
2256− ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats);
2257− if (ret)
2258− goto out;
2259− }
2260−
2261− rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
2262− if (unlikely(!rxq->skb)) {
2263− struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2260+ sinfo = xdp_get_shared_info_from_buff(xdp);
2261+ sinfo->nr_frags = 0;
22642262
2265− netdev_err(dev, "Can't allocate skb on queue %d\n", rxq->id);
2266−
2267− u64_stats_update_begin(&stats->syncp);
2268− stats->es.skb_alloc_error++;
2269− stats->rx_dropped++;
2270− u64_stats_update_end(&stats->syncp);
2271−
2272− return -ENOMEM;
2273− }
2274− page_pool_release_page(rxq->page_pool, page);
2275−
2276− skb_reserve(rxq->skb,
2277− xdp->data - xdp->data_hard_start);
2278− skb_put(rxq->skb, xdp->data_end - xdp->data);
2279− mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
2280−
2281− rxq->left_size = rx_desc->data_size - len;
2282−
2283−out:
2263+ *size = rx_desc->data_size - len;
22842264 rx_desc->buf_phys_addr = 0;
2285−
2286− return ret;
22872265 }
22882266
22892267 static void
22902268 mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
22912269 struct mvneta_rx_desc *rx_desc,
22922270 struct mvneta_rx_queue *rxq,
2271+ struct xdp_buff *xdp, int *size,
22932272 struct page *page)
22942273 {
2274+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
22952275 struct net_device *dev = pp->dev;
22962276 enum dma_data_direction dma_dir;
22972277 int data_len, len;
22982278
2299− if (rxq->left_size > MVNETA_MAX_RX_BUF_SIZE) {
2279+ if (*size > MVNETA_MAX_RX_BUF_SIZE) {
23002280 len = MVNETA_MAX_RX_BUF_SIZE;
23012281 data_len = len;
23022282 } else {
2303− len = rxq->left_size;
2283+ len = *size;
23042284 data_len = len - ETH_FCS_LEN;
23052285 }
23062286 dma_dir = page_pool_get_dma_dir(rxq->page_pool);
23072287 dma_sync_single_for_cpu(dev->dev.parent,
23082288 rx_desc->buf_phys_addr,
23092289 len, dma_dir);
2310− if (data_len > 0) {
2311− /* refill descriptor with new buffer later */
2312− skb_add_rx_frag(rxq->skb,
2313− skb_shinfo(rxq->skb)->nr_frags,
2314− page, pp->rx_offset_correction, data_len,
2315− PAGE_SIZE);
2316− }
2317− page_pool_release_page(rxq->page_pool, page);
2318− rx_desc->buf_phys_addr = 0;
2319− rxq->left_size -= len;
2290+
2291+ if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {
2292+ skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags];
2293+
2294+ skb_frag_off_set(frag, pp->rx_offset_correction);
2295+ skb_frag_size_set(frag, data_len);
2296+ __skb_frag_set_page(frag, page);
2297+ sinfo->nr_frags++;
2298+
2299+ rx_desc->buf_phys_addr = 0;
2300+ }
2301+ *size -= len;
2302+}
2303+
2304+static struct sk_buff *
2305+mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2306+ struct xdp_buff *xdp, u32 desc_status)
2307+{
2308+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2309+ int i, num_frags = sinfo->nr_frags;
2310+ skb_frag_t frags[MAX_SKB_FRAGS];
2311+ struct sk_buff *skb;
2312+
2313+ memcpy(frags, sinfo->frags, sizeof(skb_frag_t) * num_frags);
2314+
2315+ skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
2316+ if (!skb)
2317+ return ERR_PTR(-ENOMEM);
2318+
2319+ page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data));
2320+
2321+ skb_reserve(skb, xdp->data - xdp->data_hard_start);
2322+ skb_put(skb, xdp->data_end - xdp->data);
2323+ mvneta_rx_csum(pp, desc_status, skb);
2324+
2325+ for (i = 0; i < num_frags; i++) {
2326+ struct page *page = skb_frag_page(&frags[i]);
2327+
2328+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2329+ page, skb_frag_off(&frags[i]),
2330+ skb_frag_size(&frags[i]), PAGE_SIZE);
2331+ page_pool_release_page(rxq->page_pool, page);
2332+ }
2333+
2334+ return skb;
23202335 }
23212336
23222337 /* Main rx processing when using software buffer management */
@@ -2324,24 +2339,27 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 1 unmodified line
23242339 struct mvneta_port *pp, int budget,
23252340 struct mvneta_rx_queue *rxq)
23262341 {
2327− int rx_proc = 0, rx_todo, refill;
2342+ int rx_proc = 0, rx_todo, refill, size = 0;
23282343 struct net_device *dev = pp->dev;
2344+ struct xdp_buff xdp_buf = {
2345+ .frame_sz = PAGE_SIZE,
2346+ .rxq = &rxq->xdp_rxq,
2347+ };
23292348 struct mvneta_stats ps = {};
23302349 struct bpf_prog *xdp_prog;
2331− struct xdp_buff xdp_buf;
2350+ u32 desc_status, frame_sz;
23322351
23332352 /* Get number of received packets */
23342353 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
23352354
23362355 rcu_read_lock();
23372356 xdp_prog = READ_ONCE(pp->xdp_prog);
2338− xdp_buf.rxq = &rxq->xdp_rxq;
2339− xdp_buf.frame_sz = PAGE_SIZE;
23402357
23412358 /* Fairness NAPI loop */
23422359 while (rx_proc < budget && rx_proc < rx_todo) {
23432360 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
23442361 u32 rx_status, index;
2362+ struct sk_buff *skb;
23452363 struct page *page;
23462364
23472365 index = rx_desc - rxq->descs;
@@ -2352,54 +2370,66 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 4 unmodified lines
23522370 rxq->refill_num++;
23532371
23542372 if (rx_status & MVNETA_RXD_FIRST_DESC) {
2355− int err;
2356−
23572373 /* Check errors only for FIRST descriptor */
23582374 if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
23592375 mvneta_rx_error(pp, rx_desc);
2360− /* leave the descriptor untouched */
2361− continue;
2376+ goto next;
23622377 }
23632378
2364− err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
2365− xdp_prog, page, &ps);
2366− if (err)
2367− continue;
2379+ size = rx_desc->data_size;
2380+ frame_sz = size - ETH_FCS_LEN;
2381+ desc_status = rx_desc->status;
2382+
2383+ mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
2384+ &size, page, &ps);
23682385 } else {
2369− if (unlikely(!rxq->skb)) {
2370− pr_debug("no skb for rx_status 0x%x\n",
2371− rx_status);
2386+ if (unlikely(!xdp_buf.data_hard_start))
23722387 continue;
2373− }
2374− mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page);
2388+
2389+ mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,
2390+ &size, page);
23752391 } /* Middle or Last descriptor */
23762392
23772393 if (!(rx_status & MVNETA_RXD_LAST_DESC))
23782394 /* no last descriptor this time */
23792395 continue;
23802396
2381− if (rxq->left_size) {
2382− pr_err("get last desc, but left_size (%d) != 0\n",
2383− rxq->left_size);
2384− dev_kfree_skb_any(rxq->skb);
2385− rxq->left_size = 0;
2386− rxq->skb = NULL;
2387− continue;
2397+ if (size) {
2398+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2399+ goto next;
23882400 }
23892401
2390− ps.rx_bytes += rxq->skb->len;
2391− ps.rx_packets++;
2402+ if (xdp_prog &&
2403+ mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps))
2404+ goto next;
23922405
2393− /* Linux processing */
2394− rxq->skb->protocol = eth_type_trans(rxq->skb, dev);
2406+ skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status);
2407+ if (IS_ERR(skb)) {
2408+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2409+
2410+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
23952411
2396− napi_gro_receive(napi, rxq->skb);
2412+ u64_stats_update_begin(&stats->syncp);
2413+ stats->es.skb_alloc_error++;
2414+ stats->rx_dropped++;
2415+ u64_stats_update_end(&stats->syncp);
23972416
2398− /* clean uncomplete skb pointer in queue */
2399− rxq->skb = NULL;
2417+ goto next;
2418+ }
2419+
2420+ ps.rx_bytes += skb->len;
2421+ ps.rx_packets++;
2422+
2423+ skb->protocol = eth_type_trans(skb, dev);
2424+ napi_gro_receive(napi, skb);
2425+next:
2426+ xdp_buf.data_hard_start = NULL;
24002427 }
24012428 rcu_read_unlock();
24022429
2430+ if (xdp_buf.data_hard_start)
2431+ mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2432+
24032433 if (ps.xdp_redirect)
24042434 xdp_do_flush_map();
24052435
@@ -3328,9 +3358,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 922 unmodified lines
33283358 {
33293359 mvneta_rxq_drop_pkts(pp, rxq);
33303360
3331− if (rxq->skb)
3332− dev_kfree_skb_any(rxq->skb);
3333−
33343361 if (rxq->descs)
33353362 dma_free_coherent(pp->dev->dev.parent,
33363363 rxq->size * MVNETA_DESC_ALIGNED_SIZE,
@@ -3343,8 +3370,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 6 unmodified lines
33433370 rxq->descs_phys = 0;
33443371 rxq->first_to_refill = 0;
33453372 rxq->refill_num = 0;
3346− rxq->skb = NULL;
3347− rxq->left_size = 0;
33483373 }
33493374
33503375 static int mvneta_txq_sw_init(struct mvneta_port *pp,
include/net/xdp.h+15−0
@@ -85,6 +85,12 @@ struct xdp_buff { · 84 unmodified lines
8585 ((xdp)->data_hard_start + (xdp)->frame_sz - \
8686 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
8787
88+static inline struct skb_shared_info *
89+xdp_get_shared_info_from_buff(struct xdp_buff *xdp)
90+{
91+ return (struct skb_shared_info *)xdp_data_hard_end(xdp);
92+}
93+
8894 struct xdp_frame {
8995 void *data;
9096 u16 len;
@@ -98,6 +104,15 @@ struct xdp_frame { · 7 unmodified lines
98104 struct net_device *dev_rx; /* used by cpumap */
99105 };
100106
107+static inline struct skb_shared_info *
108+xdp_get_shared_info_from_frame(struct xdp_frame *frame)
109+{
110+ void *data_hard_start = frame->data - frame->headroom - sizeof(*frame);
111+
112+ return (struct skb_shared_info *)(data_hard_start + frame->frame_sz -
113+ SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
114+}
115+
101116 /* Clear kernel pointers in xdp_frame */
102117 static inline void xdp_scrub_frame(struct xdp_frame *frame)
103118 {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t