History

Merge branch 'rework-mvneta-napi_poll-loop-for-XDP-multi-buffers'

Lorenzo Bianconi says: ==================== rework mvneta napi_poll loop for XDP multi-buffers Rework mvneta_rx_swbm routine in order to process all rx descriptors before building the skb or run the xdp program attached to the interface. Introduce xdp_get_shared_info_from_{buff,frame} utility routines to get the skb_shared_info pointer from xdp_buff or xdp_frame. This is a preliminary series to enable multi-buffers and jumbo frames for XDP according to [1] [1] https://github.com/xdp-project/xdp-project/blob/master/areas/core/xdp-multi-buffer01-design.org Changes since v1: - rely on skb_frag_* utility routines to access page/offset/len of the xdp multi-buffer ====================

David S. Miller committed 6 years ago

Commit 930bc4c2 parents bed1ce7 c7a3a8c2 files · +138 −98Browse files

Unified · Split

2 files
  1. drivers/net/ethernet/marvell/mvneta.c +123 −98
  2. include/net/xdp.h +15 −0
drivers/net/ethernet/marvell/mvneta.c+123−98
@@ -698,10 +698,6 @@ struct mvneta_rx_queue { · 697 unmodified lines
698 /* Index of first RX DMA descriptor to refill */698 /* Index of first RX DMA descriptor to refill */
699 int first_to_refill;699 int first_to_refill;
700 u32 refill_num;700 u32 refill_num;
701
702 /* pointer to uncomplete skb buffer */
703 struct sk_buff *skb;
704 int left_size;
705};701};
706 702
707static enum cpuhp_state online_hpstate;703static enum cpuhp_state online_hpstate;
@@ -2026,6 +2022,20 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) · 1318 unmodified lines
2026 return i;2022 return i;
2027}2023}
2028 2024
2025static void
2026mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2027 struct xdp_buff *xdp, int sync_len, bool napi)
2028{
2029 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2030 int i;
2031
2032 page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),
2033 sync_len, napi);
2034 for (i = 0; i < sinfo->nr_frags; i++)
2035 page_pool_put_full_page(rxq->page_pool,
2036 skb_frag_page(&sinfo->frags[i]), napi);
2037}
2038
2029static int2039static int
2030mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,2040mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
2031 struct xdp_frame *xdpf, bool dma_map)2041 struct xdp_frame *xdpf, bool dma_map)
@@ -2158,13 +2168,13 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, · 126 unmodified lines
2158static int2168static int
2159mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,2169mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2160 struct bpf_prog *prog, struct xdp_buff *xdp,2170 struct bpf_prog *prog, struct xdp_buff *xdp,
2161 struct mvneta_stats *stats)2171 u32 frame_sz, struct mvneta_stats *stats)
2162{2172{
2163 unsigned int len, sync;2173 unsigned int len, data_len, sync;
2164 struct page *page;
2165 u32 ret, act;2174 u32 ret, act;
2166 2175
2167 len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;2176 len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
2177 data_len = xdp->data_end - xdp->data;
2168 act = bpf_prog_run_xdp(prog, xdp);2178 act = bpf_prog_run_xdp(prog, xdp);
2169 2179
2170 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */2180 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
@@ -2180,9 +2190,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 9 unmodified lines
2180 2190
2181 err = xdp_do_redirect(pp->dev, xdp, prog);2191 err = xdp_do_redirect(pp->dev, xdp, prog);
2182 if (unlikely(err)) {2192 if (unlikely(err)) {
2193 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
2183 ret = MVNETA_XDP_DROPPED;2194 ret = MVNETA_XDP_DROPPED;
2184 page = virt_to_head_page(xdp->data);
2185 page_pool_put_page(rxq->page_pool, page, sync, true);
2186 } else {2195 } else {
2187 ret = MVNETA_XDP_REDIR;2196 ret = MVNETA_XDP_REDIR;
2188 stats->xdp_redirect++;2197 stats->xdp_redirect++;
@@ -2191,10 +2200,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines
2191 }2200 }
2192 case XDP_TX:2201 case XDP_TX:
2193 ret = mvneta_xdp_xmit_back(pp, xdp);2202 ret = mvneta_xdp_xmit_back(pp, xdp);
2194 if (ret != MVNETA_XDP_TX) {2203 if (ret != MVNETA_XDP_TX)
2195 page = virt_to_head_page(xdp->data);2204 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
2196 page_pool_put_page(rxq->page_pool, page, sync, true);
2197 }
2198 break;2205 break;
2199 default:2206 default:
2200 bpf_warn_invalid_xdp_action(act);2207 bpf_warn_invalid_xdp_action(act);
@@ -2203,25 +2210,23 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines
2203 trace_xdp_exception(pp->dev, prog, act);2210 trace_xdp_exception(pp->dev, prog, act);
2204 /* fall through */2211 /* fall through */
2205 case XDP_DROP:2212 case XDP_DROP:
2206 page = virt_to_head_page(xdp->data);2213 mvneta_xdp_put_buff(pp, rxq, xdp, sync, true);
2207 page_pool_put_page(rxq->page_pool, page, sync, true);
2208 ret = MVNETA_XDP_DROPPED;2214 ret = MVNETA_XDP_DROPPED;
2209 stats->xdp_drop++;2215 stats->xdp_drop++;
2210 break;2216 break;
2211 }2217 }
2212 2218
2213 stats->rx_bytes += xdp->data_end - xdp->data;2219 stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len;
2214 stats->rx_packets++;2220 stats->rx_packets++;
2215 2221
2216 return ret;2222 return ret;
2217}2223}
2218 2224
2219static int2225static void
2220mvneta_swbm_rx_frame(struct mvneta_port *pp,2226mvneta_swbm_rx_frame(struct mvneta_port *pp,
2221 struct mvneta_rx_desc *rx_desc,2227 struct mvneta_rx_desc *rx_desc,
2222 struct mvneta_rx_queue *rxq,2228 struct mvneta_rx_queue *rxq,
2223 struct xdp_buff *xdp,2229 struct xdp_buff *xdp, int *size,
2224 struct bpf_prog *xdp_prog,
2225 struct page *page,2230 struct page *page,
2226 struct mvneta_stats *stats)2231 struct mvneta_stats *stats)
2227{2232{
@@ -2229,7 +2234,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 1 unmodified line
2229 int data_len = -MVNETA_MH_SIZE, len;2234 int data_len = -MVNETA_MH_SIZE, len;
2230 struct net_device *dev = pp->dev;2235 struct net_device *dev = pp->dev;
2231 enum dma_data_direction dma_dir;2236 enum dma_data_direction dma_dir;
2232 int ret = 0;2237 struct skb_shared_info *sinfo;
2233 2238
2234 if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {2239 if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) {
2235 len = MVNETA_MAX_RX_BUF_SIZE;2240 len = MVNETA_MAX_RX_BUF_SIZE;
@@ -2252,71 +2257,81 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 16 unmodified lines
2252 xdp->data_end = xdp->data + data_len;2257 xdp->data_end = xdp->data + data_len;
2253 xdp_set_data_meta_invalid(xdp);2258 xdp_set_data_meta_invalid(xdp);
2254 2259
2255 if (xdp_prog) {2260 sinfo = xdp_get_shared_info_from_buff(xdp);
2256 ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats);2261 sinfo->nr_frags = 0;
2257 if (ret)
2258 goto out;
2259 }
2260
2261 rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
2262 if (unlikely(!rxq->skb)) {
2263 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2264 2262
2265 netdev_err(dev, "Can't allocate skb on queue %d\n", rxq->id);2263 *size = rx_desc->data_size - len;
2266
2267 u64_stats_update_begin(&stats->syncp);
2268 stats->es.skb_alloc_error++;
2269 stats->rx_dropped++;
2270 u64_stats_update_end(&stats->syncp);
2271
2272 return -ENOMEM;
2273 }
2274 page_pool_release_page(rxq->page_pool, page);
2275
2276 skb_reserve(rxq->skb,
2277 xdp->data - xdp->data_hard_start);
2278 skb_put(rxq->skb, xdp->data_end - xdp->data);
2279 mvneta_rx_csum(pp, rx_desc->status, rxq->skb);
2280
2281 rxq->left_size = rx_desc->data_size - len;
2282
2283out:
2284 rx_desc->buf_phys_addr = 0;2264 rx_desc->buf_phys_addr = 0;
2285
2286 return ret;
2287}2265}
2288 2266
2289static void2267static void
2290mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,2268mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
2291 struct mvneta_rx_desc *rx_desc,2269 struct mvneta_rx_desc *rx_desc,
2292 struct mvneta_rx_queue *rxq,2270 struct mvneta_rx_queue *rxq,
2271 struct xdp_buff *xdp, int *size,
2293 struct page *page)2272 struct page *page)
2294{2273{
2274 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2295 struct net_device *dev = pp->dev;2275 struct net_device *dev = pp->dev;
2296 enum dma_data_direction dma_dir;2276 enum dma_data_direction dma_dir;
2297 int data_len, len;2277 int data_len, len;
2298 2278
2299 if (rxq->left_size > MVNETA_MAX_RX_BUF_SIZE) {2279 if (*size > MVNETA_MAX_RX_BUF_SIZE) {
2300 len = MVNETA_MAX_RX_BUF_SIZE;2280 len = MVNETA_MAX_RX_BUF_SIZE;
2301 data_len = len;2281 data_len = len;
2302 } else {2282 } else {
2303 len = rxq->left_size;2283 len = *size;
2304 data_len = len - ETH_FCS_LEN;2284 data_len = len - ETH_FCS_LEN;
2305 }2285 }
2306 dma_dir = page_pool_get_dma_dir(rxq->page_pool);2286 dma_dir = page_pool_get_dma_dir(rxq->page_pool);
2307 dma_sync_single_for_cpu(dev->dev.parent,2287 dma_sync_single_for_cpu(dev->dev.parent,
2308 rx_desc->buf_phys_addr,2288 rx_desc->buf_phys_addr,
2309 len, dma_dir);2289 len, dma_dir);
2310 if (data_len > 0) {2290
2311 /* refill descriptor with new buffer later */2291 if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {
2312 skb_add_rx_frag(rxq->skb,2292 skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags];
2313 skb_shinfo(rxq->skb)->nr_frags,2293
2314 page, pp->rx_offset_correction, data_len,2294 skb_frag_off_set(frag, pp->rx_offset_correction);
2315 PAGE_SIZE);2295 skb_frag_size_set(frag, data_len);
2316 }2296 __skb_frag_set_page(frag, page);
2317 page_pool_release_page(rxq->page_pool, page);2297 sinfo->nr_frags++;
2318 rx_desc->buf_phys_addr = 0;2298
2319 rxq->left_size -= len;2299 rx_desc->buf_phys_addr = 0;
2300 }
2301 *size -= len;
2302}
2303
2304static struct sk_buff *
2305mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2306 struct xdp_buff *xdp, u32 desc_status)
2307{
2308 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2309 int i, num_frags = sinfo->nr_frags;
2310 skb_frag_t frags[MAX_SKB_FRAGS];
2311 struct sk_buff *skb;
2312
2313 memcpy(frags, sinfo->frags, sizeof(skb_frag_t) * num_frags);
2314
2315 skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
2316 if (!skb)
2317 return ERR_PTR(-ENOMEM);
2318
2319 page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data));
2320
2321 skb_reserve(skb, xdp->data - xdp->data_hard_start);
2322 skb_put(skb, xdp->data_end - xdp->data);
2323 mvneta_rx_csum(pp, desc_status, skb);
2324
2325 for (i = 0; i < num_frags; i++) {
2326 struct page *page = skb_frag_page(&frags[i]);
2327
2328 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2329 page, skb_frag_off(&frags[i]),
2330 skb_frag_size(&frags[i]), PAGE_SIZE);
2331 page_pool_release_page(rxq->page_pool, page);
2332 }
2333
2334 return skb;
2320}2335}
2321 2336
2322/* Main rx processing when using software buffer management */2337/* Main rx processing when using software buffer management */
@@ -2324,24 +2339,27 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 1 unmodified line
2324 struct mvneta_port *pp, int budget,2339 struct mvneta_port *pp, int budget,
2325 struct mvneta_rx_queue *rxq)2340 struct mvneta_rx_queue *rxq)
2326{2341{
2327 int rx_proc = 0, rx_todo, refill;2342 int rx_proc = 0, rx_todo, refill, size = 0;
2328 struct net_device *dev = pp->dev;2343 struct net_device *dev = pp->dev;
2344 struct xdp_buff xdp_buf = {
2345 .frame_sz = PAGE_SIZE,
2346 .rxq = &rxq->xdp_rxq,
2347 };
2329 struct mvneta_stats ps = {};2348 struct mvneta_stats ps = {};
2330 struct bpf_prog *xdp_prog;2349 struct bpf_prog *xdp_prog;
2331 struct xdp_buff xdp_buf;2350 u32 desc_status, frame_sz;
2332 2351
2333 /* Get number of received packets */2352 /* Get number of received packets */
2334 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);2353 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
2335 2354
2336 rcu_read_lock();2355 rcu_read_lock();
2337 xdp_prog = READ_ONCE(pp->xdp_prog);2356 xdp_prog = READ_ONCE(pp->xdp_prog);
2338 xdp_buf.rxq = &rxq->xdp_rxq;
2339 xdp_buf.frame_sz = PAGE_SIZE;
2340 2357
2341 /* Fairness NAPI loop */2358 /* Fairness NAPI loop */
2342 while (rx_proc < budget && rx_proc < rx_todo) {2359 while (rx_proc < budget && rx_proc < rx_todo) {
2343 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);2360 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
2344 u32 rx_status, index;2361 u32 rx_status, index;
2362 struct sk_buff *skb;
2345 struct page *page;2363 struct page *page;
2346 2364
2347 index = rx_desc - rxq->descs;2365 index = rx_desc - rxq->descs;
@@ -2352,54 +2370,66 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 4 unmodified lines
2352 rxq->refill_num++;2370 rxq->refill_num++;
2353 2371
2354 if (rx_status & MVNETA_RXD_FIRST_DESC) {2372 if (rx_status & MVNETA_RXD_FIRST_DESC) {
2355 int err;
2356
2357 /* Check errors only for FIRST descriptor */2373 /* Check errors only for FIRST descriptor */
2358 if (rx_status & MVNETA_RXD_ERR_SUMMARY) {2374 if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
2359 mvneta_rx_error(pp, rx_desc);2375 mvneta_rx_error(pp, rx_desc);
2360 /* leave the descriptor untouched */2376 goto next;
2361 continue;
2362 }2377 }
2363 2378
2364 err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,2379 size = rx_desc->data_size;
2365 xdp_prog, page, &ps);2380 frame_sz = size - ETH_FCS_LEN;
2366 if (err)2381 desc_status = rx_desc->status;
2367 continue;2382
2383 mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
2384 &size, page, &ps);
2368 } else {2385 } else {
2369 if (unlikely(!rxq->skb)) {2386 if (unlikely(!xdp_buf.data_hard_start))
2370 pr_debug("no skb for rx_status 0x%x\n",
2371 rx_status);
2372 continue;2387 continue;
2373 }2388
2374 mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page);2389 mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,
2390 &size, page);
2375 } /* Middle or Last descriptor */2391 } /* Middle or Last descriptor */
2376 2392
2377 if (!(rx_status & MVNETA_RXD_LAST_DESC))2393 if (!(rx_status & MVNETA_RXD_LAST_DESC))
2378 /* no last descriptor this time */2394 /* no last descriptor this time */
2379 continue;2395 continue;
2380 2396
2381 if (rxq->left_size) {2397 if (size) {
2382 pr_err("get last desc, but left_size (%d) != 0\n",2398 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2383 rxq->left_size);2399 goto next;
2384 dev_kfree_skb_any(rxq->skb);
2385 rxq->left_size = 0;
2386 rxq->skb = NULL;
2387 continue;
2388 }2400 }
2389 2401
2390 ps.rx_bytes += rxq->skb->len;2402 if (xdp_prog &&
2391 ps.rx_packets++;2403 mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps))
2404 goto next;
2392 2405
2393 /* Linux processing */2406 skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status);
2394 rxq->skb->protocol = eth_type_trans(rxq->skb, dev);2407 if (IS_ERR(skb)) {
2408 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2409
2410 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2395 2411
2396 napi_gro_receive(napi, rxq->skb);2412 u64_stats_update_begin(&stats->syncp);
2413 stats->es.skb_alloc_error++;
2414 stats->rx_dropped++;
2415 u64_stats_update_end(&stats->syncp);
2397 2416
2398 /* clean uncomplete skb pointer in queue */2417 goto next;
2399 rxq->skb = NULL;2418 }
2419
2420 ps.rx_bytes += skb->len;
2421 ps.rx_packets++;
2422
2423 skb->protocol = eth_type_trans(skb, dev);
2424 napi_gro_receive(napi, skb);
2425next:
2426 xdp_buf.data_hard_start = NULL;
2400 }2427 }
2401 rcu_read_unlock();2428 rcu_read_unlock();
2402 2429
2430 if (xdp_buf.data_hard_start)
2431 mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true);
2432
2403 if (ps.xdp_redirect)2433 if (ps.xdp_redirect)
2404 xdp_do_flush_map();2434 xdp_do_flush_map();
2405 2435
@@ -3328,9 +3358,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 922 unmodified lines
3328{3358{
3329 mvneta_rxq_drop_pkts(pp, rxq);3359 mvneta_rxq_drop_pkts(pp, rxq);
3330 3360
3331 if (rxq->skb)
3332 dev_kfree_skb_any(rxq->skb);
3333
3334 if (rxq->descs)3361 if (rxq->descs)
3335 dma_free_coherent(pp->dev->dev.parent,3362 dma_free_coherent(pp->dev->dev.parent,
3336 rxq->size * MVNETA_DESC_ALIGNED_SIZE,3363 rxq->size * MVNETA_DESC_ALIGNED_SIZE,
@@ -3343,8 +3370,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 6 unmodified lines
3343 rxq->descs_phys = 0;3370 rxq->descs_phys = 0;
3344 rxq->first_to_refill = 0;3371 rxq->first_to_refill = 0;
3345 rxq->refill_num = 0;3372 rxq->refill_num = 0;
3346 rxq->skb = NULL;
3347 rxq->left_size = 0;
3348}3373}
3349 3374
3350static int mvneta_txq_sw_init(struct mvneta_port *pp,3375static int mvneta_txq_sw_init(struct mvneta_port *pp,
include/net/xdp.h+15−0
@@ -85,6 +85,12 @@ struct xdp_buff { · 84 unmodified lines
85 ((xdp)->data_hard_start + (xdp)->frame_sz - \85 ((xdp)->data_hard_start + (xdp)->frame_sz - \
86 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))86 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
87 87
88static inline struct skb_shared_info *
89xdp_get_shared_info_from_buff(struct xdp_buff *xdp)
90{
91 return (struct skb_shared_info *)xdp_data_hard_end(xdp);
92}
93
88struct xdp_frame {94struct xdp_frame {
89 void *data;95 void *data;
90 u16 len;96 u16 len;
@@ -98,6 +104,15 @@ struct xdp_frame { · 7 unmodified lines
98 struct net_device *dev_rx; /* used by cpumap */104 struct net_device *dev_rx; /* used by cpumap */
99};105};
100 106
107static inline struct skb_shared_info *
108xdp_get_shared_info_from_frame(struct xdp_frame *frame)
109{
110 void *data_hard_start = frame->data - frame->headroom - sizeof(*frame);
111
112 return (struct skb_shared_info *)(data_hard_start + frame->frame_sz -
113 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
114}
115
101/* Clear kernel pointers in xdp_frame */116/* Clear kernel pointers in xdp_frame */
102static inline void xdp_scrub_frame(struct xdp_frame *frame)117static inline void xdp_scrub_frame(struct xdp_frame *frame)
103{118{

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t