Lorenzo Bianconi says: ==================== rework mvneta napi_poll loop for XDP multi-buffers Rework mvneta_rx_swbm routine in order to process all rx descriptors before building the skb or run the xdp program attached to the interface. Introduce xdp_get_shared_info_from_{buff,frame} utility routines to get the skb_shared_info pointer from xdp_buff or xdp_frame. This is a preliminary series to enable multi-buffers and jumbo frames for XDP according to [1] [1] https://github.com/xdp-project/xdp-project/blob/master/areas/core/xdp-multi-buffer01-design.org Changes since v1: - rely on skb_frag_* utility routines to access page/offset/len of the xdp multi-buffer ====================
Unified · Split
| @@ -698,10 +698,6 @@ struct mvneta_rx_queue { · 697 unmodified lines | |||
| 698 | /* Index of first RX DMA descriptor to refill */ | 698 | /* Index of first RX DMA descriptor to refill */ |
| 699 | int first_to_refill; | 699 | int first_to_refill; |
| 700 | u32 refill_num; | 700 | u32 refill_num; |
| 701 | |||
| 702 | /* pointer to uncomplete skb buffer */ | ||
| 703 | struct sk_buff *skb; | ||
| 704 | int left_size; | ||
| 705 | }; | 701 | }; |
| 706 | 702 | ||
| 707 | static enum cpuhp_state online_hpstate; | 703 | static enum cpuhp_state online_hpstate; |
| @@ -2026,6 +2022,20 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) · 1318 unmodified lines | |||
| 2026 | return i; | 2022 | return i; |
| 2027 | } | 2023 | } |
| 2028 | 2024 | ||
| 2025 | static void | ||
| 2026 | mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, | ||
| 2027 | struct xdp_buff *xdp, int sync_len, bool napi) | ||
| 2028 | { | ||
| 2029 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | ||
| 2030 | int i; | ||
| 2031 | |||
| 2032 | page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data), | ||
| 2033 | sync_len, napi); | ||
| 2034 | for (i = 0; i < sinfo->nr_frags; i++) | ||
| 2035 | page_pool_put_full_page(rxq->page_pool, | ||
| 2036 | skb_frag_page(&sinfo->frags[i]), napi); | ||
| 2037 | } | ||
| 2038 | |||
| 2029 | static int | 2039 | static int |
| 2030 | mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq, | 2040 | mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq, |
| 2031 | struct xdp_frame *xdpf, bool dma_map) | 2041 | struct xdp_frame *xdpf, bool dma_map) |
| @@ -2158,13 +2168,13 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, · 126 unmodified lines | |||
| 2158 | static int | 2168 | static int |
| 2159 | mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, | 2169 | mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, |
| 2160 | struct bpf_prog *prog, struct xdp_buff *xdp, | 2170 | struct bpf_prog *prog, struct xdp_buff *xdp, |
| 2161 | struct mvneta_stats *stats) | 2171 | u32 frame_sz, struct mvneta_stats *stats) |
| 2162 | { | 2172 | { |
| 2163 | unsigned int len, sync; | 2173 | unsigned int len, data_len, sync; |
| 2164 | struct page *page; | ||
| 2165 | u32 ret, act; | 2174 | u32 ret, act; |
| 2166 | 2175 | ||
| 2167 | len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction; | 2176 | len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction; |
| 2177 | data_len = xdp->data_end - xdp->data; | ||
| 2168 | act = bpf_prog_run_xdp(prog, xdp); | 2178 | act = bpf_prog_run_xdp(prog, xdp); |
| 2169 | 2179 | ||
| 2170 | /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */ | 2180 | /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */ |
| @@ -2180,9 +2190,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 9 unmodified lines | |||
| 2180 | 2190 | ||
| 2181 | err = xdp_do_redirect(pp->dev, xdp, prog); | 2191 | err = xdp_do_redirect(pp->dev, xdp, prog); |
| 2182 | if (unlikely(err)) { | 2192 | if (unlikely(err)) { |
| 2193 | mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); | ||
| 2183 | ret = MVNETA_XDP_DROPPED; | 2194 | ret = MVNETA_XDP_DROPPED; |
| 2184 | page = virt_to_head_page(xdp->data); | ||
| 2185 | page_pool_put_page(rxq->page_pool, page, sync, true); | ||
| 2186 | } else { | 2195 | } else { |
| 2187 | ret = MVNETA_XDP_REDIR; | 2196 | ret = MVNETA_XDP_REDIR; |
| 2188 | stats->xdp_redirect++; | 2197 | stats->xdp_redirect++; |
| @@ -2191,10 +2200,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines | |||
| 2191 | } | 2200 | } |
| 2192 | case XDP_TX: | 2201 | case XDP_TX: |
| 2193 | ret = mvneta_xdp_xmit_back(pp, xdp); | 2202 | ret = mvneta_xdp_xmit_back(pp, xdp); |
| 2194 | if (ret != MVNETA_XDP_TX) { | 2203 | if (ret != MVNETA_XDP_TX) |
| 2195 | page = virt_to_head_page(xdp->data); | 2204 | mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); |
| 2196 | page_pool_put_page(rxq->page_pool, page, sync, true); | ||
| 2197 | } | ||
| 2198 | break; | 2205 | break; |
| 2199 | default: | 2206 | default: |
| 2200 | bpf_warn_invalid_xdp_action(act); | 2207 | bpf_warn_invalid_xdp_action(act); |
| @@ -2203,25 +2210,23 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines | |||
| 2203 | trace_xdp_exception(pp->dev, prog, act); | 2210 | trace_xdp_exception(pp->dev, prog, act); |
| 2204 | /* fall through */ | 2211 | /* fall through */ |
| 2205 | case XDP_DROP: | 2212 | case XDP_DROP: |
| 2206 | page = virt_to_head_page(xdp->data); | 2213 | mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); |
| 2207 | page_pool_put_page(rxq->page_pool, page, sync, true); | ||
| 2208 | ret = MVNETA_XDP_DROPPED; | 2214 | ret = MVNETA_XDP_DROPPED; |
| 2209 | stats->xdp_drop++; | 2215 | stats->xdp_drop++; |
| 2210 | break; | 2216 | break; |
| 2211 | } | 2217 | } |
| 2212 | 2218 | ||
| 2213 | stats->rx_bytes += xdp->data_end - xdp->data; | 2219 | stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len; |
| 2214 | stats->rx_packets++; | 2220 | stats->rx_packets++; |
| 2215 | 2221 | ||
| 2216 | return ret; | 2222 | return ret; |
| 2217 | } | 2223 | } |
| 2218 | 2224 | ||
| 2219 | static int | 2225 | static void |
| 2220 | mvneta_swbm_rx_frame(struct mvneta_port *pp, | 2226 | mvneta_swbm_rx_frame(struct mvneta_port *pp, |
| 2221 | struct mvneta_rx_desc *rx_desc, | 2227 | struct mvneta_rx_desc *rx_desc, |
| 2222 | struct mvneta_rx_queue *rxq, | 2228 | struct mvneta_rx_queue *rxq, |
| 2223 | struct xdp_buff *xdp, | 2229 | struct xdp_buff *xdp, int *size, |
| 2224 | struct bpf_prog *xdp_prog, | ||
| 2225 | struct page *page, | 2230 | struct page *page, |
| 2226 | struct mvneta_stats *stats) | 2231 | struct mvneta_stats *stats) |
| 2227 | { | 2232 | { |
| @@ -2229,7 +2234,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 1 unmodified line | |||
| 2229 | int data_len = -MVNETA_MH_SIZE, len; | 2234 | int data_len = -MVNETA_MH_SIZE, len; |
| 2230 | struct net_device *dev = pp->dev; | 2235 | struct net_device *dev = pp->dev; |
| 2231 | enum dma_data_direction dma_dir; | 2236 | enum dma_data_direction dma_dir; |
| 2232 | int ret = 0; | 2237 | struct skb_shared_info *sinfo; |
| 2233 | 2238 | ||
| 2234 | if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) { | 2239 | if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) { |
| 2235 | len = MVNETA_MAX_RX_BUF_SIZE; | 2240 | len = MVNETA_MAX_RX_BUF_SIZE; |
| @@ -2252,71 +2257,81 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 16 unmodified lines | |||
| 2252 | xdp->data_end = xdp->data + data_len; | 2257 | xdp->data_end = xdp->data + data_len; |
| 2253 | xdp_set_data_meta_invalid(xdp); | 2258 | xdp_set_data_meta_invalid(xdp); |
| 2254 | 2259 | ||
| 2255 | if (xdp_prog) { | 2260 | sinfo = xdp_get_shared_info_from_buff(xdp); |
| 2256 | ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats); | 2261 | sinfo->nr_frags = 0; |
| 2257 | if (ret) | ||
| 2258 | goto out; | ||
| 2259 | } | ||
| 2260 | |||
| 2261 | rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE); | ||
| 2262 | if (unlikely(!rxq->skb)) { | ||
| 2263 | struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); | ||
| 2264 | 2262 | ||
| 2265 | netdev_err(dev, "Can't allocate skb on queue %d\n", rxq->id); | 2263 | *size = rx_desc->data_size - len; |
| 2266 | |||
| 2267 | u64_stats_update_begin(&stats->syncp); | ||
| 2268 | stats->es.skb_alloc_error++; | ||
| 2269 | stats->rx_dropped++; | ||
| 2270 | u64_stats_update_end(&stats->syncp); | ||
| 2271 | |||
| 2272 | return -ENOMEM; | ||
| 2273 | } | ||
| 2274 | page_pool_release_page(rxq->page_pool, page); | ||
| 2275 | |||
| 2276 | skb_reserve(rxq->skb, | ||
| 2277 | xdp->data - xdp->data_hard_start); | ||
| 2278 | skb_put(rxq->skb, xdp->data_end - xdp->data); | ||
| 2279 | mvneta_rx_csum(pp, rx_desc->status, rxq->skb); | ||
| 2280 | |||
| 2281 | rxq->left_size = rx_desc->data_size - len; | ||
| 2282 | |||
| 2283 | out: | ||
| 2284 | rx_desc->buf_phys_addr = 0; | 2264 | rx_desc->buf_phys_addr = 0; |
| 2285 | |||
| 2286 | return ret; | ||
| 2287 | } | 2265 | } |
| 2288 | 2266 | ||
| 2289 | static void | 2267 | static void |
| 2290 | mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, | 2268 | mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, |
| 2291 | struct mvneta_rx_desc *rx_desc, | 2269 | struct mvneta_rx_desc *rx_desc, |
| 2292 | struct mvneta_rx_queue *rxq, | 2270 | struct mvneta_rx_queue *rxq, |
| 2271 | struct xdp_buff *xdp, int *size, | ||
| 2293 | struct page *page) | 2272 | struct page *page) |
| 2294 | { | 2273 | { |
| 2274 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | ||
| 2295 | struct net_device *dev = pp->dev; | 2275 | struct net_device *dev = pp->dev; |
| 2296 | enum dma_data_direction dma_dir; | 2276 | enum dma_data_direction dma_dir; |
| 2297 | int data_len, len; | 2277 | int data_len, len; |
| 2298 | 2278 | ||
| 2299 | if (rxq->left_size > MVNETA_MAX_RX_BUF_SIZE) { | 2279 | if (*size > MVNETA_MAX_RX_BUF_SIZE) { |
| 2300 | len = MVNETA_MAX_RX_BUF_SIZE; | 2280 | len = MVNETA_MAX_RX_BUF_SIZE; |
| 2301 | data_len = len; | 2281 | data_len = len; |
| 2302 | } else { | 2282 | } else { |
| 2303 | len = rxq->left_size; | 2283 | len = *size; |
| 2304 | data_len = len - ETH_FCS_LEN; | 2284 | data_len = len - ETH_FCS_LEN; |
| 2305 | } | 2285 | } |
| 2306 | dma_dir = page_pool_get_dma_dir(rxq->page_pool); | 2286 | dma_dir = page_pool_get_dma_dir(rxq->page_pool); |
| 2307 | dma_sync_single_for_cpu(dev->dev.parent, | 2287 | dma_sync_single_for_cpu(dev->dev.parent, |
| 2308 | rx_desc->buf_phys_addr, | 2288 | rx_desc->buf_phys_addr, |
| 2309 | len, dma_dir); | 2289 | len, dma_dir); |
| 2310 | if (data_len > 0) { | 2290 | |
| 2311 | /* refill descriptor with new buffer later */ | 2291 | if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) { |
| 2312 | skb_add_rx_frag(rxq->skb, | 2292 | skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags]; |
| 2313 | skb_shinfo(rxq->skb)->nr_frags, | 2293 | |
| 2314 | page, pp->rx_offset_correction, data_len, | 2294 | skb_frag_off_set(frag, pp->rx_offset_correction); |
| 2315 | PAGE_SIZE); | 2295 | skb_frag_size_set(frag, data_len); |
| 2316 | } | 2296 | __skb_frag_set_page(frag, page); |
| 2317 | page_pool_release_page(rxq->page_pool, page); | 2297 | sinfo->nr_frags++; |
| 2318 | rx_desc->buf_phys_addr = 0; | 2298 | |
| 2319 | rxq->left_size -= len; | 2299 | rx_desc->buf_phys_addr = 0; |
| 2300 | } | ||
| 2301 | *size -= len; | ||
| 2302 | } | ||
| 2303 | |||
| 2304 | static struct sk_buff * | ||
| 2305 | mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, | ||
| 2306 | struct xdp_buff *xdp, u32 desc_status) | ||
| 2307 | { | ||
| 2308 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | ||
| 2309 | int i, num_frags = sinfo->nr_frags; | ||
| 2310 | skb_frag_t frags[MAX_SKB_FRAGS]; | ||
| 2311 | struct sk_buff *skb; | ||
| 2312 | |||
| 2313 | memcpy(frags, sinfo->frags, sizeof(skb_frag_t) * num_frags); | ||
| 2314 | |||
| 2315 | skb = build_skb(xdp->data_hard_start, PAGE_SIZE); | ||
| 2316 | if (!skb) | ||
| 2317 | return ERR_PTR(-ENOMEM); | ||
| 2318 | |||
| 2319 | page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data)); | ||
| 2320 | |||
| 2321 | skb_reserve(skb, xdp->data - xdp->data_hard_start); | ||
| 2322 | skb_put(skb, xdp->data_end - xdp->data); | ||
| 2323 | mvneta_rx_csum(pp, desc_status, skb); | ||
| 2324 | |||
| 2325 | for (i = 0; i < num_frags; i++) { | ||
| 2326 | struct page *page = skb_frag_page(&frags[i]); | ||
| 2327 | |||
| 2328 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, | ||
| 2329 | page, skb_frag_off(&frags[i]), | ||
| 2330 | skb_frag_size(&frags[i]), PAGE_SIZE); | ||
| 2331 | page_pool_release_page(rxq->page_pool, page); | ||
| 2332 | } | ||
| 2333 | |||
| 2334 | return skb; | ||
| 2320 | } | 2335 | } |
| 2321 | 2336 | ||
| 2322 | /* Main rx processing when using software buffer management */ | 2337 | /* Main rx processing when using software buffer management */ |
| @@ -2324,24 +2339,27 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 1 unmodified line | |||
| 2324 | struct mvneta_port *pp, int budget, | 2339 | struct mvneta_port *pp, int budget, |
| 2325 | struct mvneta_rx_queue *rxq) | 2340 | struct mvneta_rx_queue *rxq) |
| 2326 | { | 2341 | { |
| 2327 | int rx_proc = 0, rx_todo, refill; | 2342 | int rx_proc = 0, rx_todo, refill, size = 0; |
| 2328 | struct net_device *dev = pp->dev; | 2343 | struct net_device *dev = pp->dev; |
| 2344 | struct xdp_buff xdp_buf = { | ||
| 2345 | .frame_sz = PAGE_SIZE, | ||
| 2346 | .rxq = &rxq->xdp_rxq, | ||
| 2347 | }; | ||
| 2329 | struct mvneta_stats ps = {}; | 2348 | struct mvneta_stats ps = {}; |
| 2330 | struct bpf_prog *xdp_prog; | 2349 | struct bpf_prog *xdp_prog; |
| 2331 | struct xdp_buff xdp_buf; | 2350 | u32 desc_status, frame_sz; |
| 2332 | 2351 | ||
| 2333 | /* Get number of received packets */ | 2352 | /* Get number of received packets */ |
| 2334 | rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); | 2353 | rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); |
| 2335 | 2354 | ||
| 2336 | rcu_read_lock(); | 2355 | rcu_read_lock(); |
| 2337 | xdp_prog = READ_ONCE(pp->xdp_prog); | 2356 | xdp_prog = READ_ONCE(pp->xdp_prog); |
| 2338 | xdp_buf.rxq = &rxq->xdp_rxq; | ||
| 2339 | xdp_buf.frame_sz = PAGE_SIZE; | ||
| 2340 | 2357 | ||
| 2341 | /* Fairness NAPI loop */ | 2358 | /* Fairness NAPI loop */ |
| 2342 | while (rx_proc < budget && rx_proc < rx_todo) { | 2359 | while (rx_proc < budget && rx_proc < rx_todo) { |
| 2343 | struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); | 2360 | struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); |
| 2344 | u32 rx_status, index; | 2361 | u32 rx_status, index; |
| 2362 | struct sk_buff *skb; | ||
| 2345 | struct page *page; | 2363 | struct page *page; |
| 2346 | 2364 | ||
| 2347 | index = rx_desc - rxq->descs; | 2365 | index = rx_desc - rxq->descs; |
| @@ -2352,54 +2370,66 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 4 unmodified lines | |||
| 2352 | rxq->refill_num++; | 2370 | rxq->refill_num++; |
| 2353 | 2371 | ||
| 2354 | if (rx_status & MVNETA_RXD_FIRST_DESC) { | 2372 | if (rx_status & MVNETA_RXD_FIRST_DESC) { |
| 2355 | int err; | ||
| 2356 | |||
| 2357 | /* Check errors only for FIRST descriptor */ | 2373 | /* Check errors only for FIRST descriptor */ |
| 2358 | if (rx_status & MVNETA_RXD_ERR_SUMMARY) { | 2374 | if (rx_status & MVNETA_RXD_ERR_SUMMARY) { |
| 2359 | mvneta_rx_error(pp, rx_desc); | 2375 | mvneta_rx_error(pp, rx_desc); |
| 2360 | /* leave the descriptor untouched */ | 2376 | goto next; |
| 2361 | continue; | ||
| 2362 | } | 2377 | } |
| 2363 | 2378 | ||
| 2364 | err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, | 2379 | size = rx_desc->data_size; |
| 2365 | xdp_prog, page, &ps); | 2380 | frame_sz = size - ETH_FCS_LEN; |
| 2366 | if (err) | 2381 | desc_status = rx_desc->status; |
| 2367 | continue; | 2382 | |
| 2383 | mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, | ||
| 2384 | &size, page, &ps); | ||
| 2368 | } else { | 2385 | } else { |
| 2369 | if (unlikely(!rxq->skb)) { | 2386 | if (unlikely(!xdp_buf.data_hard_start)) |
| 2370 | pr_debug("no skb for rx_status 0x%x\n", | ||
| 2371 | rx_status); | ||
| 2372 | continue; | 2387 | continue; |
| 2373 | } | 2388 | |
| 2374 | mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page); | 2389 | mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf, |
| 2390 | &size, page); | ||
| 2375 | } /* Middle or Last descriptor */ | 2391 | } /* Middle or Last descriptor */ |
| 2376 | 2392 | ||
| 2377 | if (!(rx_status & MVNETA_RXD_LAST_DESC)) | 2393 | if (!(rx_status & MVNETA_RXD_LAST_DESC)) |
| 2378 | /* no last descriptor this time */ | 2394 | /* no last descriptor this time */ |
| 2379 | continue; | 2395 | continue; |
| 2380 | 2396 | ||
| 2381 | if (rxq->left_size) { | 2397 | if (size) { |
| 2382 | pr_err("get last desc, but left_size (%d) != 0\n", | 2398 | mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); |
| 2383 | rxq->left_size); | 2399 | goto next; |
| 2384 | dev_kfree_skb_any(rxq->skb); | ||
| 2385 | rxq->left_size = 0; | ||
| 2386 | rxq->skb = NULL; | ||
| 2387 | continue; | ||
| 2388 | } | 2400 | } |
| 2389 | 2401 | ||
| 2390 | ps.rx_bytes += rxq->skb->len; | 2402 | if (xdp_prog && |
| 2391 | ps.rx_packets++; | 2403 | mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps)) |
| 2404 | goto next; | ||
| 2392 | 2405 | ||
| 2393 | /* Linux processing */ | 2406 | skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status); |
| 2394 | rxq->skb->protocol = eth_type_trans(rxq->skb, dev); | 2407 | if (IS_ERR(skb)) { |
| 2408 | struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); | ||
| 2409 | |||
| 2410 | mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); | ||
| 2395 | 2411 | ||
| 2396 | napi_gro_receive(napi, rxq->skb); | 2412 | u64_stats_update_begin(&stats->syncp); |
| 2413 | stats->es.skb_alloc_error++; | ||
| 2414 | stats->rx_dropped++; | ||
| 2415 | u64_stats_update_end(&stats->syncp); | ||
| 2397 | 2416 | ||
| 2398 | /* clean uncomplete skb pointer in queue */ | 2417 | goto next; |
| 2399 | rxq->skb = NULL; | 2418 | } |
| 2419 | |||
| 2420 | ps.rx_bytes += skb->len; | ||
| 2421 | ps.rx_packets++; | ||
| 2422 | |||
| 2423 | skb->protocol = eth_type_trans(skb, dev); | ||
| 2424 | napi_gro_receive(napi, skb); | ||
| 2425 | next: | ||
| 2426 | xdp_buf.data_hard_start = NULL; | ||
| 2400 | } | 2427 | } |
| 2401 | rcu_read_unlock(); | 2428 | rcu_read_unlock(); |
| 2402 | 2429 | ||
| 2430 | if (xdp_buf.data_hard_start) | ||
| 2431 | mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); | ||
| 2432 | |||
| 2403 | if (ps.xdp_redirect) | 2433 | if (ps.xdp_redirect) |
| 2404 | xdp_do_flush_map(); | 2434 | xdp_do_flush_map(); |
| 2405 | 2435 | ||
| @@ -3328,9 +3358,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 922 unmodified lines | |||
| 3328 | { | 3358 | { |
| 3329 | mvneta_rxq_drop_pkts(pp, rxq); | 3359 | mvneta_rxq_drop_pkts(pp, rxq); |
| 3330 | 3360 | ||
| 3331 | if (rxq->skb) | ||
| 3332 | dev_kfree_skb_any(rxq->skb); | ||
| 3333 | |||
| 3334 | if (rxq->descs) | 3361 | if (rxq->descs) |
| 3335 | dma_free_coherent(pp->dev->dev.parent, | 3362 | dma_free_coherent(pp->dev->dev.parent, |
| 3336 | rxq->size * MVNETA_DESC_ALIGNED_SIZE, | 3363 | rxq->size * MVNETA_DESC_ALIGNED_SIZE, |
| @@ -3343,8 +3370,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 6 unmodified lines | |||
| 3343 | rxq->descs_phys = 0; | 3370 | rxq->descs_phys = 0; |
| 3344 | rxq->first_to_refill = 0; | 3371 | rxq->first_to_refill = 0; |
| 3345 | rxq->refill_num = 0; | 3372 | rxq->refill_num = 0; |
| 3346 | rxq->skb = NULL; | ||
| 3347 | rxq->left_size = 0; | ||
| 3348 | } | 3373 | } |
| 3349 | 3374 | ||
| 3350 | static int mvneta_txq_sw_init(struct mvneta_port *pp, | 3375 | static int mvneta_txq_sw_init(struct mvneta_port *pp, |
| @@ -85,6 +85,12 @@ struct xdp_buff { · 84 unmodified lines | |||
| 85 | ((xdp)->data_hard_start + (xdp)->frame_sz - \ | 85 | ((xdp)->data_hard_start + (xdp)->frame_sz - \ |
| 86 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) | 86 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) |
| 87 | 87 | ||
| 88 | static inline struct skb_shared_info * | ||
| 89 | xdp_get_shared_info_from_buff(struct xdp_buff *xdp) | ||
| 90 | { | ||
| 91 | return (struct skb_shared_info *)xdp_data_hard_end(xdp); | ||
| 92 | } | ||
| 93 | |||
| 88 | struct xdp_frame { | 94 | struct xdp_frame { |
| 89 | void *data; | 95 | void *data; |
| 90 | u16 len; | 96 | u16 len; |
| @@ -98,6 +104,15 @@ struct xdp_frame { · 7 unmodified lines | |||
| 98 | struct net_device *dev_rx; /* used by cpumap */ | 104 | struct net_device *dev_rx; /* used by cpumap */ |
| 99 | }; | 105 | }; |
| 100 | 106 | ||
| 107 | static inline struct skb_shared_info * | ||
| 108 | xdp_get_shared_info_from_frame(struct xdp_frame *frame) | ||
| 109 | { | ||
| 110 | void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); | ||
| 111 | |||
| 112 | return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - | ||
| 113 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); | ||
| 114 | } | ||
| 115 | |||
| 101 | /* Clear kernel pointers in xdp_frame */ | 116 | /* Clear kernel pointers in xdp_frame */ |
| 102 | static inline void xdp_scrub_frame(struct xdp_frame *frame) | 117 | static inline void xdp_scrub_frame(struct xdp_frame *frame) |
| 103 | { | 118 | { |