Lorenzo Bianconi says: ==================== rework mvneta napi_poll loop for XDP multi-buffers Rework mvneta_rx_swbm routine in order to process all rx descriptors before building the skb or run the xdp program attached to the interface. Introduce xdp_get_shared_info_from_{buff,frame} utility routines to get the skb_shared_info pointer from xdp_buff or xdp_frame. This is a preliminary series to enable multi-buffers and jumbo frames for XDP according to [1] [1] https://github.com/xdp-project/xdp-project/blob/master/areas/core/xdp-multi-buffer01-design.org Changes since v1: - rely on skb_frag_* utility routines to access page/offset/len of the xdp multi-buffer ====================
Unified · Split
| @@ -698,10 +698,6 @@ struct mvneta_rx_queue { · 697 unmodified lines | ||
| 698 | 698 | /* Index of first RX DMA descriptor to refill */ |
| 699 | 699 | int first_to_refill; |
| 700 | 700 | u32 refill_num; |
| 701 | − | |
| 702 | − /* pointer to uncomplete skb buffer */ | |
| 703 | − struct sk_buff *skb; | |
| 704 | − int left_size; | |
| 705 | 701 | }; |
| 706 | 702 | |
| 707 | 703 | static enum cpuhp_state online_hpstate; |
| @@ -2026,6 +2022,20 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) · 1318 unmodified lines | ||
| 2026 | 2022 | return i; |
| 2027 | 2023 | } |
| 2028 | 2024 | |
| 2025 | +static void | |
| 2026 | +mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, | |
| 2027 | + struct xdp_buff *xdp, int sync_len, bool napi) | |
| 2028 | +{ | |
| 2029 | + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | |
| 2030 | + int i; | |
| 2031 | + | |
| 2032 | + page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data), | |
| 2033 | + sync_len, napi); | |
| 2034 | + for (i = 0; i < sinfo->nr_frags; i++) | |
| 2035 | + page_pool_put_full_page(rxq->page_pool, | |
| 2036 | + skb_frag_page(&sinfo->frags[i]), napi); | |
| 2037 | +} | |
| 2038 | + | |
| 2029 | 2039 | static int |
| 2030 | 2040 | mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq, |
| 2031 | 2041 | struct xdp_frame *xdpf, bool dma_map) |
| @@ -2158,13 +2168,13 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, · 126 unmodified lines | ||
| 2158 | 2168 | static int |
| 2159 | 2169 | mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, |
| 2160 | 2170 | struct bpf_prog *prog, struct xdp_buff *xdp, |
| 2161 | − struct mvneta_stats *stats) | |
| 2171 | + u32 frame_sz, struct mvneta_stats *stats) | |
| 2162 | 2172 | { |
| 2163 | − unsigned int len, sync; | |
| 2164 | − struct page *page; | |
| 2173 | + unsigned int len, data_len, sync; | |
| 2165 | 2174 | u32 ret, act; |
| 2166 | 2175 | |
| 2167 | 2176 | len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction; |
| 2177 | + data_len = xdp->data_end - xdp->data; | |
| 2168 | 2178 | act = bpf_prog_run_xdp(prog, xdp); |
| 2169 | 2179 | |
| 2170 | 2180 | /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */ |
| @@ -2180,9 +2190,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 9 unmodified lines | ||
| 2180 | 2190 | |
| 2181 | 2191 | err = xdp_do_redirect(pp->dev, xdp, prog); |
| 2182 | 2192 | if (unlikely(err)) { |
| 2193 | + mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); | |
| 2183 | 2194 | ret = MVNETA_XDP_DROPPED; |
| 2184 | − page = virt_to_head_page(xdp->data); | |
| 2185 | − page_pool_put_page(rxq->page_pool, page, sync, true); | |
| 2186 | 2195 | } else { |
| 2187 | 2196 | ret = MVNETA_XDP_REDIR; |
| 2188 | 2197 | stats->xdp_redirect++; |
| @@ -2191,10 +2200,8 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines | ||
| 2191 | 2200 | } |
| 2192 | 2201 | case XDP_TX: |
| 2193 | 2202 | ret = mvneta_xdp_xmit_back(pp, xdp); |
| 2194 | − if (ret != MVNETA_XDP_TX) { | |
| 2195 | − page = virt_to_head_page(xdp->data); | |
| 2196 | − page_pool_put_page(rxq->page_pool, page, sync, true); | |
| 2197 | − } | |
| 2203 | + if (ret != MVNETA_XDP_TX) | |
| 2204 | + mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); | |
| 2198 | 2205 | break; |
| 2199 | 2206 | default: |
| 2200 | 2207 | bpf_warn_invalid_xdp_action(act); |
| @@ -2203,25 +2210,23 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, · 2 unmodified lines | ||
| 2203 | 2210 | trace_xdp_exception(pp->dev, prog, act); |
| 2204 | 2211 | /* fall through */ |
| 2205 | 2212 | case XDP_DROP: |
| 2206 | − page = virt_to_head_page(xdp->data); | |
| 2207 | − page_pool_put_page(rxq->page_pool, page, sync, true); | |
| 2213 | + mvneta_xdp_put_buff(pp, rxq, xdp, sync, true); | |
| 2208 | 2214 | ret = MVNETA_XDP_DROPPED; |
| 2209 | 2215 | stats->xdp_drop++; |
| 2210 | 2216 | break; |
| 2211 | 2217 | } |
| 2212 | 2218 | |
| 2213 | − stats->rx_bytes += xdp->data_end - xdp->data; | |
| 2219 | + stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len; | |
| 2214 | 2220 | stats->rx_packets++; |
| 2215 | 2221 | |
| 2216 | 2222 | return ret; |
| 2217 | 2223 | } |
| 2218 | 2224 | |
| 2219 | −static int | |
| 2225 | +static void | |
| 2220 | 2226 | mvneta_swbm_rx_frame(struct mvneta_port *pp, |
| 2221 | 2227 | struct mvneta_rx_desc *rx_desc, |
| 2222 | 2228 | struct mvneta_rx_queue *rxq, |
| 2223 | − struct xdp_buff *xdp, | |
| 2224 | − struct bpf_prog *xdp_prog, | |
| 2229 | + struct xdp_buff *xdp, int *size, | |
| 2225 | 2230 | struct page *page, |
| 2226 | 2231 | struct mvneta_stats *stats) |
| 2227 | 2232 | { |
| @@ -2229,7 +2234,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 1 unmodified line | ||
| 2229 | 2234 | int data_len = -MVNETA_MH_SIZE, len; |
| 2230 | 2235 | struct net_device *dev = pp->dev; |
| 2231 | 2236 | enum dma_data_direction dma_dir; |
| 2232 | − int ret = 0; | |
| 2237 | + struct skb_shared_info *sinfo; | |
| 2233 | 2238 | |
| 2234 | 2239 | if (MVNETA_SKB_SIZE(rx_desc->data_size) > PAGE_SIZE) { |
| 2235 | 2240 | len = MVNETA_MAX_RX_BUF_SIZE; |
| @@ -2252,71 +2257,81 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, · 16 unmodified lines | ||
| 2252 | 2257 | xdp->data_end = xdp->data + data_len; |
| 2253 | 2258 | xdp_set_data_meta_invalid(xdp); |
| 2254 | 2259 | |
| 2255 | − if (xdp_prog) { | |
| 2256 | − ret = mvneta_run_xdp(pp, rxq, xdp_prog, xdp, stats); | |
| 2257 | − if (ret) | |
| 2258 | − goto out; | |
| 2259 | − } | |
| 2260 | − | |
| 2261 | − rxq->skb = build_skb(xdp->data_hard_start, PAGE_SIZE); | |
| 2262 | − if (unlikely(!rxq->skb)) { | |
| 2263 | − struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); | |
| 2260 | + sinfo = xdp_get_shared_info_from_buff(xdp); | |
| 2261 | + sinfo->nr_frags = 0; | |
| 2264 | 2262 | |
| 2265 | − netdev_err(dev, "Can't allocate skb on queue %d\n", rxq->id); | |
| 2266 | − | |
| 2267 | − u64_stats_update_begin(&stats->syncp); | |
| 2268 | − stats->es.skb_alloc_error++; | |
| 2269 | − stats->rx_dropped++; | |
| 2270 | − u64_stats_update_end(&stats->syncp); | |
| 2271 | − | |
| 2272 | − return -ENOMEM; | |
| 2273 | − } | |
| 2274 | − page_pool_release_page(rxq->page_pool, page); | |
| 2275 | − | |
| 2276 | − skb_reserve(rxq->skb, | |
| 2277 | − xdp->data - xdp->data_hard_start); | |
| 2278 | − skb_put(rxq->skb, xdp->data_end - xdp->data); | |
| 2279 | − mvneta_rx_csum(pp, rx_desc->status, rxq->skb); | |
| 2280 | − | |
| 2281 | − rxq->left_size = rx_desc->data_size - len; | |
| 2282 | − | |
| 2283 | −out: | |
| 2263 | + *size = rx_desc->data_size - len; | |
| 2284 | 2264 | rx_desc->buf_phys_addr = 0; |
| 2285 | − | |
| 2286 | − return ret; | |
| 2287 | 2265 | } |
| 2288 | 2266 | |
| 2289 | 2267 | static void |
| 2290 | 2268 | mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, |
| 2291 | 2269 | struct mvneta_rx_desc *rx_desc, |
| 2292 | 2270 | struct mvneta_rx_queue *rxq, |
| 2271 | + struct xdp_buff *xdp, int *size, | |
| 2293 | 2272 | struct page *page) |
| 2294 | 2273 | { |
| 2274 | + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | |
| 2295 | 2275 | struct net_device *dev = pp->dev; |
| 2296 | 2276 | enum dma_data_direction dma_dir; |
| 2297 | 2277 | int data_len, len; |
| 2298 | 2278 | |
| 2299 | − if (rxq->left_size > MVNETA_MAX_RX_BUF_SIZE) { | |
| 2279 | + if (*size > MVNETA_MAX_RX_BUF_SIZE) { | |
| 2300 | 2280 | len = MVNETA_MAX_RX_BUF_SIZE; |
| 2301 | 2281 | data_len = len; |
| 2302 | 2282 | } else { |
| 2303 | − len = rxq->left_size; | |
| 2283 | + len = *size; | |
| 2304 | 2284 | data_len = len - ETH_FCS_LEN; |
| 2305 | 2285 | } |
| 2306 | 2286 | dma_dir = page_pool_get_dma_dir(rxq->page_pool); |
| 2307 | 2287 | dma_sync_single_for_cpu(dev->dev.parent, |
| 2308 | 2288 | rx_desc->buf_phys_addr, |
| 2309 | 2289 | len, dma_dir); |
| 2310 | − if (data_len > 0) { | |
| 2311 | − /* refill descriptor with new buffer later */ | |
| 2312 | − skb_add_rx_frag(rxq->skb, | |
| 2313 | − skb_shinfo(rxq->skb)->nr_frags, | |
| 2314 | − page, pp->rx_offset_correction, data_len, | |
| 2315 | − PAGE_SIZE); | |
| 2316 | − } | |
| 2317 | − page_pool_release_page(rxq->page_pool, page); | |
| 2318 | − rx_desc->buf_phys_addr = 0; | |
| 2319 | − rxq->left_size -= len; | |
| 2290 | + | |
| 2291 | + if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) { | |
| 2292 | + skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags]; | |
| 2293 | + | |
| 2294 | + skb_frag_off_set(frag, pp->rx_offset_correction); | |
| 2295 | + skb_frag_size_set(frag, data_len); | |
| 2296 | + __skb_frag_set_page(frag, page); | |
| 2297 | + sinfo->nr_frags++; | |
| 2298 | + | |
| 2299 | + rx_desc->buf_phys_addr = 0; | |
| 2300 | + } | |
| 2301 | + *size -= len; | |
| 2302 | +} | |
| 2303 | + | |
| 2304 | +static struct sk_buff * | |
| 2305 | +mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, | |
| 2306 | + struct xdp_buff *xdp, u32 desc_status) | |
| 2307 | +{ | |
| 2308 | + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); | |
| 2309 | + int i, num_frags = sinfo->nr_frags; | |
| 2310 | + skb_frag_t frags[MAX_SKB_FRAGS]; | |
| 2311 | + struct sk_buff *skb; | |
| 2312 | + | |
| 2313 | + memcpy(frags, sinfo->frags, sizeof(skb_frag_t) * num_frags); | |
| 2314 | + | |
| 2315 | + skb = build_skb(xdp->data_hard_start, PAGE_SIZE); | |
| 2316 | + if (!skb) | |
| 2317 | + return ERR_PTR(-ENOMEM); | |
| 2318 | + | |
| 2319 | + page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data)); | |
| 2320 | + | |
| 2321 | + skb_reserve(skb, xdp->data - xdp->data_hard_start); | |
| 2322 | + skb_put(skb, xdp->data_end - xdp->data); | |
| 2323 | + mvneta_rx_csum(pp, desc_status, skb); | |
| 2324 | + | |
| 2325 | + for (i = 0; i < num_frags; i++) { | |
| 2326 | + struct page *page = skb_frag_page(&frags[i]); | |
| 2327 | + | |
| 2328 | + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, | |
| 2329 | + page, skb_frag_off(&frags[i]), | |
| 2330 | + skb_frag_size(&frags[i]), PAGE_SIZE); | |
| 2331 | + page_pool_release_page(rxq->page_pool, page); | |
| 2332 | + } | |
| 2333 | + | |
| 2334 | + return skb; | |
| 2320 | 2335 | } |
| 2321 | 2336 | |
| 2322 | 2337 | /* Main rx processing when using software buffer management */ |
| @@ -2324,24 +2339,27 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 1 unmodified line | ||
| 2324 | 2339 | struct mvneta_port *pp, int budget, |
| 2325 | 2340 | struct mvneta_rx_queue *rxq) |
| 2326 | 2341 | { |
| 2327 | − int rx_proc = 0, rx_todo, refill; | |
| 2342 | + int rx_proc = 0, rx_todo, refill, size = 0; | |
| 2328 | 2343 | struct net_device *dev = pp->dev; |
| 2344 | + struct xdp_buff xdp_buf = { | |
| 2345 | + .frame_sz = PAGE_SIZE, | |
| 2346 | + .rxq = &rxq->xdp_rxq, | |
| 2347 | + }; | |
| 2329 | 2348 | struct mvneta_stats ps = {}; |
| 2330 | 2349 | struct bpf_prog *xdp_prog; |
| 2331 | − struct xdp_buff xdp_buf; | |
| 2350 | + u32 desc_status, frame_sz; | |
| 2332 | 2351 | |
| 2333 | 2352 | /* Get number of received packets */ |
| 2334 | 2353 | rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); |
| 2335 | 2354 | |
| 2336 | 2355 | rcu_read_lock(); |
| 2337 | 2356 | xdp_prog = READ_ONCE(pp->xdp_prog); |
| 2338 | − xdp_buf.rxq = &rxq->xdp_rxq; | |
| 2339 | − xdp_buf.frame_sz = PAGE_SIZE; | |
| 2340 | 2357 | |
| 2341 | 2358 | /* Fairness NAPI loop */ |
| 2342 | 2359 | while (rx_proc < budget && rx_proc < rx_todo) { |
| 2343 | 2360 | struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); |
| 2344 | 2361 | u32 rx_status, index; |
| 2362 | + struct sk_buff *skb; | |
| 2345 | 2363 | struct page *page; |
| 2346 | 2364 | |
| 2347 | 2365 | index = rx_desc - rxq->descs; |
| @@ -2352,54 +2370,66 @@ static int mvneta_rx_swbm(struct napi_struct *napi, · 4 unmodified lines | ||
| 2352 | 2370 | rxq->refill_num++; |
| 2353 | 2371 | |
| 2354 | 2372 | if (rx_status & MVNETA_RXD_FIRST_DESC) { |
| 2355 | − int err; | |
| 2356 | − | |
| 2357 | 2373 | /* Check errors only for FIRST descriptor */ |
| 2358 | 2374 | if (rx_status & MVNETA_RXD_ERR_SUMMARY) { |
| 2359 | 2375 | mvneta_rx_error(pp, rx_desc); |
| 2360 | − /* leave the descriptor untouched */ | |
| 2361 | − continue; | |
| 2376 | + goto next; | |
| 2362 | 2377 | } |
| 2363 | 2378 | |
| 2364 | − err = mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, | |
| 2365 | − xdp_prog, page, &ps); | |
| 2366 | − if (err) | |
| 2367 | − continue; | |
| 2379 | + size = rx_desc->data_size; | |
| 2380 | + frame_sz = size - ETH_FCS_LEN; | |
| 2381 | + desc_status = rx_desc->status; | |
| 2382 | + | |
| 2383 | + mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf, | |
| 2384 | + &size, page, &ps); | |
| 2368 | 2385 | } else { |
| 2369 | − if (unlikely(!rxq->skb)) { | |
| 2370 | − pr_debug("no skb for rx_status 0x%x\n", | |
| 2371 | − rx_status); | |
| 2386 | + if (unlikely(!xdp_buf.data_hard_start)) | |
| 2372 | 2387 | continue; |
| 2373 | − } | |
| 2374 | − mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, page); | |
| 2388 | + | |
| 2389 | + mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf, | |
| 2390 | + &size, page); | |
| 2375 | 2391 | } /* Middle or Last descriptor */ |
| 2376 | 2392 | |
| 2377 | 2393 | if (!(rx_status & MVNETA_RXD_LAST_DESC)) |
| 2378 | 2394 | /* no last descriptor this time */ |
| 2379 | 2395 | continue; |
| 2380 | 2396 | |
| 2381 | − if (rxq->left_size) { | |
| 2382 | − pr_err("get last desc, but left_size (%d) != 0\n", | |
| 2383 | − rxq->left_size); | |
| 2384 | − dev_kfree_skb_any(rxq->skb); | |
| 2385 | − rxq->left_size = 0; | |
| 2386 | − rxq->skb = NULL; | |
| 2387 | − continue; | |
| 2397 | + if (size) { | |
| 2398 | + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); | |
| 2399 | + goto next; | |
| 2388 | 2400 | } |
| 2389 | 2401 | |
| 2390 | − ps.rx_bytes += rxq->skb->len; | |
| 2391 | − ps.rx_packets++; | |
| 2402 | + if (xdp_prog && | |
| 2403 | + mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps)) | |
| 2404 | + goto next; | |
| 2392 | 2405 | |
| 2393 | − /* Linux processing */ | |
| 2394 | − rxq->skb->protocol = eth_type_trans(rxq->skb, dev); | |
| 2406 | + skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status); | |
| 2407 | + if (IS_ERR(skb)) { | |
| 2408 | + struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); | |
| 2409 | + | |
| 2410 | + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); | |
| 2395 | 2411 | |
| 2396 | − napi_gro_receive(napi, rxq->skb); | |
| 2412 | + u64_stats_update_begin(&stats->syncp); | |
| 2413 | + stats->es.skb_alloc_error++; | |
| 2414 | + stats->rx_dropped++; | |
| 2415 | + u64_stats_update_end(&stats->syncp); | |
| 2397 | 2416 | |
| 2398 | − /* clean uncomplete skb pointer in queue */ | |
| 2399 | − rxq->skb = NULL; | |
| 2417 | + goto next; | |
| 2418 | + } | |
| 2419 | + | |
| 2420 | + ps.rx_bytes += skb->len; | |
| 2421 | + ps.rx_packets++; | |
| 2422 | + | |
| 2423 | + skb->protocol = eth_type_trans(skb, dev); | |
| 2424 | + napi_gro_receive(napi, skb); | |
| 2425 | +next: | |
| 2426 | + xdp_buf.data_hard_start = NULL; | |
| 2400 | 2427 | } |
| 2401 | 2428 | rcu_read_unlock(); |
| 2402 | 2429 | |
| 2430 | + if (xdp_buf.data_hard_start) | |
| 2431 | + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1, true); | |
| 2432 | + | |
| 2403 | 2433 | if (ps.xdp_redirect) |
| 2404 | 2434 | xdp_do_flush_map(); |
| 2405 | 2435 | |
| @@ -3328,9 +3358,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 922 unmodified lines | ||
| 3328 | 3358 | { |
| 3329 | 3359 | mvneta_rxq_drop_pkts(pp, rxq); |
| 3330 | 3360 | |
| 3331 | − if (rxq->skb) | |
| 3332 | − dev_kfree_skb_any(rxq->skb); | |
| 3333 | − | |
| 3334 | 3361 | if (rxq->descs) |
| 3335 | 3362 | dma_free_coherent(pp->dev->dev.parent, |
| 3336 | 3363 | rxq->size * MVNETA_DESC_ALIGNED_SIZE, |
| @@ -3343,8 +3370,6 @@ static void mvneta_rxq_deinit(struct mvneta_port *pp, · 6 unmodified lines | ||
| 3343 | 3370 | rxq->descs_phys = 0; |
| 3344 | 3371 | rxq->first_to_refill = 0; |
| 3345 | 3372 | rxq->refill_num = 0; |
| 3346 | − rxq->skb = NULL; | |
| 3347 | − rxq->left_size = 0; | |
| 3348 | 3373 | } |
| 3349 | 3374 | |
| 3350 | 3375 | static int mvneta_txq_sw_init(struct mvneta_port *pp, |
| @@ -85,6 +85,12 @@ struct xdp_buff { · 84 unmodified lines | ||
| 85 | 85 | ((xdp)->data_hard_start + (xdp)->frame_sz - \ |
| 86 | 86 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) |
| 87 | 87 | |
| 88 | +static inline struct skb_shared_info * | |
| 89 | +xdp_get_shared_info_from_buff(struct xdp_buff *xdp) | |
| 90 | +{ | |
| 91 | + return (struct skb_shared_info *)xdp_data_hard_end(xdp); | |
| 92 | +} | |
| 93 | + | |
| 88 | 94 | struct xdp_frame { |
| 89 | 95 | void *data; |
| 90 | 96 | u16 len; |
| @@ -98,6 +104,15 @@ struct xdp_frame { · 7 unmodified lines | ||
| 98 | 104 | struct net_device *dev_rx; /* used by cpumap */ |
| 99 | 105 | }; |
| 100 | 106 | |
| 107 | +static inline struct skb_shared_info * | |
| 108 | +xdp_get_shared_info_from_frame(struct xdp_frame *frame) | |
| 109 | +{ | |
| 110 | + void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); | |
| 111 | + | |
| 112 | + return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - | |
| 113 | + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); | |
| 114 | +} | |
| 115 | + | |
| 101 | 116 | /* Clear kernel pointers in xdp_frame */ |
| 102 | 117 | static inline void xdp_scrub_frame(struct xdp_frame *frame) |
| 103 | 118 | { |