Now that all XDP maps that can be used with bpf_redirect_map() tracks entries to be flushed in a global fashion, there is not need to track that the map has changed and flush from xdp_do_generic_map() anymore. All entries will be flushed in xdp_do_flush_map(). This means that the map_to_flush can be removed, and the corresponding checks. Moving the flush logic to one place, xdp_do_flush_map(), give a bulking behavior and performance boost.
Unified · Split
| @@ -592,7 +592,6 @@ struct bpf_redirect_info { · 591 unmodified lines | |||
| 592 | u32 tgt_index; | 592 | u32 tgt_index; |
| 593 | void *tgt_value; | 593 | void *tgt_value; |
| 594 | struct bpf_map *map; | 594 | struct bpf_map *map; |
| 595 | struct bpf_map *map_to_flush; | ||
| 596 | u32 kern_flags; | 595 | u32 kern_flags; |
| 597 | }; | 596 | }; |
| 598 | 597 | ||
| @@ -3547,26 +3547,9 @@ static int __bpf_tx_xdp_map(struct net_device *dev_rx, void *fwd, · 3546 unmodified lines | |||
| 3547 | 3547 | ||
| 3548 | void xdp_do_flush_map(void) | 3548 | void xdp_do_flush_map(void) |
| 3549 | { | 3549 | { |
| 3550 | struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); | 3550 | __dev_map_flush(); |
| 3551 | struct bpf_map *map = ri->map_to_flush; | 3551 | __cpu_map_flush(); |
| 3552 | 3552 | __xsk_map_flush(); | |
| 3553 | ri->map_to_flush = NULL; | ||
| 3554 | if (map) { | ||
| 3555 | switch (map->map_type) { | ||
| 3556 | case BPF_MAP_TYPE_DEVMAP: | ||
| 3557 | case BPF_MAP_TYPE_DEVMAP_HASH: | ||
| 3558 | __dev_map_flush(); | ||
| 3559 | break; | ||
| 3560 | case BPF_MAP_TYPE_CPUMAP: | ||
| 3561 | __cpu_map_flush(); | ||
| 3562 | break; | ||
| 3563 | case BPF_MAP_TYPE_XSKMAP: | ||
| 3564 | __xsk_map_flush(); | ||
| 3565 | break; | ||
| 3566 | default: | ||
| 3567 | break; | ||
| 3568 | } | ||
| 3569 | } | ||
| 3570 | } | 3553 | } |
| 3571 | EXPORT_SYMBOL_GPL(xdp_do_flush_map); | 3554 | EXPORT_SYMBOL_GPL(xdp_do_flush_map); |
| 3572 | 3555 | ||
| @@ -3615,14 +3598,10 @@ static int xdp_do_redirect_map(struct net_device *dev, struct xdp_buff *xdp, · 42 unmodified lines | |||
| 3615 | ri->tgt_value = NULL; | 3598 | ri->tgt_value = NULL; |
| 3616 | WRITE_ONCE(ri->map, NULL); | 3599 | WRITE_ONCE(ri->map, NULL); |
| 3617 | 3600 | ||
| 3618 | if (ri->map_to_flush && unlikely(ri->map_to_flush != map)) | ||
| 3619 | xdp_do_flush_map(); | ||
| 3620 | |||
| 3621 | err = __bpf_tx_xdp_map(dev, fwd, map, xdp); | 3601 | err = __bpf_tx_xdp_map(dev, fwd, map, xdp); |
| 3622 | if (unlikely(err)) | 3602 | if (unlikely(err)) |
| 3623 | goto err; | 3603 | goto err; |
| 3624 | 3604 | ||
| 3625 | ri->map_to_flush = map; | ||
| 3626 | _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index); | 3605 | _trace_xdp_redirect_map(dev, xdp_prog, fwd, map, index); |
| 3627 | return 0; | 3606 | return 0; |
| 3628 | err: | 3607 | err: |