History

xdp: Make cpumap flush_list common for all map instances

The cpumap flush list is used to track entries that need to flushed from via the xdp_do_flush_map() function. This list used to be per-map, but there is really no reason for that. Instead make the flush list global for all devmaps, which simplifies __cpu_map_flush() and cpu_map_alloc().

Björn Töpel committed 6 years ago

Commit cdfafe91 parent 96360003 files · +21 −21Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +2 −2
  2. kernel/bpf/cpumap.c +18 −18
  3. net/core/filter.c +1 −1
include/linux/bpf.h+2−2
@@ -966,7 +966,7 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, · 965 unmodified lines
966 struct bpf_prog *xdp_prog);966 struct bpf_prog *xdp_prog);
967 967
968struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);968struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key);
969void __cpu_map_flush(struct bpf_map *map);969void __cpu_map_flush(void);
970int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,970int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp,
971 struct net_device *dev_rx);971 struct net_device *dev_rx);
972 972
@@ -1097,7 +1097,7 @@ struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) · 124 unmodified lines
1097 return NULL;1097 return NULL;
1098}1098}
1099 1099
1100static inline void __cpu_map_flush(struct bpf_map *map)1100static inline void __cpu_map_flush(void)
1101{1101{
1102}1102}
1103 1103
kernel/bpf/cpumap.c+18−18
@@ -72,17 +72,18 @@ struct bpf_cpu_map { · 71 unmodified lines
72 struct bpf_map map;72 struct bpf_map map;
73 /* Below members specific for map type */73 /* Below members specific for map type */
74 struct bpf_cpu_map_entry **cpu_map;74 struct bpf_cpu_map_entry **cpu_map;
75 struct list_head __percpu *flush_list;
76};75};
77 76
77static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
78
78static int bq_flush_to_queue(struct xdp_bulk_queue *bq);79static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
79 80
80static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)81static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
81{82{
82 struct bpf_cpu_map *cmap;83 struct bpf_cpu_map *cmap;
83 int err = -ENOMEM;84 int err = -ENOMEM;
84 int ret, cpu;
85 u64 cost;85 u64 cost;
86 int ret;
86 87
87 if (!capable(CAP_SYS_ADMIN))88 if (!capable(CAP_SYS_ADMIN))
88 return ERR_PTR(-EPERM);89 return ERR_PTR(-EPERM);
@@ -106,7 +107,6 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) · 17 unmodified lines
106 107
107 /* make sure page count doesn't overflow */108 /* make sure page count doesn't overflow */
108 cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);109 cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
109 cost += sizeof(struct list_head) * num_possible_cpus();
110 110
111 /* Notice returns -EPERM on if map size is larger than memlock limit */111 /* Notice returns -EPERM on if map size is larger than memlock limit */
112 ret = bpf_map_charge_init(&cmap->map.memory, cost);112 ret = bpf_map_charge_init(&cmap->map.memory, cost);
@@ -115,23 +115,14 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) · 2 unmodified lines
115 goto free_cmap;115 goto free_cmap;
116 }116 }
117 117
118 cmap->flush_list = alloc_percpu(struct list_head);
119 if (!cmap->flush_list)
120 goto free_charge;
121
122 for_each_possible_cpu(cpu)
123 INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));
124
125 /* Alloc array for possible remote "destination" CPUs */118 /* Alloc array for possible remote "destination" CPUs */
126 cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *119 cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
127 sizeof(struct bpf_cpu_map_entry *),120 sizeof(struct bpf_cpu_map_entry *),
128 cmap->map.numa_node);121 cmap->map.numa_node);
129 if (!cmap->cpu_map)122 if (!cmap->cpu_map)
130 goto free_percpu;123 goto free_charge;
131 124
132 return &cmap->map;125 return &cmap->map;
133free_percpu:
134 free_percpu(cmap->flush_list);
135free_charge:126free_charge:
136 bpf_map_charge_finish(&cmap->map.memory);127 bpf_map_charge_finish(&cmap->map.memory);
137free_cmap:128free_cmap:
@@ -526,7 +517,6 @@ static void cpu_map_free(struct bpf_map *map) · 388 unmodified lines
526 /* bq flush and cleanup happens after RCU grace-period */517 /* bq flush and cleanup happens after RCU grace-period */
527 __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */518 __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
528 }519 }
529 free_percpu(cmap->flush_list);
530 bpf_map_area_free(cmap->cpu_map);520 bpf_map_area_free(cmap->cpu_map);
531 kfree(cmap);521 kfree(cmap);
532}522}
@@ -618,7 +608,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq) · 85 unmodified lines
618 */608 */
619static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)609static int bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
620{610{
621 struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list);611 struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
622 struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);612 struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
623 613
624 if (unlikely(bq->count == CPU_MAP_BULK_SIZE))614 if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
@@ -657,10 +647,9 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, · 32 unmodified lines
657 return 0;647 return 0;
658}648}
659 649
660void __cpu_map_flush(struct bpf_map *map)650void __cpu_map_flush(void)
661{651{
662 struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);652 struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
663 struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);
664 struct xdp_bulk_queue *bq, *tmp;653 struct xdp_bulk_queue *bq, *tmp;
665 654
666 list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {655 list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
@@ -670,3 +659,14 @@ void __cpu_map_flush(struct bpf_map *map) · 3 unmodified lines
670 wake_up_process(bq->obj->kthread);659 wake_up_process(bq->obj->kthread);
671 }660 }
672}661}
662
663static int __init cpu_map_init(void)
664{
665 int cpu;
666
667 for_each_possible_cpu(cpu)
668 INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
669 return 0;
670}
671
672subsys_initcall(cpu_map_init);
net/core/filter.c+1−1
@@ -3558,7 +3558,7 @@ void xdp_do_flush_map(void) · 3557 unmodified lines
3558 __dev_map_flush();3558 __dev_map_flush();
3559 break;3559 break;
3560 case BPF_MAP_TYPE_CPUMAP:3560 case BPF_MAP_TYPE_CPUMAP:
3561 __cpu_map_flush(map);3561 __cpu_map_flush();
3562 break;3562 break;
3563 case BPF_MAP_TYPE_XSKMAP:3563 case BPF_MAP_TYPE_XSKMAP:
3564 __xsk_map_flush();3564 __xsk_map_flush();