xdp: Make cpumap flush_list common for all map instances

The cpumap flush list is used to track entries that need to flushed from via the xdp_do_flush_map() function. This list used to be per-map, but there is really no reason for that. Instead make the flush list global for all devmaps, which simplifies __cpu_map_flush() and cpu_map_alloc().

Björn Töpel committed 6 years ago

Commit cdfafe91 parent 96360003 files · +21 −21Browse files

Unified · Split

3 files

include/linux/bpf.h +2 −2
kernel/bpf/cpumap.c +18 −18
net/core/filter.c +1 −1

include/linux/bpf.h+2−2

@@ -966,7 +966,7 @@ int dev_map_generic_redirect(struct bpf_dtab_netdev dst, struct sk_buff skb, · 965 unmodified lines
966	966	struct bpf_prog *xdp_prog);
967	967
968	968	struct bpf_cpu_map_entry __cpu_map_lookup_elem(struct bpf_map map, u32 key);
969		−void __cpu_map_flush(struct bpf_map *map);
	969	+void __cpu_map_flush(void);
970	970	int cpu_map_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_buff xdp,
971	971	struct net_device *dev_rx);
972	972
@@ -1097,7 +1097,7 @@ struct bpf_cpu_map_entry __cpu_map_lookup_elem(struct bpf_map map, u32 key) · 124 unmodified lines
1097	1097	return NULL;
1098	1098	}
1099	1099
1100		−static inline void __cpu_map_flush(struct bpf_map *map)
	1100	+static inline void __cpu_map_flush(void)
1101	1101	{
1102	1102	}
1103	1103

kernel/bpf/cpumap.c+18−18

@@ -72,17 +72,18 @@ struct bpf_cpu_map { · 71 unmodified lines
72	72	struct bpf_map map;
73	73	/* Below members specific for map type */
74	74	struct bpf_cpu_map_entry **cpu_map;
75		− struct list_head __percpu *flush_list;
76	75	};
77	76
	77	+static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
	78	+
78	79	static int bq_flush_to_queue(struct xdp_bulk_queue *bq);
79	80
80	81	static struct bpf_map cpu_map_alloc(union bpf_attr attr)
81	82	{
82	83	struct bpf_cpu_map *cmap;
83	84	int err = -ENOMEM;
84		− int ret, cpu;
85	85	u64 cost;
	86	+ int ret;
86	87
87	88	if (!capable(CAP_SYS_ADMIN))
88	89	return ERR_PTR(-EPERM);
@@ -106,7 +107,6 @@ static struct bpf_map cpu_map_alloc(union bpf_attr attr) · 17 unmodified lines
106	107
107	108	/* make sure page count doesn't overflow */
108	109	cost = (u64) cmap->map.max_entries * sizeof(struct bpf_cpu_map_entry *);
109		− cost += sizeof(struct list_head) * num_possible_cpus();
110	110
111	111	/* Notice returns -EPERM on if map size is larger than memlock limit */
112	112	ret = bpf_map_charge_init(&cmap->map.memory, cost);
@@ -115,23 +115,14 @@ static struct bpf_map cpu_map_alloc(union bpf_attr attr) · 2 unmodified lines
115	115	goto free_cmap;
116	116	}
117	117
118		− cmap->flush_list = alloc_percpu(struct list_head);
119		− if (!cmap->flush_list)
120		− goto free_charge;
121		−
122		− for_each_possible_cpu(cpu)
123		− INIT_LIST_HEAD(per_cpu_ptr(cmap->flush_list, cpu));
124		−
125	118	/* Alloc array for possible remote "destination" CPUs */
126	119	cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
127	120	sizeof(struct bpf_cpu_map_entry *),
128	121	cmap->map.numa_node);
129	122	if (!cmap->cpu_map)
130		− goto free_percpu;
	123	+ goto free_charge;
131	124
132	125	return &cmap->map;
133		−free_percpu:
134		− free_percpu(cmap->flush_list);
135	126	free_charge:
136	127	bpf_map_charge_finish(&cmap->map.memory);
137	128	free_cmap:
@@ -526,7 +517,6 @@ static void cpu_map_free(struct bpf_map *map) · 388 unmodified lines
526	517	/* bq flush and cleanup happens after RCU grace-period */
527	518	__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
528	519	}
529		− free_percpu(cmap->flush_list);
530	520	bpf_map_area_free(cmap->cpu_map);
531	521	kfree(cmap);
532	522	}
@@ -618,7 +608,7 @@ static int bq_flush_to_queue(struct xdp_bulk_queue *bq) · 85 unmodified lines
618	608	*/
619	609	static int bq_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_frame xdpf)
620	610	{
621		− struct list_head *flush_list = this_cpu_ptr(rcpu->cmap->flush_list);
	611	+ struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
622	612	struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
623	613
624	614	if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
@@ -657,10 +647,9 @@ int cpu_map_enqueue(struct bpf_cpu_map_entry rcpu, struct xdp_buff xdp, · 32 unmodified lines
657	647	return 0;
658	648	}
659	649
660		−void __cpu_map_flush(struct bpf_map *map)
	650	+void __cpu_map_flush(void)
661	651	{
662		− struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map);
663		− struct list_head *flush_list = this_cpu_ptr(cmap->flush_list);
	652	+ struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
664	653	struct xdp_bulk_queue bq, tmp;
665	654
666	655	list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
@@ -670,3 +659,14 @@ void __cpu_map_flush(struct bpf_map *map) · 3 unmodified lines
670	659	wake_up_process(bq->obj->kthread);
671	660	}
672	661	}
	662	+
	663	+static int __init cpu_map_init(void)
	664	+{
	665	+ int cpu;
	666	+
	667	+ for_each_possible_cpu(cpu)
	668	+ INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
	669	+ return 0;
	670	+}
	671	+
	672	+subsys_initcall(cpu_map_init);

net/core/filter.c+1−1

@@ -3558,7 +3558,7 @@ void xdp_do_flush_map(void) · 3557 unmodified lines
3558	3558	__dev_map_flush();
3559	3559	break;
3560	3560	case BPF_MAP_TYPE_CPUMAP:
3561		− __cpu_map_flush(map);
	3561	+ __cpu_map_flush();
3562	3562	break;
3563	3563	case BPF_MAP_TYPE_XSKMAP:
3564	3564	__xsk_map_flush();