bpf: Allow RCU-protected lookups to happen from bh context

XDP programs are called from a NAPI poll context, which means the RCU reference liveness is ensured by local_bh_disable(). Add rcu_read_lock_bh_held() as a condition to the RCU checks for map lookups so lockdep understands that the dereferences are safe from inside *either* an rcu_read_lock() section *or* a local_bh_disable() section. While both bh_disabled and rcu_read_lock() provide RCU protection, they are semantically distinct, so we need both conditions to prevent lockdep complaints. This change is done in preparation for removing the redundant rcu_read_lock()s from drivers.

Toke Høiland-Jørgensen committed 5 years ago

Commit 694cea31 parent e74c74f3 files · +21 −12Browse files

Unified · Split

3 files

kernel/bpf/hashtab.c +14 −7
kernel/bpf/helpers.c +3 −3
kernel/bpf/lpm_trie.c +4 −2

kernel/bpf/hashtab.c+14−7

@@ -596,7 +596,8 @@ static void __htab_map_lookup_elem(struct bpf_map map, void *key) · 595 unmodified lines
596	596	struct htab_elem *l;
597	597	u32 hash, key_size;
598	598
599		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	599	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	600	+ !rcu_read_lock_bh_held());
600	601
601	602	key_size = map->key_size;
602	603
@@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map map, void key, void *value, · 386 unmodified lines
989	990	/* unknown flags */
990	991	return -EINVAL;
991	992
992		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	993	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	994	+ !rcu_read_lock_bh_held());
993	995
994	996	key_size = map->key_size;
995	997
@@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map map, void key, void *value, · 86 unmodified lines
1082	1084	/* unknown flags */
1083	1085	return -EINVAL;
1084	1086
1085		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1087	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1088	+ !rcu_read_lock_bh_held());
1086	1089
1087	1090	key_size = map->key_size;
1088	1091
@@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map map, void key, · 59 unmodified lines
1148	1151	/* unknown flags */
1149	1152	return -EINVAL;
1150	1153
1151		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1154	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1155	+ !rcu_read_lock_bh_held());
1152	1156
1153	1157	key_size = map->key_size;
1154	1158
@@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map map, void key, · 47 unmodified lines
1202	1206	/* unknown flags */
1203	1207	return -EINVAL;
1204	1208
1205		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1209	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1210	+ !rcu_read_lock_bh_held());
1206	1211
1207	1212	key_size = map->key_size;
1208	1213
@@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map map, void key) · 67 unmodified lines
1276	1281	u32 hash, key_size;
1277	1282	int ret;
1278	1283
1279		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1284	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1285	+ !rcu_read_lock_bh_held());
1280	1286
1281	1287	key_size = map->key_size;
1282	1288
@@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map map, void key) · 28 unmodified lines
1311	1317	u32 hash, key_size;
1312	1318	int ret;
1313	1319
1314		− WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
	1320	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
	1321	+ !rcu_read_lock_bh_held());
1315	1322
1316	1323	key_size = map->key_size;
1317	1324

kernel/bpf/helpers.c+3−3

@@ -29,7 +29,7 @@ · 28 unmodified lines
29	29	*/
30	30	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)
31	31	{
32		− WARN_ON_ONCE(!rcu_read_lock_held());
	32	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
33	33	return (unsigned long) map->ops->map_lookup_elem(map, key);
34	34	}
35	35
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines
45	45	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,
46	46	void *, value, u64, flags)
47	47	{
48		− WARN_ON_ONCE(!rcu_read_lock_held());
	48	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
49	49	return map->ops->map_update_elem(map, key, value, flags);
50	50	}
51	51
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines
62	62
63	63	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)
64	64	{
65		− WARN_ON_ONCE(!rcu_read_lock_held());
	65	+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
66	66	return map->ops->map_delete_elem(map, key);
67	67	}
68	68

kernel/bpf/lpm_trie.c+4−2

@@ -232,7 +232,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 231 unmodified lines
232	232
233	233	/* Start walking the trie from the root node ... */
234	234
235		− for (node = rcu_dereference(trie->root); node;) {
	235	+ for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
	236	+ node;) {
236	237	unsigned int next_bit;
237	238	size_t matchlen;
238	239
@@ -264,7 +265,8 @@ static void trie_lookup_elem(struct bpf_map map, void *_key) · 25 unmodified lines
264	265	* traverse down.
265	266	*/
266	267	next_bit = extract_bit(key->data, node->prefixlen);
267		− node = rcu_dereference(node->child[next_bit]);
	268	+ node = rcu_dereference_check(node->child[next_bit],
	269	+ rcu_read_lock_bh_held());
268	270	}
269	271
270	272	if (!found)

bpf: Allow RCU-protected lookups to happen from bh context

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code