History

bpf: Allow RCU-protected lookups to happen from bh context

XDP programs are called from a NAPI poll context, which means the RCU reference liveness is ensured by local_bh_disable(). Add rcu_read_lock_bh_held() as a condition to the RCU checks for map lookups so lockdep understands that the dereferences are safe from inside *either* an rcu_read_lock() section *or* a local_bh_disable() section. While both bh_disabled and rcu_read_lock() provide RCU protection, they are semantically distinct, so we need both conditions to prevent lockdep complaints. This change is done in preparation for removing the redundant rcu_read_lock()s from drivers.

Toke Høiland-Jørgensen committed 5 years ago

Commit 694cea31 parent e74c74f3 files · +21 −12Browse files

Unified · Split

3 files
  1. kernel/bpf/hashtab.c +14 −7
  2. kernel/bpf/helpers.c +3 −3
  3. kernel/bpf/lpm_trie.c +4 −2
kernel/bpf/hashtab.c+14−7
@@ -596,7 +596,8 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) · 595 unmodified lines
596 struct htab_elem *l;596 struct htab_elem *l;
597 u32 hash, key_size;597 u32 hash, key_size;
598 598
599 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());599 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
600 !rcu_read_lock_bh_held());
600 601
601 key_size = map->key_size;602 key_size = map->key_size;
602 603
@@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 386 unmodified lines
989 /* unknown flags */990 /* unknown flags */
990 return -EINVAL;991 return -EINVAL;
991 992
992 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());993 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
994 !rcu_read_lock_bh_held());
993 995
994 key_size = map->key_size;996 key_size = map->key_size;
995 997
@@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, · 86 unmodified lines
1082 /* unknown flags */1084 /* unknown flags */
1083 return -EINVAL;1085 return -EINVAL;
1084 1086
1085 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());1087 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
1088 !rcu_read_lock_bh_held());
1086 1089
1087 key_size = map->key_size;1090 key_size = map->key_size;
1088 1091
@@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 59 unmodified lines
1148 /* unknown flags */1151 /* unknown flags */
1149 return -EINVAL;1152 return -EINVAL;
1150 1153
1151 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());1154 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
1155 !rcu_read_lock_bh_held());
1152 1156
1153 key_size = map->key_size;1157 key_size = map->key_size;
1154 1158
@@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines
1202 /* unknown flags */1206 /* unknown flags */
1203 return -EINVAL;1207 return -EINVAL;
1204 1208
1205 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());1209 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
1210 !rcu_read_lock_bh_held());
1206 1211
1207 key_size = map->key_size;1212 key_size = map->key_size;
1208 1213
@@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key) · 67 unmodified lines
1276 u32 hash, key_size;1281 u32 hash, key_size;
1277 int ret;1282 int ret;
1278 1283
1279 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());1284 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
1285 !rcu_read_lock_bh_held());
1280 1286
1281 key_size = map->key_size;1287 key_size = map->key_size;
1282 1288
@@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) · 28 unmodified lines
1311 u32 hash, key_size;1317 u32 hash, key_size;
1312 int ret;1318 int ret;
1313 1319
1314 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());1320 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() &&
1321 !rcu_read_lock_bh_held());
1315 1322
1316 key_size = map->key_size;1323 key_size = map->key_size;
1317 1324
kernel/bpf/helpers.c+3−3
@@ -29,7 +29,7 @@ · 28 unmodified lines
29 */29 */
30BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)30BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
31{31{
32 WARN_ON_ONCE(!rcu_read_lock_held());32 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
33 return (unsigned long) map->ops->map_lookup_elem(map, key);33 return (unsigned long) map->ops->map_lookup_elem(map, key);
34}34}
35 35
@@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines
45BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,45BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
46 void *, value, u64, flags)46 void *, value, u64, flags)
47{47{
48 WARN_ON_ONCE(!rcu_read_lock_held());48 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
49 return map->ops->map_update_elem(map, key, value, flags);49 return map->ops->map_update_elem(map, key, value, flags);
50}50}
51 51
@@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines
62 62
63BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)63BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
64{64{
65 WARN_ON_ONCE(!rcu_read_lock_held());65 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
66 return map->ops->map_delete_elem(map, key);66 return map->ops->map_delete_elem(map, key);
67}67}
68 68
kernel/bpf/lpm_trie.c+4−2
@@ -232,7 +232,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 231 unmodified lines
232 232
233 /* Start walking the trie from the root node ... */233 /* Start walking the trie from the root node ... */
234 234
235 for (node = rcu_dereference(trie->root); node;) {235 for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
236 node;) {
236 unsigned int next_bit;237 unsigned int next_bit;
237 size_t matchlen;238 size_t matchlen;
238 239
@@ -264,7 +265,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 25 unmodified lines
264 * traverse down.265 * traverse down.
265 */266 */
266 next_bit = extract_bit(key->data, node->prefixlen);267 next_bit = extract_bit(key->data, node->prefixlen);
267 node = rcu_dereference(node->child[next_bit]);268 node = rcu_dereference_check(node->child[next_bit],
269 rcu_read_lock_bh_held());
268 }270 }
269 271
270 if (!found)272 if (!found)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t