XDP programs are called from a NAPI poll context, which means the RCU reference liveness is ensured by local_bh_disable(). Add rcu_read_lock_bh_held() as a condition to the RCU checks for map lookups so lockdep understands that the dereferences are safe from inside *either* an rcu_read_lock() section *or* a local_bh_disable() section. While both bh_disabled and rcu_read_lock() provide RCU protection, they are semantically distinct, so we need both conditions to prevent lockdep complaints. This change is done in preparation for removing the redundant rcu_read_lock()s from drivers.
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -596,7 +596,8 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) · 595 unmodified lines | ||
| 596 | 596 | struct htab_elem *l; |
| 597 | 597 | u32 hash, key_size; |
| 598 | 598 | |
| 599 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 599 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 600 | + !rcu_read_lock_bh_held()); | |
| 600 | 601 | |
| 601 | 602 | key_size = map->key_size; |
| 602 | 603 | |
| @@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 386 unmodified lines | ||
| 989 | 990 | /* unknown flags */ |
| 990 | 991 | return -EINVAL; |
| 991 | 992 | |
| 992 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 993 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 994 | + !rcu_read_lock_bh_held()); | |
| 993 | 995 | |
| 994 | 996 | key_size = map->key_size; |
| 995 | 997 | |
| @@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, · 86 unmodified lines | ||
| 1082 | 1084 | /* unknown flags */ |
| 1083 | 1085 | return -EINVAL; |
| 1084 | 1086 | |
| 1085 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 1087 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 1088 | + !rcu_read_lock_bh_held()); | |
| 1086 | 1089 | |
| 1087 | 1090 | key_size = map->key_size; |
| 1088 | 1091 | |
| @@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 59 unmodified lines | ||
| 1148 | 1151 | /* unknown flags */ |
| 1149 | 1152 | return -EINVAL; |
| 1150 | 1153 | |
| 1151 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 1154 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 1155 | + !rcu_read_lock_bh_held()); | |
| 1152 | 1156 | |
| 1153 | 1157 | key_size = map->key_size; |
| 1154 | 1158 | |
| @@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines | ||
| 1202 | 1206 | /* unknown flags */ |
| 1203 | 1207 | return -EINVAL; |
| 1204 | 1208 | |
| 1205 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 1209 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 1210 | + !rcu_read_lock_bh_held()); | |
| 1206 | 1211 | |
| 1207 | 1212 | key_size = map->key_size; |
| 1208 | 1213 | |
| @@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key) · 67 unmodified lines | ||
| 1276 | 1281 | u32 hash, key_size; |
| 1277 | 1282 | int ret; |
| 1278 | 1283 | |
| 1279 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 1284 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 1285 | + !rcu_read_lock_bh_held()); | |
| 1280 | 1286 | |
| 1281 | 1287 | key_size = map->key_size; |
| 1282 | 1288 | |
| @@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) · 28 unmodified lines | ||
| 1311 | 1317 | u32 hash, key_size; |
| 1312 | 1318 | int ret; |
| 1313 | 1319 | |
| 1314 | − WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | |
| 1320 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && | |
| 1321 | + !rcu_read_lock_bh_held()); | |
| 1315 | 1322 | |
| 1316 | 1323 | key_size = map->key_size; |
| 1317 | 1324 | |
| @@ -29,7 +29,7 @@ · 28 unmodified lines | ||
| 29 | 29 | */ |
| 30 | 30 | BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) |
| 31 | 31 | { |
| 32 | − WARN_ON_ONCE(!rcu_read_lock_held()); | |
| 32 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); | |
| 33 | 33 | return (unsigned long) map->ops->map_lookup_elem(map, key); |
| 34 | 34 | } |
| 35 | 35 | |
| @@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines | ||
| 45 | 45 | BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, |
| 46 | 46 | void *, value, u64, flags) |
| 47 | 47 | { |
| 48 | − WARN_ON_ONCE(!rcu_read_lock_held()); | |
| 48 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); | |
| 49 | 49 | return map->ops->map_update_elem(map, key, value, flags); |
| 50 | 50 | } |
| 51 | 51 | |
| @@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines | ||
| 62 | 62 | |
| 63 | 63 | BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) |
| 64 | 64 | { |
| 65 | − WARN_ON_ONCE(!rcu_read_lock_held()); | |
| 65 | + WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); | |
| 66 | 66 | return map->ops->map_delete_elem(map, key); |
| 67 | 67 | } |
| 68 | 68 | |
| @@ -232,7 +232,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 231 unmodified lines | ||
| 232 | 232 | |
| 233 | 233 | /* Start walking the trie from the root node ... */ |
| 234 | 234 | |
| 235 | − for (node = rcu_dereference(trie->root); node;) { | |
| 235 | + for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held()); | |
| 236 | + node;) { | |
| 236 | 237 | unsigned int next_bit; |
| 237 | 238 | size_t matchlen; |
| 238 | 239 | |
| @@ -264,7 +265,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 25 unmodified lines | ||
| 264 | 265 | * traverse down. |
| 265 | 266 | */ |
| 266 | 267 | next_bit = extract_bit(key->data, node->prefixlen); |
| 267 | − node = rcu_dereference(node->child[next_bit]); | |
| 268 | + node = rcu_dereference_check(node->child[next_bit], | |
| 269 | + rcu_read_lock_bh_held()); | |
| 268 | 270 | } |
| 269 | 271 | |
| 270 | 272 | if (!found) |