XDP programs are called from a NAPI poll context, which means the RCU reference liveness is ensured by local_bh_disable(). Add rcu_read_lock_bh_held() as a condition to the RCU checks for map lookups so lockdep understands that the dereferences are safe from inside *either* an rcu_read_lock() section *or* a local_bh_disable() section. While both bh_disabled and rcu_read_lock() provide RCU protection, they are semantically distinct, so we need both conditions to prevent lockdep complaints. This change is done in preparation for removing the redundant rcu_read_lock()s from drivers.
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -596,7 +596,8 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) · 595 unmodified lines | |||
| 596 | struct htab_elem *l; | 596 | struct htab_elem *l; |
| 597 | u32 hash, key_size; | 597 | u32 hash, key_size; |
| 598 | 598 | ||
| 599 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 599 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 600 | !rcu_read_lock_bh_held()); | ||
| 600 | 601 | ||
| 601 | key_size = map->key_size; | 602 | key_size = map->key_size; |
| 602 | 603 | ||
| @@ -989,7 +990,8 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 386 unmodified lines | |||
| 989 | /* unknown flags */ | 990 | /* unknown flags */ |
| 990 | return -EINVAL; | 991 | return -EINVAL; |
| 991 | 992 | ||
| 992 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 993 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 994 | !rcu_read_lock_bh_held()); | ||
| 993 | 995 | ||
| 994 | key_size = map->key_size; | 996 | key_size = map->key_size; |
| 995 | 997 | ||
| @@ -1082,7 +1084,8 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, · 86 unmodified lines | |||
| 1082 | /* unknown flags */ | 1084 | /* unknown flags */ |
| 1083 | return -EINVAL; | 1085 | return -EINVAL; |
| 1084 | 1086 | ||
| 1085 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 1087 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 1088 | !rcu_read_lock_bh_held()); | ||
| 1086 | 1089 | ||
| 1087 | key_size = map->key_size; | 1090 | key_size = map->key_size; |
| 1088 | 1091 | ||
| @@ -1148,7 +1151,8 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 59 unmodified lines | |||
| 1148 | /* unknown flags */ | 1151 | /* unknown flags */ |
| 1149 | return -EINVAL; | 1152 | return -EINVAL; |
| 1150 | 1153 | ||
| 1151 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 1154 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 1155 | !rcu_read_lock_bh_held()); | ||
| 1152 | 1156 | ||
| 1153 | key_size = map->key_size; | 1157 | key_size = map->key_size; |
| 1154 | 1158 | ||
| @@ -1202,7 +1206,8 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines | |||
| 1202 | /* unknown flags */ | 1206 | /* unknown flags */ |
| 1203 | return -EINVAL; | 1207 | return -EINVAL; |
| 1204 | 1208 | ||
| 1205 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 1209 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 1210 | !rcu_read_lock_bh_held()); | ||
| 1206 | 1211 | ||
| 1207 | key_size = map->key_size; | 1212 | key_size = map->key_size; |
| 1208 | 1213 | ||
| @@ -1276,7 +1281,8 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key) · 67 unmodified lines | |||
| 1276 | u32 hash, key_size; | 1281 | u32 hash, key_size; |
| 1277 | int ret; | 1282 | int ret; |
| 1278 | 1283 | ||
| 1279 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 1284 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 1285 | !rcu_read_lock_bh_held()); | ||
| 1280 | 1286 | ||
| 1281 | key_size = map->key_size; | 1287 | key_size = map->key_size; |
| 1282 | 1288 | ||
| @@ -1311,7 +1317,8 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) · 28 unmodified lines | |||
| 1311 | u32 hash, key_size; | 1317 | u32 hash, key_size; |
| 1312 | int ret; | 1318 | int ret; |
| 1313 | 1319 | ||
| 1314 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held()); | 1320 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held() && |
| 1321 | !rcu_read_lock_bh_held()); | ||
| 1315 | 1322 | ||
| 1316 | key_size = map->key_size; | 1323 | key_size = map->key_size; |
| 1317 | 1324 | ||
| @@ -29,7 +29,7 @@ · 28 unmodified lines | |||
| 29 | */ | 29 | */ |
| 30 | BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) | 30 | BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) |
| 31 | { | 31 | { |
| 32 | WARN_ON_ONCE(!rcu_read_lock_held()); | 32 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); |
| 33 | return (unsigned long) map->ops->map_lookup_elem(map, key); | 33 | return (unsigned long) map->ops->map_lookup_elem(map, key); |
| 34 | } | 34 | } |
| 35 | 35 | ||
| @@ -45,7 +45,7 @@ const struct bpf_func_proto bpf_map_lookup_elem_proto = { · 9 unmodified lines | |||
| 45 | BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, | 45 | BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, |
| 46 | void *, value, u64, flags) | 46 | void *, value, u64, flags) |
| 47 | { | 47 | { |
| 48 | WARN_ON_ONCE(!rcu_read_lock_held()); | 48 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); |
| 49 | return map->ops->map_update_elem(map, key, value, flags); | 49 | return map->ops->map_update_elem(map, key, value, flags); |
| 50 | } | 50 | } |
| 51 | 51 | ||
| @@ -62,7 +62,7 @@ const struct bpf_func_proto bpf_map_update_elem_proto = { · 10 unmodified lines | |||
| 62 | 62 | ||
| 63 | BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) | 63 | BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) |
| 64 | { | 64 | { |
| 65 | WARN_ON_ONCE(!rcu_read_lock_held()); | 65 | WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); |
| 66 | return map->ops->map_delete_elem(map, key); | 66 | return map->ops->map_delete_elem(map, key); |
| 67 | } | 67 | } |
| 68 | 68 | ||
| @@ -232,7 +232,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 231 unmodified lines | |||
| 232 | 232 | ||
| 233 | /* Start walking the trie from the root node ... */ | 233 | /* Start walking the trie from the root node ... */ |
| 234 | 234 | ||
| 235 | for (node = rcu_dereference(trie->root); node;) { | 235 | for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held()); |
| 236 | node;) { | ||
| 236 | unsigned int next_bit; | 237 | unsigned int next_bit; |
| 237 | size_t matchlen; | 238 | size_t matchlen; |
| 238 | 239 | ||
| @@ -264,7 +265,8 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key) · 25 unmodified lines | |||
| 264 | * traverse down. | 265 | * traverse down. |
| 265 | */ | 266 | */ |
| 266 | next_bit = extract_bit(key->data, node->prefixlen); | 267 | next_bit = extract_bit(key->data, node->prefixlen); |
| 267 | node = rcu_dereference(node->child[next_bit]); | 268 | node = rcu_dereference_check(node->child[next_bit], |
| 269 | rcu_read_lock_bh_held()); | ||
| 268 | } | 270 | } |
| 269 | 271 | ||
| 270 | if (!found) | 272 | if (!found) |