Introduce BPF_F_LOCK flag for map_lookup and map_update syscall commands and for map_update() helper function. In all these cases take a lock of existing element (which was provided in BTF description) before copying (in or out) the rest of map value. Implementation details that are part of uapi: Array: The array map takes the element lock for lookup/update. Hash: hash map also takes the lock for lookup/update and tries to avoid the bucket lock. If old element exists it takes the element lock and updates the element in place. If element doesn't exist it allocates new one and inserts into hash table while holding the bucket lock. In rare case the hashmap has to take both the bucket lock and the element lock to update old value in place. Cgroup local storage: It is similar to array. update in place and lookup are done with lock taken.
Showing the changes to kernel/bpf/arraymap.c (1 of 7 files). Show all
Unified · Split
| @@ -253,8 +253,9 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, · 252 unmodified lines | |||
| 253 | { | 253 | { |
| 254 | struct bpf_array *array = container_of(map, struct bpf_array, map); | 254 | struct bpf_array *array = container_of(map, struct bpf_array, map); |
| 255 | u32 index = *(u32 *)key; | 255 | u32 index = *(u32 *)key; |
| 256 | char *val; | ||
| 256 | 257 | ||
| 257 | if (unlikely(map_flags > BPF_EXIST)) | 258 | if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) |
| 258 | /* unknown flags */ | 259 | /* unknown flags */ |
| 259 | return -EINVAL; | 260 | return -EINVAL; |
| 260 | 261 | ||
| @@ -262,18 +263,25 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, · 1 unmodified line | |||
| 262 | /* all elements were pre-allocated, cannot insert a new one */ | 263 | /* all elements were pre-allocated, cannot insert a new one */ |
| 263 | return -E2BIG; | 264 | return -E2BIG; |
| 264 | 265 | ||
| 265 | if (unlikely(map_flags == BPF_NOEXIST)) | 266 | if (unlikely(map_flags & BPF_NOEXIST)) |
| 266 | /* all elements already exist */ | 267 | /* all elements already exist */ |
| 267 | return -EEXIST; | 268 | return -EEXIST; |
| 268 | 269 | ||
| 269 | if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) | 270 | if (unlikely((map_flags & BPF_F_LOCK) && |
| 271 | !map_value_has_spin_lock(map))) | ||
| 272 | return -EINVAL; | ||
| 273 | |||
| 274 | if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { | ||
| 270 | memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), | 275 | memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), |
| 271 | value, map->value_size); | 276 | value, map->value_size); |
| 272 | else | 277 | } else { |
| 273 | copy_map_value(map, | 278 | val = array->value + |
| 274 | array->value + | 279 | array->elem_size * (index & array->index_mask); |
| 275 | array->elem_size * (index & array->index_mask), | 280 | if (map_flags & BPF_F_LOCK) |
| 276 | value); | 281 | copy_map_value_locked(map, val, value, false); |
| 282 | else | ||
| 283 | copy_map_value(map, val, value); | ||
| 284 | } | ||
| 277 | return 0; | 285 | return 0; |
| 278 | } | 286 | } |
| 279 | 287 | ||