Introduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs. Introduce support for the BPF_F_CPU flag in percpu_array maps to allow: * update value for specified CPU for both update_elem and update_batch APIs. * lookup value for specified CPU for both lookup_elem and lookup_batch APIs. The BPF_F_CPU flag is passed via: * map_flags of lookup_elem and update_elem APIs along with embedded cpu info. * elem_flags of lookup_batch and update_batch APIs along with embedded cpu info.
Showing the changes to kernel/bpf/arraymap.c (1 of 3 files). Show all
Unified · Split
| @@ -307,7 +307,7 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, · 306 unmodified lines | ||
| 307 | 307 | return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu); |
| 308 | 308 | } |
| 309 | 309 | |
| 310 | −int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) | |
| 310 | +int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags) | |
| 311 | 311 | { |
| 312 | 312 | struct bpf_array *array = container_of(map, struct bpf_array, map); |
| 313 | 313 | u32 index = *(u32 *)key; |
| @@ -325,11 +325,18 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) · 11 unmodified lines | ||
| 325 | 325 | size = array->elem_size; |
| 326 | 326 | rcu_read_lock(); |
| 327 | 327 | pptr = array->pptrs[index & array->index_mask]; |
| 328 | + if (map_flags & BPF_F_CPU) { | |
| 329 | + cpu = map_flags >> 32; | |
| 330 | + copy_map_value(map, value, per_cpu_ptr(pptr, cpu)); | |
| 331 | + check_and_init_map_value(map, value); | |
| 332 | + goto unlock; | |
| 333 | + } | |
| 328 | 334 | for_each_possible_cpu(cpu) { |
| 329 | 335 | copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); |
| 330 | 336 | check_and_init_map_value(map, value + off); |
| 331 | 337 | off += size; |
| 332 | 338 | } |
| 339 | +unlock: | |
| 333 | 340 | rcu_read_unlock(); |
| 334 | 341 | return 0; |
| 335 | 342 | } |
| @@ -398,10 +405,11 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 62 unmodified lines | ||
| 398 | 405 | struct bpf_array *array = container_of(map, struct bpf_array, map); |
| 399 | 406 | u32 index = *(u32 *)key; |
| 400 | 407 | void __percpu *pptr; |
| 401 | − int cpu, off = 0; | |
| 408 | + void *ptr, *val; | |
| 402 | 409 | u32 size; |
| 410 | + int cpu; | |
| 403 | 411 | |
| 404 | − if (unlikely(map_flags > BPF_EXIST)) | |
| 412 | + if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)) | |
| 405 | 413 | /* unknown flags */ |
| 406 | 414 | return -EINVAL; |
| 407 | 415 | |
| @@ -422,11 +430,20 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 14 unmodified lines | ||
| 422 | 430 | size = array->elem_size; |
| 423 | 431 | rcu_read_lock(); |
| 424 | 432 | pptr = array->pptrs[index & array->index_mask]; |
| 433 | + if (map_flags & BPF_F_CPU) { | |
| 434 | + cpu = map_flags >> 32; | |
| 435 | + ptr = per_cpu_ptr(pptr, cpu); | |
| 436 | + copy_map_value(map, ptr, value); | |
| 437 | + bpf_obj_free_fields(array->map.record, ptr); | |
| 438 | + goto unlock; | |
| 439 | + } | |
| 425 | 440 | for_each_possible_cpu(cpu) { |
| 426 | − copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); | |
| 427 | − bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu)); | |
| 428 | − off += size; | |
| 441 | + ptr = per_cpu_ptr(pptr, cpu); | |
| 442 | + val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu; | |
| 443 | + copy_map_value(map, ptr, val); | |
| 444 | + bpf_obj_free_fields(array->map.record, ptr); | |
| 429 | 445 | } |
| 446 | +unlock: | |
| 430 | 447 | rcu_read_unlock(); |
| 431 | 448 | return 0; |
| 432 | 449 | } |