History

bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps

Introduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs. Introduce support for the BPF_F_CPU flag in percpu_array maps to allow: * update value for specified CPU for both update_elem and update_batch APIs. * lookup value for specified CPU for both lookup_elem and lookup_batch APIs. The BPF_F_CPU flag is passed via: * map_flags of lookup_elem and update_elem APIs along with embedded cpu info. * elem_flags of lookup_batch and update_batch APIs along with embedded cpu info.

Leon Hwang committed 10 months ago

Commit 8eb76cb1 parent 2b421663 files · +31 −9Browse files

Showing the changes to kernel/bpf/arraymap.c (1 of 3 files). Show all

Unified · Split

kernel/bpf/arraymap.c+23−6
@@ -307,7 +307,7 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, · 306 unmodified lines
307 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);307 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
308}308}
309 309
310int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)310int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
311{311{
312 struct bpf_array *array = container_of(map, struct bpf_array, map);312 struct bpf_array *array = container_of(map, struct bpf_array, map);
313 u32 index = *(u32 *)key;313 u32 index = *(u32 *)key;
@@ -325,11 +325,18 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) · 11 unmodified lines
325 size = array->elem_size;325 size = array->elem_size;
326 rcu_read_lock();326 rcu_read_lock();
327 pptr = array->pptrs[index & array->index_mask];327 pptr = array->pptrs[index & array->index_mask];
328 if (map_flags & BPF_F_CPU) {
329 cpu = map_flags >> 32;
330 copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
331 check_and_init_map_value(map, value);
332 goto unlock;
333 }
328 for_each_possible_cpu(cpu) {334 for_each_possible_cpu(cpu) {
329 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));335 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
330 check_and_init_map_value(map, value + off);336 check_and_init_map_value(map, value + off);
331 off += size;337 off += size;
332 }338 }
339unlock:
333 rcu_read_unlock();340 rcu_read_unlock();
334 return 0;341 return 0;
335}342}
@@ -398,10 +405,11 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 62 unmodified lines
398 struct bpf_array *array = container_of(map, struct bpf_array, map);405 struct bpf_array *array = container_of(map, struct bpf_array, map);
399 u32 index = *(u32 *)key;406 u32 index = *(u32 *)key;
400 void __percpu *pptr;407 void __percpu *pptr;
401 int cpu, off = 0;408 void *ptr, *val;
402 u32 size;409 u32 size;
410 int cpu;
403 411
404 if (unlikely(map_flags > BPF_EXIST))412 if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))
405 /* unknown flags */413 /* unknown flags */
406 return -EINVAL;414 return -EINVAL;
407 415
@@ -422,11 +430,20 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 14 unmodified lines
422 size = array->elem_size;430 size = array->elem_size;
423 rcu_read_lock();431 rcu_read_lock();
424 pptr = array->pptrs[index & array->index_mask];432 pptr = array->pptrs[index & array->index_mask];
433 if (map_flags & BPF_F_CPU) {
434 cpu = map_flags >> 32;
435 ptr = per_cpu_ptr(pptr, cpu);
436 copy_map_value(map, ptr, value);
437 bpf_obj_free_fields(array->map.record, ptr);
438 goto unlock;
439 }
425 for_each_possible_cpu(cpu) {440 for_each_possible_cpu(cpu) {
426 copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);441 ptr = per_cpu_ptr(pptr, cpu);
427 bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));442 val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
428 off += size;443 copy_map_value(map, ptr, val);
444 bpf_obj_free_fields(array->map.record, ptr);
429 }445 }
446unlock:
430 rcu_read_unlock();447 rcu_read_unlock();
431 return 0;448 return 0;
432}449}