History

bpf: Add BPF_F_CPU and BPF_F_ALL_CPUS flags support for percpu_array maps

Introduce support for the BPF_F_ALL_CPUS flag in percpu_array maps to allow updating values for all CPUs with a single value for both update_elem and update_batch APIs. Introduce support for the BPF_F_CPU flag in percpu_array maps to allow: * update value for specified CPU for both update_elem and update_batch APIs. * lookup value for specified CPU for both lookup_elem and lookup_batch APIs. The BPF_F_CPU flag is passed via: * map_flags of lookup_elem and update_elem APIs along with embedded cpu info. * elem_flags of lookup_batch and update_batch APIs along with embedded cpu info.

Leon Hwang committed 10 months ago

Commit 8eb76cb1 parent 2b421663 files · +31 −9Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +7 −2
  2. kernel/bpf/arraymap.c +23 −6
  3. kernel/bpf/syscall.c +1 −1
include/linux/bpf.h+7−2
@@ -2848,7 +2848,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, · 2847 unmodified lines
28482848 struct bpf_func_state *callee);
28492849
28502850 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2851−int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2851+int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags);
28522852 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
28532853 u64 flags);
28542854 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
@@ -3917,7 +3917,12 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) · 1062 unmodified lines
39173917
39183918 static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type)
39193919 {
3920− return false;
3920+ switch (map_type) {
3921+ case BPF_MAP_TYPE_PERCPU_ARRAY:
3922+ return true;
3923+ default:
3924+ return false;
3925+ }
39213926 }
39223927
39233928 static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)
kernel/bpf/arraymap.c+23−6
@@ -307,7 +307,7 @@ static void *percpu_array_map_lookup_percpu_elem(struct bpf_map *map, void *key, · 306 unmodified lines
307307 return per_cpu_ptr(array->pptrs[index & array->index_mask], cpu);
308308 }
309309
310−int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
310+int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
311311 {
312312 struct bpf_array *array = container_of(map, struct bpf_array, map);
313313 u32 index = *(u32 *)key;
@@ -325,11 +325,18 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) · 11 unmodified lines
325325 size = array->elem_size;
326326 rcu_read_lock();
327327 pptr = array->pptrs[index & array->index_mask];
328+ if (map_flags & BPF_F_CPU) {
329+ cpu = map_flags >> 32;
330+ copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
331+ check_and_init_map_value(map, value);
332+ goto unlock;
333+ }
328334 for_each_possible_cpu(cpu) {
329335 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
330336 check_and_init_map_value(map, value + off);
331337 off += size;
332338 }
339+unlock:
333340 rcu_read_unlock();
334341 return 0;
335342 }
@@ -398,10 +405,11 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 62 unmodified lines
398405 struct bpf_array *array = container_of(map, struct bpf_array, map);
399406 u32 index = *(u32 *)key;
400407 void __percpu *pptr;
401− int cpu, off = 0;
408+ void *ptr, *val;
402409 u32 size;
410+ int cpu;
403411
404− if (unlikely(map_flags > BPF_EXIST))
412+ if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))
405413 /* unknown flags */
406414 return -EINVAL;
407415
@@ -422,11 +430,20 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, · 14 unmodified lines
422430 size = array->elem_size;
423431 rcu_read_lock();
424432 pptr = array->pptrs[index & array->index_mask];
433+ if (map_flags & BPF_F_CPU) {
434+ cpu = map_flags >> 32;
435+ ptr = per_cpu_ptr(pptr, cpu);
436+ copy_map_value(map, ptr, value);
437+ bpf_obj_free_fields(array->map.record, ptr);
438+ goto unlock;
439+ }
425440 for_each_possible_cpu(cpu) {
426− copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
427− bpf_obj_free_fields(array->map.record, per_cpu_ptr(pptr, cpu));
428− off += size;
441+ ptr = per_cpu_ptr(pptr, cpu);
442+ val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
443+ copy_map_value(map, ptr, val);
444+ bpf_obj_free_fields(array->map.record, ptr);
429445 }
446+unlock:
430447 rcu_read_unlock();
431448 return 0;
432449 }
kernel/bpf/syscall.c+1−1
@@ -318,7 +318,7 @@ static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value, · 317 unmodified lines
318318 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
319319 err = bpf_percpu_hash_copy(map, key, value);
320320 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
321− err = bpf_percpu_array_copy(map, key, value);
321+ err = bpf_percpu_array_copy(map, key, value, flags);
322322 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
323323 err = bpf_percpu_cgroup_storage_copy(map, key, value);
324324 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {