| @@ -52,7 +52,6 @@ struct xdp_bulk_queue { · 51 unmodified lines |
| 52 | struct bpf_cpu_map_entry { | 52 | struct bpf_cpu_map_entry { |
| 53 | u32 cpu; /* kthread CPU and map index */ | 53 | u32 cpu; /* kthread CPU and map index */ |
| 54 | int map_id; /* Back reference to map */ | 54 | int map_id; /* Back reference to map */ |
| 55 | u32 qsize; /* Queue size placeholder for map lookup */ | | |
| 56 | | 55 | |
| 57 | /* XDP can run multiple RX-ring queues, need __percpu enqueue store */ | 56 | /* XDP can run multiple RX-ring queues, need __percpu enqueue store */ |
| 58 | struct xdp_bulk_queue __percpu *bulkq; | 57 | struct xdp_bulk_queue __percpu *bulkq; |
| @@ -62,10 +61,13 @@ struct bpf_cpu_map_entry { · 3 unmodified lines |
| 62 | /* Queue with potential multi-producers, and single-consumer kthread */ | 61 | /* Queue with potential multi-producers, and single-consumer kthread */ |
| 63 | struct ptr_ring *queue; | 62 | struct ptr_ring *queue; |
| 64 | struct task_struct *kthread; | 63 | struct task_struct *kthread; |
| 65 | struct work_struct kthread_stop_wq; | 64 | |
| | 65 | struct bpf_cpumap_val value; |
| 66 | | 66 | |
| 67 | atomic_t refcnt; /* Control when this struct can be free'ed */ | 67 | atomic_t refcnt; /* Control when this struct can be free'ed */ |
| 68 | struct rcu_head rcu; | 68 | struct rcu_head rcu; |
| | 69 | |
| | 70 | struct work_struct kthread_stop_wq; |
| 69 | }; | 71 | }; |
| 70 | | 72 | |
| 71 | struct bpf_cpu_map { | 73 | struct bpf_cpu_map { |
| @@ -307,8 +309,8 @@ static int cpu_map_kthread_run(void *data) · 235 unmodified lines |
| 307 | return 0; | 309 | return 0; |
| 308 | } | 310 | } |
| 309 | | 311 | |
| 310 | static struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, | 312 | static struct bpf_cpu_map_entry * |
| 311 | int map_id) | 313 | __cpu_map_entry_alloc(struct bpf_cpumap_val *value, u32 cpu, int map_id) |
| 312 | { | 314 | { |
| 313 | gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; | 315 | gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; |
| 314 | struct bpf_cpu_map_entry *rcpu; | 316 | struct bpf_cpu_map_entry *rcpu; |
| @@ -338,13 +340,13 @@ static struct bpf_cpu_map_entry *__cpu_map_entry_alloc(u32 qsize, u32 cpu, · 23 unmodified lines |
| 338 | if (!rcpu->queue) | 340 | if (!rcpu->queue) |
| 339 | goto free_bulkq; | 341 | goto free_bulkq; |
| 340 | | 342 | |
| 341 | err = ptr_ring_init(rcpu->queue, qsize, gfp); | 343 | err = ptr_ring_init(rcpu->queue, value->qsize, gfp); |
| 342 | if (err) | 344 | if (err) |
| 343 | goto free_queue; | 345 | goto free_queue; |
| 344 | | 346 | |
| 345 | rcpu->cpu = cpu; | 347 | rcpu->cpu = cpu; |
| 346 | rcpu->map_id = map_id; | 348 | rcpu->map_id = map_id; |
| 347 | rcpu->qsize = qsize; | 349 | rcpu->value.qsize = value->qsize; |
| 348 | | 350 | |
| 349 | /* Setup kthread */ | 351 | /* Setup kthread */ |
| 350 | rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa, | 352 | rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa, |
| @@ -437,12 +439,12 @@ static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, · 86 unmodified lines |
| 437 | u64 map_flags) | 439 | u64 map_flags) |
| 438 | { | 440 | { |
| 439 | struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); | 441 | struct bpf_cpu_map *cmap = container_of(map, struct bpf_cpu_map, map); |
| | 442 | struct bpf_cpumap_val cpumap_value = {}; |
| 440 | struct bpf_cpu_map_entry *rcpu; | 443 | struct bpf_cpu_map_entry *rcpu; |
| 441 | | | |
| 442 | /* Array index key correspond to CPU number */ | 444 | /* Array index key correspond to CPU number */ |
| 443 | u32 key_cpu = *(u32 *)key; | 445 | u32 key_cpu = *(u32 *)key; |
| 444 | /* Value is the queue size */ | 446 | |
| 445 | u32 qsize = *(u32 *)value; | 447 | memcpy(&cpumap_value, value, map->value_size); |
| 446 | | 448 | |
| 447 | if (unlikely(map_flags > BPF_EXIST)) | 449 | if (unlikely(map_flags > BPF_EXIST)) |
| 448 | return -EINVAL; | 450 | return -EINVAL; |
| @@ -450,18 +452,18 @@ static int cpu_map_update_elem(struct bpf_map *map, void *key, void *value, · 1 unmodified line |
| 450 | return -E2BIG; | 452 | return -E2BIG; |
| 451 | if (unlikely(map_flags == BPF_NOEXIST)) | 453 | if (unlikely(map_flags == BPF_NOEXIST)) |
| 452 | return -EEXIST; | 454 | return -EEXIST; |
| 453 | if (unlikely(qsize > 16384)) /* sanity limit on qsize */ | 455 | if (unlikely(cpumap_value.qsize > 16384)) /* sanity limit on qsize */ |
| 454 | return -EOVERFLOW; | 456 | return -EOVERFLOW; |
| 455 | | 457 | |
| 456 | /* Make sure CPU is a valid possible cpu */ | 458 | /* Make sure CPU is a valid possible cpu */ |
| 457 | if (key_cpu >= nr_cpumask_bits || !cpu_possible(key_cpu)) | 459 | if (key_cpu >= nr_cpumask_bits || !cpu_possible(key_cpu)) |
| 458 | return -ENODEV; | 460 | return -ENODEV; |
| 459 | | 461 | |
| 460 | if (qsize == 0) { | 462 | if (cpumap_value.qsize == 0) { |
| 461 | rcpu = NULL; /* Same as deleting */ | 463 | rcpu = NULL; /* Same as deleting */ |
| 462 | } else { | 464 | } else { |
| 463 | /* Updating qsize cause re-allocation of bpf_cpu_map_entry */ | 465 | /* Updating qsize cause re-allocation of bpf_cpu_map_entry */ |
| 464 | rcpu = __cpu_map_entry_alloc(qsize, key_cpu, map->id); | 466 | rcpu = __cpu_map_entry_alloc(&cpumap_value, key_cpu, map->id); |
| 465 | if (!rcpu) | 467 | if (!rcpu) |
| 466 | return -ENOMEM; | 468 | return -ENOMEM; |
| 467 | rcpu->cmap = cmap; | 469 | rcpu->cmap = cmap; |
| @@ -523,7 +525,7 @@ static void *cpu_map_lookup_elem(struct bpf_map *map, void *key) · 55 unmodified lines |
| 523 | struct bpf_cpu_map_entry *rcpu = | 525 | struct bpf_cpu_map_entry *rcpu = |
| 524 | __cpu_map_lookup_elem(map, *(u32 *)key); | 526 | __cpu_map_lookup_elem(map, *(u32 *)key); |
| 525 | | 527 | |
| 526 | return rcpu ? &rcpu->qsize : NULL; | 528 | return rcpu ? &rcpu->value : NULL; |
| 527 | } | 529 | } |
| 528 | | 530 | |
| 529 | static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key) | 531 | static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key) |