schedule_on_each_cpu() presently does a large kmalloc - 96 kbytes on 1024 CPU 64-bit. Rework it so that we do one 8192-byte allocation and then a pile of tiny ones, via alloc_percpu(). This has a much higher chance of success (100% in the current VM). This also has the effect of reducing the memory requirements from NR_CPUS*n to num_possible_cpus()*n.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -428,22 +428,34 @@ int schedule_delayed_work_on(int cpu, · 427 unmodified lines | ||
| 428 | 428 | return ret; |
| 429 | 429 | } |
| 430 | 430 | |
| 431 | −int schedule_on_each_cpu(void (*func) (void *info), void *info) | |
| 431 | +/** | |
| 432 | + * schedule_on_each_cpu - call a function on each online CPU from keventd | |
| 433 | + * @func: the function to call | |
| 434 | + * @info: a pointer to pass to func() | |
| 435 | + * | |
| 436 | + * Returns zero on success. | |
| 437 | + * Returns -ve errno on failure. | |
| 438 | + * | |
| 439 | + * Appears to be racy against CPU hotplug. | |
| 440 | + * | |
| 441 | + * schedule_on_each_cpu() is very slow. | |
| 442 | + */ | |
| 443 | +int schedule_on_each_cpu(void (*func)(void *info), void *info) | |
| 432 | 444 | { |
| 433 | 445 | int cpu; |
| 434 | − struct work_struct *work; | |
| 446 | + struct work_struct *works; | |
| 435 | 447 | |
| 436 | − work = kmalloc(NR_CPUS * sizeof(struct work_struct), GFP_KERNEL); | |
| 437 | − | |
| 438 | − if (!work) | |
| 448 | + works = alloc_percpu(struct work_struct); | |
| 449 | + if (!works) | |
| 439 | 450 | return -ENOMEM; |
| 451 | + | |
| 440 | 452 | for_each_online_cpu(cpu) { |
| 441 | − INIT_WORK(work + cpu, func, info); | |
| 453 | + INIT_WORK(per_cpu_ptr(works, cpu), func, info); | |
| 442 | 454 | __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), |
| 443 | − work + cpu); | |
| 455 | + per_cpu_ptr(works, cpu)); | |
| 444 | 456 | } |
| 445 | 457 | flush_workqueue(keventd_wq); |
| 446 | − kfree(work); | |
| 458 | + free_percpu(works); | |
| 447 | 459 | return 0; |
| 448 | 460 | } |
| 449 | 461 | |