schedule_on_each_cpu() presently does a large kmalloc - 96 kbytes on 1024 CPU 64-bit. Rework it so that we do one 8192-byte allocation and then a pile of tiny ones, via alloc_percpu(). This has a much higher chance of success (100% in the current VM). This also has the effect of reducing the memory requirements from NR_CPUS*n to num_possible_cpus()*n.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -428,22 +428,34 @@ int schedule_delayed_work_on(int cpu, · 427 unmodified lines | |||
| 428 | return ret; | 428 | return ret; |
| 429 | } | 429 | } |
| 430 | 430 | ||
| 431 | int schedule_on_each_cpu(void (*func) (void *info), void *info) | 431 | /** |
| 432 | * schedule_on_each_cpu - call a function on each online CPU from keventd | ||
| 433 | * @func: the function to call | ||
| 434 | * @info: a pointer to pass to func() | ||
| 435 | * | ||
| 436 | * Returns zero on success. | ||
| 437 | * Returns -ve errno on failure. | ||
| 438 | * | ||
| 439 | * Appears to be racy against CPU hotplug. | ||
| 440 | * | ||
| 441 | * schedule_on_each_cpu() is very slow. | ||
| 442 | */ | ||
| 443 | int schedule_on_each_cpu(void (*func)(void *info), void *info) | ||
| 432 | { | 444 | { |
| 433 | int cpu; | 445 | int cpu; |
| 434 | struct work_struct *work; | 446 | struct work_struct *works; |
| 435 | 447 | ||
| 436 | work = kmalloc(NR_CPUS * sizeof(struct work_struct), GFP_KERNEL); | 448 | works = alloc_percpu(struct work_struct); |
| 437 | 449 | if (!works) | |
| 438 | if (!work) | ||
| 439 | return -ENOMEM; | 450 | return -ENOMEM; |
| 451 | |||
| 440 | for_each_online_cpu(cpu) { | 452 | for_each_online_cpu(cpu) { |
| 441 | INIT_WORK(work + cpu, func, info); | 453 | INIT_WORK(per_cpu_ptr(works, cpu), func, info); |
| 442 | __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), | 454 | __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), |
| 443 | work + cpu); | 455 | per_cpu_ptr(works, cpu)); |
| 444 | } | 456 | } |
| 445 | flush_workqueue(keventd_wq); | 457 | flush_workqueue(keventd_wq); |
| 446 | kfree(work); | 458 | free_percpu(works); |
| 447 | return 0; | 459 | return 0; |
| 448 | } | 460 | } |
| 449 | 461 | ||