padata currently uses one per-CPU workqueue per instance for all work. Prepare for running parallel jobs on an unbound workqueue by introducing dedicated workqueues for parallel and serial work.
Unified · Split
| @@ -127,7 +127,8 @@ struct parallel_data { · 126 unmodified lines | ||
| 127 | 127 | * struct padata_instance - The overall control structure. |
| 128 | 128 | * |
| 129 | 129 | * @cpu_notifier: cpu hotplug notifier. |
| 130 | − * @wq: The workqueue in use. | |
| 130 | + * @parallel_wq: The workqueue used for parallel work. | |
| 131 | + * @serial_wq: The workqueue used for serial work. | |
| 131 | 132 | * @pd: The internal control structure. |
| 132 | 133 | * @cpumask: User supplied cpumasks for parallel and serial works. |
| 133 | 134 | * @cpumask_change_notifier: Notifiers chain for user-defined notify |
| @@ -139,7 +140,8 @@ struct parallel_data { · 5 unmodified lines | ||
| 139 | 140 | */ |
| 140 | 141 | struct padata_instance { |
| 141 | 142 | struct hlist_node node; |
| 142 | − struct workqueue_struct *wq; | |
| 143 | + struct workqueue_struct *parallel_wq; | |
| 144 | + struct workqueue_struct *serial_wq; | |
| 143 | 145 | struct parallel_data *pd; |
| 144 | 146 | struct padata_cpumask cpumask; |
| 145 | 147 | struct blocking_notifier_head cpumask_change_notifier; |
| @@ -152,7 +152,7 @@ int padata_do_parallel(struct padata_instance *pinst, · 151 unmodified lines | ||
| 152 | 152 | list_add_tail(&padata->list, &queue->parallel.list); |
| 153 | 153 | spin_unlock(&queue->parallel.lock); |
| 154 | 154 | |
| 155 | − queue_work_on(target_cpu, pinst->wq, &queue->work); | |
| 155 | + queue_work_on(target_cpu, pinst->parallel_wq, &queue->work); | |
| 156 | 156 | |
| 157 | 157 | out: |
| 158 | 158 | rcu_read_unlock_bh(); |
| @@ -261,7 +261,7 @@ static void padata_reorder(struct parallel_data *pd) · 102 unmodified lines | ||
| 261 | 261 | list_add_tail(&padata->list, &squeue->serial.list); |
| 262 | 262 | spin_unlock(&squeue->serial.lock); |
| 263 | 263 | |
| 264 | − queue_work_on(cb_cpu, pinst->wq, &squeue->work); | |
| 264 | + queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work); | |
| 265 | 265 | } |
| 266 | 266 | |
| 267 | 267 | spin_unlock_bh(&pd->lock); |
| @@ -278,7 +278,7 @@ static void padata_reorder(struct parallel_data *pd) · 10 unmodified lines | ||
| 278 | 278 | |
| 279 | 279 | next_queue = per_cpu_ptr(pd->pqueue, pd->cpu); |
| 280 | 280 | if (!list_empty(&next_queue->reorder.list)) |
| 281 | − queue_work(pinst->wq, &pd->reorder_work); | |
| 281 | + queue_work(pinst->serial_wq, &pd->reorder_work); | |
| 282 | 282 | } |
| 283 | 283 | |
| 284 | 284 | static void invoke_padata_reorder(struct work_struct *work) |
| @@ -818,7 +818,8 @@ static void __padata_free(struct padata_instance *pinst) · 533 unmodified lines | ||
| 818 | 818 | padata_free_pd(pinst->pd); |
| 819 | 819 | free_cpumask_var(pinst->cpumask.pcpu); |
| 820 | 820 | free_cpumask_var(pinst->cpumask.cbcpu); |
| 821 | − destroy_workqueue(pinst->wq); | |
| 821 | + destroy_workqueue(pinst->serial_wq); | |
| 822 | + destroy_workqueue(pinst->parallel_wq); | |
| 822 | 823 | kfree(pinst); |
| 823 | 824 | } |
| 824 | 825 | |
| @@ -967,18 +968,23 @@ static struct padata_instance *padata_alloc(const char *name, · 142 unmodified lines | ||
| 967 | 968 | if (!pinst) |
| 968 | 969 | goto err; |
| 969 | 970 | |
| 970 | − pinst->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, | |
| 971 | − 1, name); | |
| 972 | − if (!pinst->wq) | |
| 971 | + pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_MEM_RECLAIM | | |
| 972 | + WQ_CPU_INTENSIVE, 1, name); | |
| 973 | + if (!pinst->parallel_wq) | |
| 973 | 974 | goto err_free_inst; |
| 974 | 975 | |
| 975 | 976 | get_online_cpus(); |
| 976 | 977 | |
| 977 | − if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) | |
| 978 | + pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM | | |
| 979 | + WQ_CPU_INTENSIVE, 1, name); | |
| 980 | + if (!pinst->serial_wq) | |
| 978 | 981 | goto err_put_cpus; |
| 982 | + | |
| 983 | + if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) | |
| 984 | + goto err_free_serial_wq; | |
| 979 | 985 | if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { |
| 980 | 986 | free_cpumask_var(pinst->cpumask.pcpu); |
| 981 | − goto err_put_cpus; | |
| 987 | + goto err_free_serial_wq; | |
| 982 | 988 | } |
| 983 | 989 | if (!padata_validate_cpumask(pinst, pcpumask) || |
| 984 | 990 | !padata_validate_cpumask(pinst, cbcpumask)) |
| @@ -1010,9 +1016,11 @@ static struct padata_instance *padata_alloc(const char *name, · 25 unmodified lines | ||
| 1010 | 1016 | err_free_masks: |
| 1011 | 1017 | free_cpumask_var(pinst->cpumask.pcpu); |
| 1012 | 1018 | free_cpumask_var(pinst->cpumask.cbcpu); |
| 1019 | +err_free_serial_wq: | |
| 1020 | + destroy_workqueue(pinst->serial_wq); | |
| 1013 | 1021 | err_put_cpus: |
| 1014 | 1022 | put_online_cpus(); |
| 1015 | − destroy_workqueue(pinst->wq); | |
| 1023 | + destroy_workqueue(pinst->parallel_wq); | |
| 1016 | 1024 | err_free_inst: |
| 1017 | 1025 | kfree(pinst); |
| 1018 | 1026 | err: |