padata currently uses one per-CPU workqueue per instance for all work. Prepare for running parallel jobs on an unbound workqueue by introducing dedicated workqueues for parallel and serial work.
Unified · Split
| @@ -127,7 +127,8 @@ struct parallel_data { · 126 unmodified lines | |||
| 127 | * struct padata_instance - The overall control structure. | 127 | * struct padata_instance - The overall control structure. |
| 128 | * | 128 | * |
| 129 | * @cpu_notifier: cpu hotplug notifier. | 129 | * @cpu_notifier: cpu hotplug notifier. |
| 130 | * @wq: The workqueue in use. | 130 | * @parallel_wq: The workqueue used for parallel work. |
| 131 | * @serial_wq: The workqueue used for serial work. | ||
| 131 | * @pd: The internal control structure. | 132 | * @pd: The internal control structure. |
| 132 | * @cpumask: User supplied cpumasks for parallel and serial works. | 133 | * @cpumask: User supplied cpumasks for parallel and serial works. |
| 133 | * @cpumask_change_notifier: Notifiers chain for user-defined notify | 134 | * @cpumask_change_notifier: Notifiers chain for user-defined notify |
| @@ -139,7 +140,8 @@ struct parallel_data { · 5 unmodified lines | |||
| 139 | */ | 140 | */ |
| 140 | struct padata_instance { | 141 | struct padata_instance { |
| 141 | struct hlist_node node; | 142 | struct hlist_node node; |
| 142 | struct workqueue_struct *wq; | 143 | struct workqueue_struct *parallel_wq; |
| 144 | struct workqueue_struct *serial_wq; | ||
| 143 | struct parallel_data *pd; | 145 | struct parallel_data *pd; |
| 144 | struct padata_cpumask cpumask; | 146 | struct padata_cpumask cpumask; |
| 145 | struct blocking_notifier_head cpumask_change_notifier; | 147 | struct blocking_notifier_head cpumask_change_notifier; |
| @@ -152,7 +152,7 @@ int padata_do_parallel(struct padata_instance *pinst, · 151 unmodified lines | |||
| 152 | list_add_tail(&padata->list, &queue->parallel.list); | 152 | list_add_tail(&padata->list, &queue->parallel.list); |
| 153 | spin_unlock(&queue->parallel.lock); | 153 | spin_unlock(&queue->parallel.lock); |
| 154 | 154 | ||
| 155 | queue_work_on(target_cpu, pinst->wq, &queue->work); | 155 | queue_work_on(target_cpu, pinst->parallel_wq, &queue->work); |
| 156 | 156 | ||
| 157 | out: | 157 | out: |
| 158 | rcu_read_unlock_bh(); | 158 | rcu_read_unlock_bh(); |
| @@ -261,7 +261,7 @@ static void padata_reorder(struct parallel_data *pd) · 102 unmodified lines | |||
| 261 | list_add_tail(&padata->list, &squeue->serial.list); | 261 | list_add_tail(&padata->list, &squeue->serial.list); |
| 262 | spin_unlock(&squeue->serial.lock); | 262 | spin_unlock(&squeue->serial.lock); |
| 263 | 263 | ||
| 264 | queue_work_on(cb_cpu, pinst->wq, &squeue->work); | 264 | queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work); |
| 265 | } | 265 | } |
| 266 | 266 | ||
| 267 | spin_unlock_bh(&pd->lock); | 267 | spin_unlock_bh(&pd->lock); |
| @@ -278,7 +278,7 @@ static void padata_reorder(struct parallel_data *pd) · 10 unmodified lines | |||
| 278 | 278 | ||
| 279 | next_queue = per_cpu_ptr(pd->pqueue, pd->cpu); | 279 | next_queue = per_cpu_ptr(pd->pqueue, pd->cpu); |
| 280 | if (!list_empty(&next_queue->reorder.list)) | 280 | if (!list_empty(&next_queue->reorder.list)) |
| 281 | queue_work(pinst->wq, &pd->reorder_work); | 281 | queue_work(pinst->serial_wq, &pd->reorder_work); |
| 282 | } | 282 | } |
| 283 | 283 | ||
| 284 | static void invoke_padata_reorder(struct work_struct *work) | 284 | static void invoke_padata_reorder(struct work_struct *work) |
| @@ -818,7 +818,8 @@ static void __padata_free(struct padata_instance *pinst) · 533 unmodified lines | |||
| 818 | padata_free_pd(pinst->pd); | 818 | padata_free_pd(pinst->pd); |
| 819 | free_cpumask_var(pinst->cpumask.pcpu); | 819 | free_cpumask_var(pinst->cpumask.pcpu); |
| 820 | free_cpumask_var(pinst->cpumask.cbcpu); | 820 | free_cpumask_var(pinst->cpumask.cbcpu); |
| 821 | destroy_workqueue(pinst->wq); | 821 | destroy_workqueue(pinst->serial_wq); |
| 822 | destroy_workqueue(pinst->parallel_wq); | ||
| 822 | kfree(pinst); | 823 | kfree(pinst); |
| 823 | } | 824 | } |
| 824 | 825 | ||
| @@ -967,18 +968,23 @@ static struct padata_instance *padata_alloc(const char *name, · 142 unmodified lines | |||
| 967 | if (!pinst) | 968 | if (!pinst) |
| 968 | goto err; | 969 | goto err; |
| 969 | 970 | ||
| 970 | pinst->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, | 971 | pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_MEM_RECLAIM | |
| 971 | 1, name); | 972 | WQ_CPU_INTENSIVE, 1, name); |
| 972 | if (!pinst->wq) | 973 | if (!pinst->parallel_wq) |
| 973 | goto err_free_inst; | 974 | goto err_free_inst; |
| 974 | 975 | ||
| 975 | get_online_cpus(); | 976 | get_online_cpus(); |
| 976 | 977 | ||
| 977 | if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) | 978 | pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM | |
| 979 | WQ_CPU_INTENSIVE, 1, name); | ||
| 980 | if (!pinst->serial_wq) | ||
| 978 | goto err_put_cpus; | 981 | goto err_put_cpus; |
| 982 | |||
| 983 | if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) | ||
| 984 | goto err_free_serial_wq; | ||
| 979 | if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { | 985 | if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { |
| 980 | free_cpumask_var(pinst->cpumask.pcpu); | 986 | free_cpumask_var(pinst->cpumask.pcpu); |
| 981 | goto err_put_cpus; | 987 | goto err_free_serial_wq; |
| 982 | } | 988 | } |
| 983 | if (!padata_validate_cpumask(pinst, pcpumask) || | 989 | if (!padata_validate_cpumask(pinst, pcpumask) || |
| 984 | !padata_validate_cpumask(pinst, cbcpumask)) | 990 | !padata_validate_cpumask(pinst, cbcpumask)) |
| @@ -1010,9 +1016,11 @@ static struct padata_instance *padata_alloc(const char *name, · 25 unmodified lines | |||
| 1010 | err_free_masks: | 1016 | err_free_masks: |
| 1011 | free_cpumask_var(pinst->cpumask.pcpu); | 1017 | free_cpumask_var(pinst->cpumask.pcpu); |
| 1012 | free_cpumask_var(pinst->cpumask.cbcpu); | 1018 | free_cpumask_var(pinst->cpumask.cbcpu); |
| 1019 | err_free_serial_wq: | ||
| 1020 | destroy_workqueue(pinst->serial_wq); | ||
| 1013 | err_put_cpus: | 1021 | err_put_cpus: |
| 1014 | put_online_cpus(); | 1022 | put_online_cpus(); |
| 1015 | destroy_workqueue(pinst->wq); | 1023 | destroy_workqueue(pinst->parallel_wq); |
| 1016 | err_free_inst: | 1024 | err_free_inst: |
| 1017 | kfree(pinst); | 1025 | kfree(pinst); |
| 1018 | err: | 1026 | err: |