get_pwq() takes @cpu, which can also be WORK_CPU_UNBOUND, and @wq and returns the matching pwq (pool_workqueue). We want to move away from using @cpu for identifying pools and pwqs for unbound pools with custom attributes and there is only one user - workqueue_congested() - which makes use of the WQ_UNBOUND conditional in get_pwq(). All other users already know whether they're dealing with a per-cpu or unbound workqueue. Replace get_pwq() with explicit per_cpu_ptr(wq->cpu_pwqs, cpu) for per-cpu workqueues and first_pwq() for unbound ones, and open-code WQ_UNBOUND conditional in workqueue_congested(). Note that this makes workqueue_congested() behave sligntly differently when @cpu other than WORK_CPU_UNBOUND is specified. It ignores @cpu for unbound workqueues and always uses the first pwq instead of oopsing.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -463,16 +463,9 @@ static struct worker_pool *get_std_worker_pool(int cpu, bool highpri) · 462 unmodified lines | |||
| 463 | return &pools[highpri]; | 463 | return &pools[highpri]; |
| 464 | } | 464 | } |
| 465 | 465 | ||
| 466 | static struct pool_workqueue *get_pwq(int cpu, struct workqueue_struct *wq) | 466 | static struct pool_workqueue *first_pwq(struct workqueue_struct *wq) |
| 467 | { | 467 | { |
| 468 | if (!(wq->flags & WQ_UNBOUND)) { | 468 | return list_first_entry(&wq->pwqs, struct pool_workqueue, pwqs_node); |
| 469 | if (likely(cpu < nr_cpu_ids)) | ||
| 470 | return per_cpu_ptr(wq->cpu_pwqs, cpu); | ||
| 471 | } else if (likely(cpu == WORK_CPU_UNBOUND)) { | ||
| 472 | return list_first_entry(&wq->pwqs, struct pool_workqueue, | ||
| 473 | pwqs_node); | ||
| 474 | } | ||
| 475 | return NULL; | ||
| 476 | } | 469 | } |
| 477 | 470 | ||
| 478 | static unsigned int work_color_to_flags(int color) | 471 | static unsigned int work_color_to_flags(int color) |
| @@ -1191,7 +1184,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, · 712 unmodified lines | |||
| 1191 | * work needs to be queued on that cpu to guarantee | 1184 | * work needs to be queued on that cpu to guarantee |
| 1192 | * non-reentrancy. | 1185 | * non-reentrancy. |
| 1193 | */ | 1186 | */ |
| 1194 | pwq = get_pwq(cpu, wq); | 1187 | pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); |
| 1195 | last_pool = get_work_pool(work); | 1188 | last_pool = get_work_pool(work); |
| 1196 | 1189 | ||
| 1197 | if (last_pool && last_pool != pwq->pool) { | 1190 | if (last_pool && last_pool != pwq->pool) { |
| @@ -1202,7 +1195,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, · 4 unmodified lines | |||
| 1202 | worker = find_worker_executing_work(last_pool, work); | 1195 | worker = find_worker_executing_work(last_pool, work); |
| 1203 | 1196 | ||
| 1204 | if (worker && worker->current_pwq->wq == wq) { | 1197 | if (worker && worker->current_pwq->wq == wq) { |
| 1205 | pwq = get_pwq(last_pool->cpu, wq); | 1198 | pwq = per_cpu_ptr(wq->cpu_pwqs, last_pool->cpu); |
| 1206 | } else { | 1199 | } else { |
| 1207 | /* meh... not running there, queue here */ | 1200 | /* meh... not running there, queue here */ |
| 1208 | spin_unlock(&last_pool->lock); | 1201 | spin_unlock(&last_pool->lock); |
| @@ -1212,7 +1205,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, · 3 unmodified lines | |||
| 1212 | spin_lock(&pwq->pool->lock); | 1205 | spin_lock(&pwq->pool->lock); |
| 1213 | } | 1206 | } |
| 1214 | } else { | 1207 | } else { |
| 1215 | pwq = get_pwq(WORK_CPU_UNBOUND, wq); | 1208 | pwq = first_pwq(wq); |
| 1216 | spin_lock(&pwq->pool->lock); | 1209 | spin_lock(&pwq->pool->lock); |
| 1217 | } | 1210 | } |
| 1218 | 1211 | ||
| @@ -1650,7 +1643,7 @@ static void rebind_workers(struct worker_pool *pool) · 431 unmodified lines | |||
| 1650 | else | 1643 | else |
| 1651 | wq = system_wq; | 1644 | wq = system_wq; |
| 1652 | 1645 | ||
| 1653 | insert_work(get_pwq(pool->cpu, wq), rebind_work, | 1646 | insert_work(per_cpu_ptr(wq->cpu_pwqs, pool->cpu), rebind_work, |
| 1654 | worker->scheduled.next, | 1647 | worker->scheduled.next, |
| 1655 | work_color_to_flags(WORK_NO_COLOR)); | 1648 | work_color_to_flags(WORK_NO_COLOR)); |
| 1656 | } | 1649 | } |
| @@ -3088,7 +3081,8 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) · 1431 unmodified lines | |||
| 3088 | return -ENOMEM; | 3081 | return -ENOMEM; |
| 3089 | 3082 | ||
| 3090 | for_each_possible_cpu(cpu) { | 3083 | for_each_possible_cpu(cpu) { |
| 3091 | struct pool_workqueue *pwq = get_pwq(cpu, wq); | 3084 | struct pool_workqueue *pwq = |
| 3085 | per_cpu_ptr(wq->cpu_pwqs, cpu); | ||
| 3092 | 3086 | ||
| 3093 | pwq->pool = get_std_worker_pool(cpu, highpri); | 3087 | pwq->pool = get_std_worker_pool(cpu, highpri); |
| 3094 | list_add_tail(&pwq->pwqs_node, &wq->pwqs); | 3088 | list_add_tail(&pwq->pwqs_node, &wq->pwqs); |
| @@ -3343,7 +3337,12 @@ EXPORT_SYMBOL_GPL(workqueue_set_max_active); · 248 unmodified lines | |||
| 3343 | */ | 3337 | */ |
| 3344 | bool workqueue_congested(int cpu, struct workqueue_struct *wq) | 3338 | bool workqueue_congested(int cpu, struct workqueue_struct *wq) |
| 3345 | { | 3339 | { |
| 3346 | struct pool_workqueue *pwq = get_pwq(cpu, wq); | 3340 | struct pool_workqueue *pwq; |
| 3341 | |||
| 3342 | if (!(wq->flags & WQ_UNBOUND)) | ||
| 3343 | pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); | ||
| 3344 | else | ||
| 3345 | pwq = first_pwq(wq); | ||
| 3347 | 3346 | ||
| 3348 | return !list_empty(&pwq->delayed_works); | 3347 | return !list_empty(&pwq->delayed_works); |
| 3349 | } | 3348 | } |