worker_idr has the iteration (iterating for attached workers) and worker ID duties. These two duties don't have to be tied together. We can separate them and use a list for tracking attached workers and iteration. Before this separation, it wasn't possible to add rescuer workers to worker_idr due to rescuer workers couldn't allocate ID dynamically because ID-allocation depends on memory-allocation, which rescuer can't depend on. After separation, we can easily add the rescuer workers to the list for iteration without any memory-allocation. It is required when we attach the rescuer worker to the pool in later patch. tj: Minor description updates.
Showing the changes to kernel/workqueue.c (1 of 2 files). Show all
Unified · Split
| @@ -161,7 +161,8 @@ struct worker_pool { · 160 unmodified lines | |||
| 161 | /* see manage_workers() for details on the two manager mutexes */ | 161 | /* see manage_workers() for details on the two manager mutexes */ |
| 162 | struct mutex manager_arb; /* manager arbitration */ | 162 | struct mutex manager_arb; /* manager arbitration */ |
| 163 | struct mutex manager_mutex; /* manager exclusion */ | 163 | struct mutex manager_mutex; /* manager exclusion */ |
| 164 | struct idr worker_idr; /* M: worker IDs and iteration */ | 164 | struct idr worker_idr; /* M: worker IDs */ |
| 165 | struct list_head workers; /* M: attached workers */ | ||
| 165 | struct completion *detach_completion; /* all workers detached */ | 166 | struct completion *detach_completion; /* all workers detached */ |
| 166 | 167 | ||
| 167 | struct workqueue_attrs *attrs; /* I: worker attributes */ | 168 | struct workqueue_attrs *attrs; /* I: worker attributes */ |
| @@ -363,7 +364,6 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 195 unmodified lines | |||
| 363 | /** | 364 | /** |
| 364 | * for_each_pool_worker - iterate through all workers of a worker_pool | 365 | * for_each_pool_worker - iterate through all workers of a worker_pool |
| 365 | * @worker: iteration cursor | 366 | * @worker: iteration cursor |
| 366 | * @wi: integer used for iteration | ||
| 367 | * @pool: worker_pool to iterate workers of | 367 | * @pool: worker_pool to iterate workers of |
| 368 | * | 368 | * |
| 369 | * This must be called with @pool->manager_mutex. | 369 | * This must be called with @pool->manager_mutex. |
| @@ -371,8 +371,8 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 1 unmodified line | |||
| 371 | * The if/else clause exists only for the lockdep assertion and can be | 371 | * The if/else clause exists only for the lockdep assertion and can be |
| 372 | * ignored. | 372 | * ignored. |
| 373 | */ | 373 | */ |
| 374 | #define for_each_pool_worker(worker, wi, pool) \ | 374 | #define for_each_pool_worker(worker, pool) \ |
| 375 | idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \ | 375 | list_for_each_entry((worker), &(pool)->workers, node) \ |
| 376 | if (({ lockdep_assert_held(&pool->manager_mutex); false; })) { } \ | 376 | if (({ lockdep_assert_held(&pool->manager_mutex); false; })) { } \ |
| 377 | else | 377 | else |
| 378 | 378 | ||
| @@ -1674,6 +1674,7 @@ static struct worker *alloc_worker(void) · 1295 unmodified lines | |||
| 1674 | if (worker) { | 1674 | if (worker) { |
| 1675 | INIT_LIST_HEAD(&worker->entry); | 1675 | INIT_LIST_HEAD(&worker->entry); |
| 1676 | INIT_LIST_HEAD(&worker->scheduled); | 1676 | INIT_LIST_HEAD(&worker->scheduled); |
| 1677 | INIT_LIST_HEAD(&worker->node); | ||
| 1677 | /* on creation a worker is in !idle && prep state */ | 1678 | /* on creation a worker is in !idle && prep state */ |
| 1678 | worker->flags = WORKER_PREP; | 1679 | worker->flags = WORKER_PREP; |
| 1679 | } | 1680 | } |
| @@ -1696,7 +1697,8 @@ static void worker_detach_from_pool(struct worker *worker, · 16 unmodified lines | |||
| 1696 | 1697 | ||
| 1697 | mutex_lock(&pool->manager_mutex); | 1698 | mutex_lock(&pool->manager_mutex); |
| 1698 | idr_remove(&pool->worker_idr, worker->id); | 1699 | idr_remove(&pool->worker_idr, worker->id); |
| 1699 | if (idr_is_empty(&pool->worker_idr)) | 1700 | list_del(&worker->node); |
| 1701 | if (list_empty(&pool->workers)) | ||
| 1700 | detach_completion = pool->detach_completion; | 1702 | detach_completion = pool->detach_completion; |
| 1701 | mutex_unlock(&pool->manager_mutex); | 1703 | mutex_unlock(&pool->manager_mutex); |
| 1702 | 1704 | ||
| @@ -1772,6 +1774,8 @@ static struct worker *create_worker(struct worker_pool *pool) · 69 unmodified lines | |||
| 1772 | 1774 | ||
| 1773 | /* successful, commit the pointer to idr */ | 1775 | /* successful, commit the pointer to idr */ |
| 1774 | idr_replace(&pool->worker_idr, worker, worker->id); | 1776 | idr_replace(&pool->worker_idr, worker, worker->id); |
| 1777 | /* successful, attach the worker to the pool */ | ||
| 1778 | list_add_tail(&worker->node, &pool->workers); | ||
| 1775 | 1779 | ||
| 1776 | return worker; | 1780 | return worker; |
| 1777 | 1781 | ||
| @@ -3483,6 +3487,7 @@ static int init_worker_pool(struct worker_pool *pool) · 1705 unmodified lines | |||
| 3483 | mutex_init(&pool->manager_arb); | 3487 | mutex_init(&pool->manager_arb); |
| 3484 | mutex_init(&pool->manager_mutex); | 3488 | mutex_init(&pool->manager_mutex); |
| 3485 | idr_init(&pool->worker_idr); | 3489 | idr_init(&pool->worker_idr); |
| 3490 | INIT_LIST_HEAD(&pool->workers); | ||
| 3486 | 3491 | ||
| 3487 | INIT_HLIST_NODE(&pool->hash_node); | 3492 | INIT_HLIST_NODE(&pool->hash_node); |
| 3488 | pool->refcnt = 1; | 3493 | pool->refcnt = 1; |
| @@ -3548,7 +3553,7 @@ static void put_unbound_pool(struct worker_pool *pool) · 59 unmodified lines | |||
| 3548 | spin_unlock_irq(&pool->lock); | 3553 | spin_unlock_irq(&pool->lock); |
| 3549 | 3554 | ||
| 3550 | mutex_lock(&pool->manager_mutex); | 3555 | mutex_lock(&pool->manager_mutex); |
| 3551 | if (!idr_is_empty(&pool->worker_idr)) | 3556 | if (!list_empty(&pool->workers)) |
| 3552 | pool->detach_completion = &detach_completion; | 3557 | pool->detach_completion = &detach_completion; |
| 3553 | mutex_unlock(&pool->manager_mutex); | 3558 | mutex_unlock(&pool->manager_mutex); |
| 3554 | 3559 | ||
| @@ -4533,7 +4538,6 @@ static void wq_unbind_fn(struct work_struct *work) · 978 unmodified lines | |||
| 4533 | int cpu = smp_processor_id(); | 4538 | int cpu = smp_processor_id(); |
| 4534 | struct worker_pool *pool; | 4539 | struct worker_pool *pool; |
| 4535 | struct worker *worker; | 4540 | struct worker *worker; |
| 4536 | int wi; | ||
| 4537 | 4541 | ||
| 4538 | for_each_cpu_worker_pool(pool, cpu) { | 4542 | for_each_cpu_worker_pool(pool, cpu) { |
| 4539 | WARN_ON_ONCE(cpu != smp_processor_id()); | 4543 | WARN_ON_ONCE(cpu != smp_processor_id()); |
| @@ -4548,7 +4552,7 @@ static void wq_unbind_fn(struct work_struct *work) · 8 unmodified lines | |||
| 4548 | * before the last CPU down must be on the cpu. After | 4552 | * before the last CPU down must be on the cpu. After |
| 4549 | * this, they may become diasporas. | 4553 | * this, they may become diasporas. |
| 4550 | */ | 4554 | */ |
| 4551 | for_each_pool_worker(worker, wi, pool) | 4555 | for_each_pool_worker(worker, pool) |
| 4552 | worker->flags |= WORKER_UNBOUND; | 4556 | worker->flags |= WORKER_UNBOUND; |
| 4553 | 4557 | ||
| 4554 | pool->flags |= POOL_DISASSOCIATED; | 4558 | pool->flags |= POOL_DISASSOCIATED; |
| @@ -4594,7 +4598,6 @@ static void wq_unbind_fn(struct work_struct *work) · 39 unmodified lines | |||
| 4594 | static void rebind_workers(struct worker_pool *pool) | 4598 | static void rebind_workers(struct worker_pool *pool) |
| 4595 | { | 4599 | { |
| 4596 | struct worker *worker; | 4600 | struct worker *worker; |
| 4597 | int wi; | ||
| 4598 | 4601 | ||
| 4599 | lockdep_assert_held(&pool->manager_mutex); | 4602 | lockdep_assert_held(&pool->manager_mutex); |
| 4600 | 4603 | ||
| @@ -4605,13 +4608,13 @@ static void rebind_workers(struct worker_pool *pool) · 4 unmodified lines | |||
| 4605 | * of all workers first and then clear UNBOUND. As we're called | 4608 | * of all workers first and then clear UNBOUND. As we're called |
| 4606 | * from CPU_ONLINE, the following shouldn't fail. | 4609 | * from CPU_ONLINE, the following shouldn't fail. |
| 4607 | */ | 4610 | */ |
| 4608 | for_each_pool_worker(worker, wi, pool) | 4611 | for_each_pool_worker(worker, pool) |
| 4609 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, | 4612 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, |
| 4610 | pool->attrs->cpumask) < 0); | 4613 | pool->attrs->cpumask) < 0); |
| 4611 | 4614 | ||
| 4612 | spin_lock_irq(&pool->lock); | 4615 | spin_lock_irq(&pool->lock); |
| 4613 | 4616 | ||
| 4614 | for_each_pool_worker(worker, wi, pool) { | 4617 | for_each_pool_worker(worker, pool) { |
| 4615 | unsigned int worker_flags = worker->flags; | 4618 | unsigned int worker_flags = worker->flags; |
| 4616 | 4619 | ||
| 4617 | /* | 4620 | /* |
| @@ -4663,7 +4666,6 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 45 unmodified lines | |||
| 4663 | { | 4666 | { |
| 4664 | static cpumask_t cpumask; | 4667 | static cpumask_t cpumask; |
| 4665 | struct worker *worker; | 4668 | struct worker *worker; |
| 4666 | int wi; | ||
| 4667 | 4669 | ||
| 4668 | lockdep_assert_held(&pool->manager_mutex); | 4670 | lockdep_assert_held(&pool->manager_mutex); |
| 4669 | 4671 | ||
| @@ -4677,7 +4679,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 7 unmodified lines | |||
| 4677 | return; | 4679 | return; |
| 4678 | 4680 | ||
| 4679 | /* as we're called from CPU_ONLINE, the following shouldn't fail */ | 4681 | /* as we're called from CPU_ONLINE, the following shouldn't fail */ |
| 4680 | for_each_pool_worker(worker, wi, pool) | 4682 | for_each_pool_worker(worker, pool) |
| 4681 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, | 4683 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, |
| 4682 | pool->attrs->cpumask) < 0); | 4684 | pool->attrs->cpumask) < 0); |
| 4683 | } | 4685 | } |