worker_idr has the iteration (iterating for attached workers) and worker ID duties. These two duties don't have to be tied together. We can separate them and use a list for tracking attached workers and iteration. Before this separation, it wasn't possible to add rescuer workers to worker_idr due to rescuer workers couldn't allocate ID dynamically because ID-allocation depends on memory-allocation, which rescuer can't depend on. After separation, we can easily add the rescuer workers to the list for iteration without any memory-allocation. It is required when we attach the rescuer worker to the pool in later patch. tj: Minor description updates.
Showing the changes to kernel/workqueue.c (1 of 2 files). Show all
Unified · Split
| @@ -161,7 +161,8 @@ struct worker_pool { · 160 unmodified lines | ||
| 161 | 161 | /* see manage_workers() for details on the two manager mutexes */ |
| 162 | 162 | struct mutex manager_arb; /* manager arbitration */ |
| 163 | 163 | struct mutex manager_mutex; /* manager exclusion */ |
| 164 | − struct idr worker_idr; /* M: worker IDs and iteration */ | |
| 164 | + struct idr worker_idr; /* M: worker IDs */ | |
| 165 | + struct list_head workers; /* M: attached workers */ | |
| 165 | 166 | struct completion *detach_completion; /* all workers detached */ |
| 166 | 167 | |
| 167 | 168 | struct workqueue_attrs *attrs; /* I: worker attributes */ |
| @@ -363,7 +364,6 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 195 unmodified lines | ||
| 363 | 364 | /** |
| 364 | 365 | * for_each_pool_worker - iterate through all workers of a worker_pool |
| 365 | 366 | * @worker: iteration cursor |
| 366 | − * @wi: integer used for iteration | |
| 367 | 367 | * @pool: worker_pool to iterate workers of |
| 368 | 368 | * |
| 369 | 369 | * This must be called with @pool->manager_mutex. |
| @@ -371,8 +371,8 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to, · 1 unmodified line | ||
| 371 | 371 | * The if/else clause exists only for the lockdep assertion and can be |
| 372 | 372 | * ignored. |
| 373 | 373 | */ |
| 374 | −#define for_each_pool_worker(worker, wi, pool) \ | |
| 375 | − idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \ | |
| 374 | +#define for_each_pool_worker(worker, pool) \ | |
| 375 | + list_for_each_entry((worker), &(pool)->workers, node) \ | |
| 376 | 376 | if (({ lockdep_assert_held(&pool->manager_mutex); false; })) { } \ |
| 377 | 377 | else |
| 378 | 378 | |
| @@ -1674,6 +1674,7 @@ static struct worker *alloc_worker(void) · 1295 unmodified lines | ||
| 1674 | 1674 | if (worker) { |
| 1675 | 1675 | INIT_LIST_HEAD(&worker->entry); |
| 1676 | 1676 | INIT_LIST_HEAD(&worker->scheduled); |
| 1677 | + INIT_LIST_HEAD(&worker->node); | |
| 1677 | 1678 | /* on creation a worker is in !idle && prep state */ |
| 1678 | 1679 | worker->flags = WORKER_PREP; |
| 1679 | 1680 | } |
| @@ -1696,7 +1697,8 @@ static void worker_detach_from_pool(struct worker *worker, · 16 unmodified lines | ||
| 1696 | 1697 | |
| 1697 | 1698 | mutex_lock(&pool->manager_mutex); |
| 1698 | 1699 | idr_remove(&pool->worker_idr, worker->id); |
| 1699 | − if (idr_is_empty(&pool->worker_idr)) | |
| 1700 | + list_del(&worker->node); | |
| 1701 | + if (list_empty(&pool->workers)) | |
| 1700 | 1702 | detach_completion = pool->detach_completion; |
| 1701 | 1703 | mutex_unlock(&pool->manager_mutex); |
| 1702 | 1704 | |
| @@ -1772,6 +1774,8 @@ static struct worker *create_worker(struct worker_pool *pool) · 69 unmodified lines | ||
| 1772 | 1774 | |
| 1773 | 1775 | /* successful, commit the pointer to idr */ |
| 1774 | 1776 | idr_replace(&pool->worker_idr, worker, worker->id); |
| 1777 | + /* successful, attach the worker to the pool */ | |
| 1778 | + list_add_tail(&worker->node, &pool->workers); | |
| 1775 | 1779 | |
| 1776 | 1780 | return worker; |
| 1777 | 1781 | |
| @@ -3483,6 +3487,7 @@ static int init_worker_pool(struct worker_pool *pool) · 1705 unmodified lines | ||
| 3483 | 3487 | mutex_init(&pool->manager_arb); |
| 3484 | 3488 | mutex_init(&pool->manager_mutex); |
| 3485 | 3489 | idr_init(&pool->worker_idr); |
| 3490 | + INIT_LIST_HEAD(&pool->workers); | |
| 3486 | 3491 | |
| 3487 | 3492 | INIT_HLIST_NODE(&pool->hash_node); |
| 3488 | 3493 | pool->refcnt = 1; |
| @@ -3548,7 +3553,7 @@ static void put_unbound_pool(struct worker_pool *pool) · 59 unmodified lines | ||
| 3548 | 3553 | spin_unlock_irq(&pool->lock); |
| 3549 | 3554 | |
| 3550 | 3555 | mutex_lock(&pool->manager_mutex); |
| 3551 | − if (!idr_is_empty(&pool->worker_idr)) | |
| 3556 | + if (!list_empty(&pool->workers)) | |
| 3552 | 3557 | pool->detach_completion = &detach_completion; |
| 3553 | 3558 | mutex_unlock(&pool->manager_mutex); |
| 3554 | 3559 | |
| @@ -4533,7 +4538,6 @@ static void wq_unbind_fn(struct work_struct *work) · 978 unmodified lines | ||
| 4533 | 4538 | int cpu = smp_processor_id(); |
| 4534 | 4539 | struct worker_pool *pool; |
| 4535 | 4540 | struct worker *worker; |
| 4536 | − int wi; | |
| 4537 | 4541 | |
| 4538 | 4542 | for_each_cpu_worker_pool(pool, cpu) { |
| 4539 | 4543 | WARN_ON_ONCE(cpu != smp_processor_id()); |
| @@ -4548,7 +4552,7 @@ static void wq_unbind_fn(struct work_struct *work) · 8 unmodified lines | ||
| 4548 | 4552 | * before the last CPU down must be on the cpu. After |
| 4549 | 4553 | * this, they may become diasporas. |
| 4550 | 4554 | */ |
| 4551 | − for_each_pool_worker(worker, wi, pool) | |
| 4555 | + for_each_pool_worker(worker, pool) | |
| 4552 | 4556 | worker->flags |= WORKER_UNBOUND; |
| 4553 | 4557 | |
| 4554 | 4558 | pool->flags |= POOL_DISASSOCIATED; |
| @@ -4594,7 +4598,6 @@ static void wq_unbind_fn(struct work_struct *work) · 39 unmodified lines | ||
| 4594 | 4598 | static void rebind_workers(struct worker_pool *pool) |
| 4595 | 4599 | { |
| 4596 | 4600 | struct worker *worker; |
| 4597 | − int wi; | |
| 4598 | 4601 | |
| 4599 | 4602 | lockdep_assert_held(&pool->manager_mutex); |
| 4600 | 4603 | |
| @@ -4605,13 +4608,13 @@ static void rebind_workers(struct worker_pool *pool) · 4 unmodified lines | ||
| 4605 | 4608 | * of all workers first and then clear UNBOUND. As we're called |
| 4606 | 4609 | * from CPU_ONLINE, the following shouldn't fail. |
| 4607 | 4610 | */ |
| 4608 | − for_each_pool_worker(worker, wi, pool) | |
| 4611 | + for_each_pool_worker(worker, pool) | |
| 4609 | 4612 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, |
| 4610 | 4613 | pool->attrs->cpumask) < 0); |
| 4611 | 4614 | |
| 4612 | 4615 | spin_lock_irq(&pool->lock); |
| 4613 | 4616 | |
| 4614 | − for_each_pool_worker(worker, wi, pool) { | |
| 4617 | + for_each_pool_worker(worker, pool) { | |
| 4615 | 4618 | unsigned int worker_flags = worker->flags; |
| 4616 | 4619 | |
| 4617 | 4620 | /* |
| @@ -4663,7 +4666,6 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 45 unmodified lines | ||
| 4663 | 4666 | { |
| 4664 | 4667 | static cpumask_t cpumask; |
| 4665 | 4668 | struct worker *worker; |
| 4666 | − int wi; | |
| 4667 | 4669 | |
| 4668 | 4670 | lockdep_assert_held(&pool->manager_mutex); |
| 4669 | 4671 | |
| @@ -4677,7 +4679,7 @@ static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu) · 7 unmodified lines | ||
| 4677 | 4679 | return; |
| 4678 | 4680 | |
| 4679 | 4681 | /* as we're called from CPU_ONLINE, the following shouldn't fail */ |
| 4680 | − for_each_pool_worker(worker, wi, pool) | |
| 4682 | + for_each_pool_worker(worker, pool) | |
| 4681 | 4683 | WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, |
| 4682 | 4684 | pool->attrs->cpumask) < 0); |
| 4683 | 4685 | } |