__queue_work() is the enqueue hot path: it inserts the work item and calls kick_pool() while holding pool->lock. kick_pool() ends in a wakeup, which takes the target task's rq->lock, so rq->lock nests under pool->lock on every enqueue that wakes a worker on a contended unbound pool. Use kick_pool_pick() to select and claim the worker under pool->lock and issue the wakeup with wake_up_process() right after dropping the lock.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -2302,6 +2302,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, · 2301 unmodified lines | ||
| 2302 | 2302 | { |
| 2303 | 2303 | struct pool_workqueue *pwq; |
| 2304 | 2304 | struct worker_pool *last_pool, *pool; |
| 2305 | + struct task_struct *wake_task = NULL; | |
| 2305 | 2306 | unsigned int work_flags; |
| 2306 | 2307 | unsigned int req_cpu = cpu; |
| 2307 | 2308 | |
| @@ -2424,7 +2425,7 @@ retry: · 116 unmodified lines | ||
| 2424 | 2425 | |
| 2425 | 2426 | trace_workqueue_activate_work(work); |
| 2426 | 2427 | insert_work(pwq, work, &pool->worklist, work_flags); |
| 2427 | − kick_pool(pool); | |
| 2428 | + kick_pool_pick(pool, &wake_task); | |
| 2428 | 2429 | } else { |
| 2429 | 2430 | work_flags |= WORK_STRUCT_INACTIVE; |
| 2430 | 2431 | insert_work(pwq, work, &pwq->inactive_works, work_flags); |
| @@ -2432,6 +2433,8 @@ retry: · 1 unmodified line | ||
| 2432 | 2433 | |
| 2433 | 2434 | out: |
| 2434 | 2435 | raw_spin_unlock(&pool->lock); |
| 2436 | + if (wake_task) | |
| 2437 | + wake_up_process(wake_task); | |
| 2435 | 2438 | rcu_read_unlock(); |
| 2436 | 2439 | } |
| 2437 | 2440 | |