Unbound pwqs (pool_workqueues) will be dynamically created and destroyed with the scheduled unbound workqueue w/ custom attributes support. This patch synchronizes pwq linking and unlinking against flush_workqueue() so that its operation isn't disturbed by pwqs coming and going. Linking and unlinking a pwq into wq->pwqs is now protected also by wq->flush_mutex and a new pwq's work_color is initialized to wq->work_color during linking. This ensures that pwqs changes don't disturb flush_workqueue() in progress and the new pwq's work coloring stays in sync with the rest of the workqueue. flush_mutex during unlinking isn't strictly necessary but it's simpler to do it anyway.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -122,6 +122,9 @@ enum { · 121 unmodified lines | |||
| 122 | * W: workqueue_lock protected. | 122 | * W: workqueue_lock protected. |
| 123 | * | 123 | * |
| 124 | * R: workqueue_lock protected for writes. Sched-RCU protected for reads. | 124 | * R: workqueue_lock protected for writes. Sched-RCU protected for reads. |
| 125 | * | ||
| 126 | * FR: wq->flush_mutex and workqueue_lock protected for writes. Sched-RCU | ||
| 127 | * protected for reads. | ||
| 125 | */ | 128 | */ |
| 126 | 129 | ||
| 127 | /* struct worker is defined in workqueue_internal.h */ | 130 | /* struct worker is defined in workqueue_internal.h */ |
| @@ -185,7 +188,7 @@ struct pool_workqueue { · 57 unmodified lines | |||
| 185 | int nr_active; /* L: nr of active works */ | 188 | int nr_active; /* L: nr of active works */ |
| 186 | int max_active; /* L: max active works */ | 189 | int max_active; /* L: max active works */ |
| 187 | struct list_head delayed_works; /* L: delayed works */ | 190 | struct list_head delayed_works; /* L: delayed works */ |
| 188 | struct list_head pwqs_node; /* R: node on wq->pwqs */ | 191 | struct list_head pwqs_node; /* FR: node on wq->pwqs */ |
| 189 | struct list_head mayday_node; /* W: node on wq->maydays */ | 192 | struct list_head mayday_node; /* W: node on wq->maydays */ |
| 190 | 193 | ||
| 191 | /* | 194 | /* |
| @@ -214,7 +217,7 @@ struct wq_flusher { · 22 unmodified lines | |||
| 214 | struct workqueue_struct { | 217 | struct workqueue_struct { |
| 215 | unsigned int flags; /* W: WQ_* flags */ | 218 | unsigned int flags; /* W: WQ_* flags */ |
| 216 | struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */ | 219 | struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */ |
| 217 | struct list_head pwqs; /* R: all pwqs of this wq */ | 220 | struct list_head pwqs; /* FR: all pwqs of this wq */ |
| 218 | struct list_head list; /* W: list of all workqueues */ | 221 | struct list_head list; /* W: list of all workqueues */ |
| 219 | 222 | ||
| 220 | struct mutex flush_mutex; /* protects wq flushing */ | 223 | struct mutex flush_mutex; /* protects wq flushing */ |
| @@ -3402,9 +3405,16 @@ static void pwq_unbound_release_workfn(struct work_struct *work) · 3181 unmodified lines | |||
| 3402 | if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) | 3405 | if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) |
| 3403 | return; | 3406 | return; |
| 3404 | 3407 | ||
| 3408 | /* | ||
| 3409 | * Unlink @pwq. Synchronization against flush_mutex isn't strictly | ||
| 3410 | * necessary on release but do it anyway. It's easier to verify | ||
| 3411 | * and consistent with the linking path. | ||
| 3412 | */ | ||
| 3413 | mutex_lock(&wq->flush_mutex); | ||
| 3405 | spin_lock_irq(&workqueue_lock); | 3414 | spin_lock_irq(&workqueue_lock); |
| 3406 | list_del_rcu(&pwq->pwqs_node); | 3415 | list_del_rcu(&pwq->pwqs_node); |
| 3407 | spin_unlock_irq(&workqueue_lock); | 3416 | spin_unlock_irq(&workqueue_lock); |
| 3417 | mutex_unlock(&wq->flush_mutex); | ||
| 3408 | 3418 | ||
| 3409 | put_unbound_pool(pool); | 3419 | put_unbound_pool(pool); |
| 3410 | call_rcu_sched(&pwq->rcu, rcu_free_pwq); | 3420 | call_rcu_sched(&pwq->rcu, rcu_free_pwq); |
| @@ -3432,7 +3442,18 @@ static void init_and_link_pwq(struct pool_workqueue *pwq, · 21 unmodified lines | |||
| 3432 | INIT_LIST_HEAD(&pwq->mayday_node); | 3442 | INIT_LIST_HEAD(&pwq->mayday_node); |
| 3433 | INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); | 3443 | INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); |
| 3434 | 3444 | ||
| 3445 | /* | ||
| 3446 | * Link @pwq and set the matching work_color. This is synchronized | ||
| 3447 | * with flush_mutex to avoid confusing flush_workqueue(). | ||
| 3448 | */ | ||
| 3449 | mutex_lock(&wq->flush_mutex); | ||
| 3450 | spin_lock_irq(&workqueue_lock); | ||
| 3451 | |||
| 3452 | pwq->work_color = wq->work_color; | ||
| 3435 | list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); | 3453 | list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); |
| 3454 | |||
| 3455 | spin_unlock_irq(&workqueue_lock); | ||
| 3456 | mutex_unlock(&wq->flush_mutex); | ||
| 3436 | } | 3457 | } |
| 3437 | 3458 | ||
| 3438 | static int alloc_and_link_pwqs(struct workqueue_struct *wq) | 3459 | static int alloc_and_link_pwqs(struct workqueue_struct *wq) |