Unbound pwqs (pool_workqueues) will be dynamically created and destroyed with the scheduled unbound workqueue w/ custom attributes support. This patch synchronizes pwq linking and unlinking against flush_workqueue() so that its operation isn't disturbed by pwqs coming and going. Linking and unlinking a pwq into wq->pwqs is now protected also by wq->flush_mutex and a new pwq's work_color is initialized to wq->work_color during linking. This ensures that pwqs changes don't disturb flush_workqueue() in progress and the new pwq's work coloring stays in sync with the rest of the workqueue. flush_mutex during unlinking isn't strictly necessary but it's simpler to do it anyway.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -122,6 +122,9 @@ enum { · 121 unmodified lines | ||
| 122 | 122 | * W: workqueue_lock protected. |
| 123 | 123 | * |
| 124 | 124 | * R: workqueue_lock protected for writes. Sched-RCU protected for reads. |
| 125 | + * | |
| 126 | + * FR: wq->flush_mutex and workqueue_lock protected for writes. Sched-RCU | |
| 127 | + * protected for reads. | |
| 125 | 128 | */ |
| 126 | 129 | |
| 127 | 130 | /* struct worker is defined in workqueue_internal.h */ |
| @@ -185,7 +188,7 @@ struct pool_workqueue { · 57 unmodified lines | ||
| 185 | 188 | int nr_active; /* L: nr of active works */ |
| 186 | 189 | int max_active; /* L: max active works */ |
| 187 | 190 | struct list_head delayed_works; /* L: delayed works */ |
| 188 | − struct list_head pwqs_node; /* R: node on wq->pwqs */ | |
| 191 | + struct list_head pwqs_node; /* FR: node on wq->pwqs */ | |
| 189 | 192 | struct list_head mayday_node; /* W: node on wq->maydays */ |
| 190 | 193 | |
| 191 | 194 | /* |
| @@ -214,7 +217,7 @@ struct wq_flusher { · 22 unmodified lines | ||
| 214 | 217 | struct workqueue_struct { |
| 215 | 218 | unsigned int flags; /* W: WQ_* flags */ |
| 216 | 219 | struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwq's */ |
| 217 | − struct list_head pwqs; /* R: all pwqs of this wq */ | |
| 220 | + struct list_head pwqs; /* FR: all pwqs of this wq */ | |
| 218 | 221 | struct list_head list; /* W: list of all workqueues */ |
| 219 | 222 | |
| 220 | 223 | struct mutex flush_mutex; /* protects wq flushing */ |
| @@ -3402,9 +3405,16 @@ static void pwq_unbound_release_workfn(struct work_struct *work) · 3181 unmodified lines | ||
| 3402 | 3405 | if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) |
| 3403 | 3406 | return; |
| 3404 | 3407 | |
| 3408 | + /* | |
| 3409 | + * Unlink @pwq. Synchronization against flush_mutex isn't strictly | |
| 3410 | + * necessary on release but do it anyway. It's easier to verify | |
| 3411 | + * and consistent with the linking path. | |
| 3412 | + */ | |
| 3413 | + mutex_lock(&wq->flush_mutex); | |
| 3405 | 3414 | spin_lock_irq(&workqueue_lock); |
| 3406 | 3415 | list_del_rcu(&pwq->pwqs_node); |
| 3407 | 3416 | spin_unlock_irq(&workqueue_lock); |
| 3417 | + mutex_unlock(&wq->flush_mutex); | |
| 3408 | 3418 | |
| 3409 | 3419 | put_unbound_pool(pool); |
| 3410 | 3420 | call_rcu_sched(&pwq->rcu, rcu_free_pwq); |
| @@ -3432,7 +3442,18 @@ static void init_and_link_pwq(struct pool_workqueue *pwq, · 21 unmodified lines | ||
| 3432 | 3442 | INIT_LIST_HEAD(&pwq->mayday_node); |
| 3433 | 3443 | INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); |
| 3434 | 3444 | |
| 3445 | + /* | |
| 3446 | + * Link @pwq and set the matching work_color. This is synchronized | |
| 3447 | + * with flush_mutex to avoid confusing flush_workqueue(). | |
| 3448 | + */ | |
| 3449 | + mutex_lock(&wq->flush_mutex); | |
| 3450 | + spin_lock_irq(&workqueue_lock); | |
| 3451 | + | |
| 3452 | + pwq->work_color = wq->work_color; | |
| 3435 | 3453 | list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs); |
| 3454 | + | |
| 3455 | + spin_unlock_irq(&workqueue_lock); | |
| 3456 | + mutex_unlock(&wq->flush_mutex); | |
| 3436 | 3457 | } |
| 3437 | 3458 | |
| 3438 | 3459 | static int alloc_and_link_pwqs(struct workqueue_struct *wq) |