Now that work_grab_pending() can always grab the PENDING bit without sleeping, the only thing that prevents allowing cancel_work_sync() of a BH work item from an atomic context is the flushing of the in-flight instance. When we're flushing a BH work item for cancel_work_sync(), we know that the work item is not queued and must be executing in a BH context, which means that it's safe to busy-wait for its completion from a non-hardirq atomic context. This patch updates __flush_work() so that it busy-waits when flushing a BH work item for cancel_work_sync(). might_sleep() is pushed from start_flush_work() to its callers - when operating on a BH work item, __cancel_work_sync() now enforces !in_hardirq() instead of might_sleep(). This allows cancel_work_sync() and disable_work() to be called from non-hardirq atomic contexts on BH work items. v3: In __flush_work(), test WORK_OFFQ_BH to tell whether a work item being canceled can be busy waited instead of making start_flush_work() return the pool. (Lai) v2: Lai pointed out that __flush_work() was accessing pool->flags outside the RCU critical section protecting the pool pointer. Fix it by testing and remembering the result inside the RCU critical section.
Showing the changes to kernel/workqueue.c (1 of 1 file). Show all
Unified · Split
| @@ -4105,8 +4105,6 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 4104 unmodified lines | ||
| 4105 | 4105 | struct pool_workqueue *pwq; |
| 4106 | 4106 | struct workqueue_struct *wq; |
| 4107 | 4107 | |
| 4108 | − might_sleep(); | |
| 4109 | − | |
| 4110 | 4108 | rcu_read_lock(); |
| 4111 | 4109 | pool = get_work_pool(work); |
| 4112 | 4110 | if (!pool) { |
| @@ -4158,6 +4156,7 @@ already_gone: · 45 unmodified lines | ||
| 4158 | 4156 | static bool __flush_work(struct work_struct *work, bool from_cancel) |
| 4159 | 4157 | { |
| 4160 | 4158 | struct wq_barrier barr; |
| 4159 | + unsigned long data; | |
| 4161 | 4160 | |
| 4162 | 4161 | if (WARN_ON(!wq_online)) |
| 4163 | 4162 | return false; |
| @@ -4165,13 +4164,41 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 1 unmodified line | ||
| 4165 | 4164 | if (WARN_ON(!work->func)) |
| 4166 | 4165 | return false; |
| 4167 | 4166 | |
| 4168 | − if (start_flush_work(work, &barr, from_cancel)) { | |
| 4169 | − wait_for_completion(&barr.done); | |
| 4170 | − destroy_work_on_stack(&barr.work); | |
| 4171 | − return true; | |
| 4172 | − } else { | |
| 4167 | + if (!start_flush_work(work, &barr, from_cancel)) | |
| 4173 | 4168 | return false; |
| 4169 | + | |
| 4170 | + /* | |
| 4171 | + * start_flush_work() returned %true. If @from_cancel is set, we know | |
| 4172 | + * that @work must have been executing during start_flush_work() and | |
| 4173 | + * can't currently be queued. Its data must contain OFFQ bits. If @work | |
| 4174 | + * was queued on a BH workqueue, we also know that it was running in the | |
| 4175 | + * BH context and thus can be busy-waited. | |
| 4176 | + */ | |
| 4177 | + data = *work_data_bits(work); | |
| 4178 | + if (from_cancel && | |
| 4179 | + !WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_BH)) { | |
| 4180 | + /* | |
| 4181 | + * On RT, prevent a live lock when %current preempted soft | |
| 4182 | + * interrupt processing or prevents ksoftirqd from running by | |
| 4183 | + * keeping flipping BH. If the BH work item runs on a different | |
| 4184 | + * CPU then this has no effect other than doing the BH | |
| 4185 | + * disable/enable dance for nothing. This is copied from | |
| 4186 | + * kernel/softirq.c::tasklet_unlock_spin_wait(). | |
| 4187 | + */ | |
| 4188 | + while (!try_wait_for_completion(&barr.done)) { | |
| 4189 | + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { | |
| 4190 | + local_bh_disable(); | |
| 4191 | + local_bh_enable(); | |
| 4192 | + } else { | |
| 4193 | + cpu_relax(); | |
| 4194 | + } | |
| 4195 | + } | |
| 4196 | + } else { | |
| 4197 | + wait_for_completion(&barr.done); | |
| 4174 | 4198 | } |
| 4199 | + | |
| 4200 | + destroy_work_on_stack(&barr.work); | |
| 4201 | + return true; | |
| 4175 | 4202 | } |
| 4176 | 4203 | |
| 4177 | 4204 | /** |
| @@ -4187,6 +4214,7 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 9 unmodified lines | ||
| 4187 | 4214 | */ |
| 4188 | 4215 | bool flush_work(struct work_struct *work) |
| 4189 | 4216 | { |
| 4217 | + might_sleep(); | |
| 4190 | 4218 | return __flush_work(work, false); |
| 4191 | 4219 | } |
| 4192 | 4220 | EXPORT_SYMBOL_GPL(flush_work); |
| @@ -4276,6 +4304,11 @@ static bool __cancel_work_sync(struct work_struct *work, u32 cflags) · 83 unmodified lines | ||
| 4276 | 4304 | |
| 4277 | 4305 | ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE); |
| 4278 | 4306 | |
| 4307 | + if (*work_data_bits(work) & WORK_OFFQ_BH) | |
| 4308 | + WARN_ON_ONCE(in_hardirq()); | |
| 4309 | + else | |
| 4310 | + might_sleep(); | |
| 4311 | + | |
| 4279 | 4312 | /* |
| 4280 | 4313 | * Skip __flush_work() during early boot when we know that @work isn't |
| 4281 | 4314 | * executing. This allows canceling during early boot. |
| @@ -4302,19 +4335,19 @@ EXPORT_SYMBOL(cancel_work); · 20 unmodified lines | ||
| 4302 | 4335 | * cancel_work_sync - cancel a work and wait for it to finish |
| 4303 | 4336 | * @work: the work to cancel |
| 4304 | 4337 | * |
| 4305 | − * Cancel @work and wait for its execution to finish. This function | |
| 4306 | − * can be used even if the work re-queues itself or migrates to | |
| 4307 | − * another workqueue. On return from this function, @work is | |
| 4308 | − * guaranteed to be not pending or executing on any CPU. | |
| 4338 | + * Cancel @work and wait for its execution to finish. This function can be used | |
| 4339 | + * even if the work re-queues itself or migrates to another workqueue. On return | |
| 4340 | + * from this function, @work is guaranteed to be not pending or executing on any | |
| 4341 | + * CPU as long as there aren't racing enqueues. | |
| 4309 | 4342 | * |
| 4310 | − * cancel_work_sync(&delayed_work->work) must not be used for | |
| 4311 | − * delayed_work's. Use cancel_delayed_work_sync() instead. | |
| 4343 | + * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's. | |
| 4344 | + * Use cancel_delayed_work_sync() instead. | |
| 4312 | 4345 | * |
| 4313 | − * The caller must ensure that the workqueue on which @work was last | |
| 4314 | − * queued can't be destroyed before this function returns. | |
| 4346 | + * Must be called from a sleepable context if @work was last queued on a non-BH | |
| 4347 | + * workqueue. Can also be called from non-hardirq atomic contexts including BH | |
| 4348 | + * if @work was last queued on a BH workqueue. | |
| 4315 | 4349 | * |
| 4316 | − * Return: | |
| 4317 | − * %true if @work was pending, %false otherwise. | |
| 4350 | + * Returns %true if @work was pending, %false otherwise. | |
| 4318 | 4351 | */ |
| 4319 | 4352 | bool cancel_work_sync(struct work_struct *work) |
| 4320 | 4353 | { |
| @@ -4384,8 +4417,11 @@ EXPORT_SYMBOL_GPL(disable_work); · 63 unmodified lines | ||
| 4384 | 4417 | * Similar to disable_work() but also wait for @work to finish if currently |
| 4385 | 4418 | * executing. |
| 4386 | 4419 | * |
| 4387 | − * Must be called from a sleepable context. Returns %true if @work was pending, | |
| 4388 | − * %false otherwise. | |
| 4420 | + * Must be called from a sleepable context if @work was last queued on a non-BH | |
| 4421 | + * workqueue. Can also be called from non-hardirq atomic contexts including BH | |
| 4422 | + * if @work was last queued on a BH workqueue. | |
| 4423 | + * | |
| 4424 | + * Returns %true if @work was pending, %false otherwise. | |
| 4389 | 4425 | */ |
| 4390 | 4426 | bool disable_work_sync(struct work_struct *work) |
| 4391 | 4427 | { |