History

workqueue: Allow cancel_work_sync() and disable_work() from atomic contexts on BH work items

Now that work_grab_pending() can always grab the PENDING bit without sleeping, the only thing that prevents allowing cancel_work_sync() of a BH work item from an atomic context is the flushing of the in-flight instance. When we're flushing a BH work item for cancel_work_sync(), we know that the work item is not queued and must be executing in a BH context, which means that it's safe to busy-wait for its completion from a non-hardirq atomic context. This patch updates __flush_work() so that it busy-waits when flushing a BH work item for cancel_work_sync(). might_sleep() is pushed from start_flush_work() to its callers - when operating on a BH work item, __cancel_work_sync() now enforces !in_hardirq() instead of might_sleep(). This allows cancel_work_sync() and disable_work() to be called from non-hardirq atomic contexts on BH work items. v3: In __flush_work(), test WORK_OFFQ_BH to tell whether a work item being canceled can be busy waited instead of making start_flush_work() return the pool. (Lai) v2: Lai pointed out that __flush_work() was accessing pool->flags outside the RCU critical section protecting the pool pointer. Fix it by testing and remembering the result inside the RCU critical section.

Tejun Heo committed 2 years ago

Commit 134874e1 parent 456a78e1 file · +55 −19Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+55−19
@@ -4105,8 +4105,6 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr, · 4104 unmodified lines
4105 struct pool_workqueue *pwq;4105 struct pool_workqueue *pwq;
4106 struct workqueue_struct *wq;4106 struct workqueue_struct *wq;
4107 4107
4108 might_sleep();
4109
4110 rcu_read_lock();4108 rcu_read_lock();
4111 pool = get_work_pool(work);4109 pool = get_work_pool(work);
4112 if (!pool) {4110 if (!pool) {
@@ -4158,6 +4156,7 @@ already_gone: · 45 unmodified lines
4158static bool __flush_work(struct work_struct *work, bool from_cancel)4156static bool __flush_work(struct work_struct *work, bool from_cancel)
4159{4157{
4160 struct wq_barrier barr;4158 struct wq_barrier barr;
4159 unsigned long data;
4161 4160
4162 if (WARN_ON(!wq_online))4161 if (WARN_ON(!wq_online))
4163 return false;4162 return false;
@@ -4165,13 +4164,41 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 1 unmodified line
4165 if (WARN_ON(!work->func))4164 if (WARN_ON(!work->func))
4166 return false;4165 return false;
4167 4166
4168 if (start_flush_work(work, &barr, from_cancel)) {4167 if (!start_flush_work(work, &barr, from_cancel))
4169 wait_for_completion(&barr.done);
4170 destroy_work_on_stack(&barr.work);
4171 return true;
4172 } else {
4173 return false;4168 return false;
4169
4170 /*
4171 * start_flush_work() returned %true. If @from_cancel is set, we know
4172 * that @work must have been executing during start_flush_work() and
4173 * can't currently be queued. Its data must contain OFFQ bits. If @work
4174 * was queued on a BH workqueue, we also know that it was running in the
4175 * BH context and thus can be busy-waited.
4176 */
4177 data = *work_data_bits(work);
4178 if (from_cancel &&
4179 !WARN_ON_ONCE(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_BH)) {
4180 /*
4181 * On RT, prevent a live lock when %current preempted soft
4182 * interrupt processing or prevents ksoftirqd from running by
4183 * keeping flipping BH. If the BH work item runs on a different
4184 * CPU then this has no effect other than doing the BH
4185 * disable/enable dance for nothing. This is copied from
4186 * kernel/softirq.c::tasklet_unlock_spin_wait().
4187 */
4188 while (!try_wait_for_completion(&barr.done)) {
4189 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
4190 local_bh_disable();
4191 local_bh_enable();
4192 } else {
4193 cpu_relax();
4194 }
4195 }
4196 } else {
4197 wait_for_completion(&barr.done);
4174 }4198 }
4199
4200 destroy_work_on_stack(&barr.work);
4201 return true;
4175}4202}
4176 4203
4177/**4204/**
@@ -4187,6 +4214,7 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) · 9 unmodified lines
4187 */4214 */
4188bool flush_work(struct work_struct *work)4215bool flush_work(struct work_struct *work)
4189{4216{
4217 might_sleep();
4190 return __flush_work(work, false);4218 return __flush_work(work, false);
4191}4219}
4192EXPORT_SYMBOL_GPL(flush_work);4220EXPORT_SYMBOL_GPL(flush_work);
@@ -4276,6 +4304,11 @@ static bool __cancel_work_sync(struct work_struct *work, u32 cflags) · 83 unmodified lines
4276 4304
4277 ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);4305 ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);
4278 4306
4307 if (*work_data_bits(work) & WORK_OFFQ_BH)
4308 WARN_ON_ONCE(in_hardirq());
4309 else
4310 might_sleep();
4311
4279 /*4312 /*
4280 * Skip __flush_work() during early boot when we know that @work isn't4313 * Skip __flush_work() during early boot when we know that @work isn't
4281 * executing. This allows canceling during early boot.4314 * executing. This allows canceling during early boot.
@@ -4302,19 +4335,19 @@ EXPORT_SYMBOL(cancel_work); · 20 unmodified lines
4302 * cancel_work_sync - cancel a work and wait for it to finish4335 * cancel_work_sync - cancel a work and wait for it to finish
4303 * @work: the work to cancel4336 * @work: the work to cancel
4304 *4337 *
4305 * Cancel @work and wait for its execution to finish. This function4338 * Cancel @work and wait for its execution to finish. This function can be used
4306 * can be used even if the work re-queues itself or migrates to4339 * even if the work re-queues itself or migrates to another workqueue. On return
4307 * another workqueue. On return from this function, @work is4340 * from this function, @work is guaranteed to be not pending or executing on any
4308 * guaranteed to be not pending or executing on any CPU.4341 * CPU as long as there aren't racing enqueues.
4309 *4342 *
4310 * cancel_work_sync(&delayed_work->work) must not be used for4343 * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
4311 * delayed_work's. Use cancel_delayed_work_sync() instead.4344 * Use cancel_delayed_work_sync() instead.
4312 *4345 *
4313 * The caller must ensure that the workqueue on which @work was last4346 * Must be called from a sleepable context if @work was last queued on a non-BH
4314 * queued can't be destroyed before this function returns.4347 * workqueue. Can also be called from non-hardirq atomic contexts including BH
4348 * if @work was last queued on a BH workqueue.
4315 *4349 *
4316 * Return:4350 * Returns %true if @work was pending, %false otherwise.
4317 * %true if @work was pending, %false otherwise.
4318 */4351 */
4319bool cancel_work_sync(struct work_struct *work)4352bool cancel_work_sync(struct work_struct *work)
4320{4353{
@@ -4384,8 +4417,11 @@ EXPORT_SYMBOL_GPL(disable_work); · 63 unmodified lines
4384 * Similar to disable_work() but also wait for @work to finish if currently4417 * Similar to disable_work() but also wait for @work to finish if currently
4385 * executing.4418 * executing.
4386 *4419 *
4387 * Must be called from a sleepable context. Returns %true if @work was pending,4420 * Must be called from a sleepable context if @work was last queued on a non-BH
4388 * %false otherwise.4421 * workqueue. Can also be called from non-hardirq atomic contexts including BH
4422 * if @work was last queued on a BH workqueue.
4423 *
4424 * Returns %true if @work was pending, %false otherwise.
4389 */4425 */
4390bool disable_work_sync(struct work_struct *work)4426bool disable_work_sync(struct work_struct *work)
4391{4427{

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t