There are cases where RCU callback needs to be bounced to a sleepable context. This is currently done by the RCU callback queueing a work item, which can be cumbersome to write and confusing to read. This patch introduces rcu_work, a workqueue work variant which gets executed after a RCU grace period, and converts the open coded bouncing in fs/aio and kernel/cgroup. v3: Dropped queue_rcu_work_on(). Documented rcu grace period behavior after queue_rcu_work(). v2: Use rcu_barrier() instead of synchronize_rcu() to wait for completion of previously queued rcu callback as per Paul.
Showing the changes to kernel/workqueue.c (1 of 2 files). Show all
Unified · Split
| @@ -1604,6 +1604,40 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, · 1603 unmodified lines | ||
| 1604 | 1604 | } |
| 1605 | 1605 | EXPORT_SYMBOL_GPL(mod_delayed_work_on); |
| 1606 | 1606 | |
| 1607 | +static void rcu_work_rcufn(struct rcu_head *rcu) | |
| 1608 | +{ | |
| 1609 | + struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); | |
| 1610 | + | |
| 1611 | + /* read the comment in __queue_work() */ | |
| 1612 | + local_irq_disable(); | |
| 1613 | + __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); | |
| 1614 | + local_irq_enable(); | |
| 1615 | +} | |
| 1616 | + | |
| 1617 | +/** | |
| 1618 | + * queue_rcu_work - queue work after a RCU grace period | |
| 1619 | + * @wq: workqueue to use | |
| 1620 | + * @rwork: work to queue | |
| 1621 | + * | |
| 1622 | + * Return: %false if @rwork was already pending, %true otherwise. Note | |
| 1623 | + * that a full RCU grace period is guaranteed only after a %true return. | |
| 1624 | + * While @rwork is guarnateed to be executed after a %false return, the | |
| 1625 | + * execution may happen before a full RCU grace period has passed. | |
| 1626 | + */ | |
| 1627 | +bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) | |
| 1628 | +{ | |
| 1629 | + struct work_struct *work = &rwork->work; | |
| 1630 | + | |
| 1631 | + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { | |
| 1632 | + rwork->wq = wq; | |
| 1633 | + call_rcu(&rwork->rcu, rcu_work_rcufn); | |
| 1634 | + return true; | |
| 1635 | + } | |
| 1636 | + | |
| 1637 | + return false; | |
| 1638 | +} | |
| 1639 | +EXPORT_SYMBOL(queue_rcu_work); | |
| 1640 | + | |
| 1607 | 1641 | /** |
| 1608 | 1642 | * worker_enter_idle - enter idle state |
| 1609 | 1643 | * @worker: worker which is entering idle state |
| @@ -3001,6 +3035,26 @@ bool flush_delayed_work(struct delayed_work *dwork) · 1391 unmodified lines | ||
| 3001 | 3035 | } |
| 3002 | 3036 | EXPORT_SYMBOL(flush_delayed_work); |
| 3003 | 3037 | |
| 3038 | +/** | |
| 3039 | + * flush_rcu_work - wait for a rwork to finish executing the last queueing | |
| 3040 | + * @rwork: the rcu work to flush | |
| 3041 | + * | |
| 3042 | + * Return: | |
| 3043 | + * %true if flush_rcu_work() waited for the work to finish execution, | |
| 3044 | + * %false if it was already idle. | |
| 3045 | + */ | |
| 3046 | +bool flush_rcu_work(struct rcu_work *rwork) | |
| 3047 | +{ | |
| 3048 | + if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { | |
| 3049 | + rcu_barrier(); | |
| 3050 | + flush_work(&rwork->work); | |
| 3051 | + return true; | |
| 3052 | + } else { | |
| 3053 | + return flush_work(&rwork->work); | |
| 3054 | + } | |
| 3055 | +} | |
| 3056 | +EXPORT_SYMBOL(flush_rcu_work); | |
| 3057 | + | |
| 3004 | 3058 | static bool __cancel_work(struct work_struct *work, bool is_dwork) |
| 3005 | 3059 | { |
| 3006 | 3060 | unsigned long flags; |