There are cases where RCU callback needs to be bounced to a sleepable context. This is currently done by the RCU callback queueing a work item, which can be cumbersome to write and confusing to read. This patch introduces rcu_work, a workqueue work variant which gets executed after a RCU grace period, and converts the open coded bouncing in fs/aio and kernel/cgroup. v3: Dropped queue_rcu_work_on(). Documented rcu grace period behavior after queue_rcu_work(). v2: Use rcu_barrier() instead of synchronize_rcu() to wait for completion of previously queued rcu callback as per Paul.
Showing the changes to kernel/workqueue.c (1 of 2 files). Show all
Unified · Split
| @@ -1604,6 +1604,40 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, · 1603 unmodified lines | |||
| 1604 | } | 1604 | } |
| 1605 | EXPORT_SYMBOL_GPL(mod_delayed_work_on); | 1605 | EXPORT_SYMBOL_GPL(mod_delayed_work_on); |
| 1606 | 1606 | ||
| 1607 | static void rcu_work_rcufn(struct rcu_head *rcu) | ||
| 1608 | { | ||
| 1609 | struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); | ||
| 1610 | |||
| 1611 | /* read the comment in __queue_work() */ | ||
| 1612 | local_irq_disable(); | ||
| 1613 | __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); | ||
| 1614 | local_irq_enable(); | ||
| 1615 | } | ||
| 1616 | |||
| 1617 | /** | ||
| 1618 | * queue_rcu_work - queue work after a RCU grace period | ||
| 1619 | * @wq: workqueue to use | ||
| 1620 | * @rwork: work to queue | ||
| 1621 | * | ||
| 1622 | * Return: %false if @rwork was already pending, %true otherwise. Note | ||
| 1623 | * that a full RCU grace period is guaranteed only after a %true return. | ||
| 1624 | * While @rwork is guarnateed to be executed after a %false return, the | ||
| 1625 | * execution may happen before a full RCU grace period has passed. | ||
| 1626 | */ | ||
| 1627 | bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) | ||
| 1628 | { | ||
| 1629 | struct work_struct *work = &rwork->work; | ||
| 1630 | |||
| 1631 | if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { | ||
| 1632 | rwork->wq = wq; | ||
| 1633 | call_rcu(&rwork->rcu, rcu_work_rcufn); | ||
| 1634 | return true; | ||
| 1635 | } | ||
| 1636 | |||
| 1637 | return false; | ||
| 1638 | } | ||
| 1639 | EXPORT_SYMBOL(queue_rcu_work); | ||
| 1640 | |||
| 1607 | /** | 1641 | /** |
| 1608 | * worker_enter_idle - enter idle state | 1642 | * worker_enter_idle - enter idle state |
| 1609 | * @worker: worker which is entering idle state | 1643 | * @worker: worker which is entering idle state |
| @@ -3001,6 +3035,26 @@ bool flush_delayed_work(struct delayed_work *dwork) · 1391 unmodified lines | |||
| 3001 | } | 3035 | } |
| 3002 | EXPORT_SYMBOL(flush_delayed_work); | 3036 | EXPORT_SYMBOL(flush_delayed_work); |
| 3003 | 3037 | ||
| 3038 | /** | ||
| 3039 | * flush_rcu_work - wait for a rwork to finish executing the last queueing | ||
| 3040 | * @rwork: the rcu work to flush | ||
| 3041 | * | ||
| 3042 | * Return: | ||
| 3043 | * %true if flush_rcu_work() waited for the work to finish execution, | ||
| 3044 | * %false if it was already idle. | ||
| 3045 | */ | ||
| 3046 | bool flush_rcu_work(struct rcu_work *rwork) | ||
| 3047 | { | ||
| 3048 | if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { | ||
| 3049 | rcu_barrier(); | ||
| 3050 | flush_work(&rwork->work); | ||
| 3051 | return true; | ||
| 3052 | } else { | ||
| 3053 | return flush_work(&rwork->work); | ||
| 3054 | } | ||
| 3055 | } | ||
| 3056 | EXPORT_SYMBOL(flush_rcu_work); | ||
| 3057 | |||
| 3004 | static bool __cancel_work(struct work_struct *work, bool is_dwork) | 3058 | static bool __cancel_work(struct work_struct *work, bool is_dwork) |
| 3005 | { | 3059 | { |
| 3006 | unsigned long flags; | 3060 | unsigned long flags; |