Deferred wakeup of rcuog kthreads upon RCU idle mode entry is going to be handled differently whether initiated by idle, user or guest. Prepare with pulling that control up to rcu_eqs_enter() callers.
Unified · Split
| @@ -644,7 +644,6 @@ static noinstr void rcu_eqs_enter(bool user) · 643 unmodified lines | |||
| 644 | trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); | 644 | trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks)); |
| 645 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); | 645 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
| 646 | rdp = this_cpu_ptr(&rcu_data); | 646 | rdp = this_cpu_ptr(&rcu_data); |
| 647 | do_nocb_deferred_wakeup(rdp); | ||
| 648 | rcu_prepare_for_idle(); | 647 | rcu_prepare_for_idle(); |
| 649 | rcu_preempt_deferred_qs(current); | 648 | rcu_preempt_deferred_qs(current); |
| 650 | 649 | ||
| @@ -672,7 +671,10 @@ static noinstr void rcu_eqs_enter(bool user) · 21 unmodified lines | |||
| 672 | */ | 671 | */ |
| 673 | void rcu_idle_enter(void) | 672 | void rcu_idle_enter(void) |
| 674 | { | 673 | { |
| 674 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | ||
| 675 | |||
| 675 | lockdep_assert_irqs_disabled(); | 676 | lockdep_assert_irqs_disabled(); |
| 677 | do_nocb_deferred_wakeup(rdp); | ||
| 676 | rcu_eqs_enter(false); | 678 | rcu_eqs_enter(false); |
| 677 | } | 679 | } |
| 678 | EXPORT_SYMBOL_GPL(rcu_idle_enter); | 680 | EXPORT_SYMBOL_GPL(rcu_idle_enter); |
| @@ -691,7 +693,14 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); · 12 unmodified lines | |||
| 691 | */ | 693 | */ |
| 692 | noinstr void rcu_user_enter(void) | 694 | noinstr void rcu_user_enter(void) |
| 693 | { | 695 | { |
| 696 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | ||
| 697 | |||
| 694 | lockdep_assert_irqs_disabled(); | 698 | lockdep_assert_irqs_disabled(); |
| 699 | |||
| 700 | instrumentation_begin(); | ||
| 701 | do_nocb_deferred_wakeup(rdp); | ||
| 702 | instrumentation_end(); | ||
| 703 | |||
| 695 | rcu_eqs_enter(true); | 704 | rcu_eqs_enter(true); |
| 696 | } | 705 | } |
| 697 | #endif /* CONFIG_NO_HZ_FULL */ | 706 | #endif /* CONFIG_NO_HZ_FULL */ |