It is not as clear as it might be just where in RCU's idle entry/exit code RCU stops and starts watching the current CPU. This commit therefore adds comments calling out the transitions.
Unified · Split
| @@ -238,7 +238,9 @@ void rcu_softirq_qs(void) · 237 unmodified lines | |||
| 238 | 238 | ||
| 239 | /* | 239 | /* |
| 240 | * Record entry into an extended quiescent state. This is only to be | 240 | * Record entry into an extended quiescent state. This is only to be |
| 241 | * called when not already in an extended quiescent state. | 241 | * called when not already in an extended quiescent state, that is, |
| 242 | * RCU is watching prior to the call to this function and is no longer | ||
| 243 | * watching upon return. | ||
| 242 | */ | 244 | */ |
| 243 | static void rcu_dynticks_eqs_enter(void) | 245 | static void rcu_dynticks_eqs_enter(void) |
| 244 | { | 246 | { |
| @@ -251,7 +253,7 @@ static void rcu_dynticks_eqs_enter(void) · 6 unmodified lines | |||
| 251 | * next idle sojourn. | 253 | * next idle sojourn. |
| 252 | */ | 254 | */ |
| 253 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); | 255 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); |
| 254 | /* Better be in an extended quiescent state! */ | 256 | // RCU is no longer watching. Better be in extended quiescent state! |
| 255 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && | 257 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 256 | (seq & RCU_DYNTICK_CTRL_CTR)); | 258 | (seq & RCU_DYNTICK_CTRL_CTR)); |
| 257 | /* Better not have special action (TLB flush) pending! */ | 259 | /* Better not have special action (TLB flush) pending! */ |
| @@ -261,7 +263,8 @@ static void rcu_dynticks_eqs_enter(void) · 3 unmodified lines | |||
| 261 | 263 | ||
| 262 | /* | 264 | /* |
| 263 | * Record exit from an extended quiescent state. This is only to be | 265 | * Record exit from an extended quiescent state. This is only to be |
| 264 | * called from an extended quiescent state. | 266 | * called from an extended quiescent state, that is, RCU is not watching |
| 267 | * prior to the call to this function and is watching upon return. | ||
| 265 | */ | 268 | */ |
| 266 | static void rcu_dynticks_eqs_exit(void) | 269 | static void rcu_dynticks_eqs_exit(void) |
| 267 | { | 270 | { |
| @@ -274,6 +277,7 @@ static void rcu_dynticks_eqs_exit(void) · 6 unmodified lines | |||
| 274 | * critical section. | 277 | * critical section. |
| 275 | */ | 278 | */ |
| 276 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); | 279 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); |
| 280 | // RCU is now watching. Better not be in an extended quiescent state! | ||
| 277 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && | 281 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 278 | !(seq & RCU_DYNTICK_CTRL_CTR)); | 282 | !(seq & RCU_DYNTICK_CTRL_CTR)); |
| 279 | if (seq & RCU_DYNTICK_CTRL_MASK) { | 283 | if (seq & RCU_DYNTICK_CTRL_MASK) { |
| @@ -584,6 +588,7 @@ static void rcu_eqs_enter(bool user) · 304 unmodified lines | |||
| 584 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && | 588 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 585 | rdp->dynticks_nesting == 0); | 589 | rdp->dynticks_nesting == 0); |
| 586 | if (rdp->dynticks_nesting != 1) { | 590 | if (rdp->dynticks_nesting != 1) { |
| 591 | // RCU will still be watching, so just do accounting and leave. | ||
| 587 | rdp->dynticks_nesting--; | 592 | rdp->dynticks_nesting--; |
| 588 | return; | 593 | return; |
| 589 | } | 594 | } |
| @@ -596,7 +601,9 @@ static void rcu_eqs_enter(bool user) · 6 unmodified lines | |||
| 596 | rcu_prepare_for_idle(); | 601 | rcu_prepare_for_idle(); |
| 597 | rcu_preempt_deferred_qs(current); | 602 | rcu_preempt_deferred_qs(current); |
| 598 | WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ | 603 | WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ |
| 604 | // RCU is watching here ... | ||
| 599 | rcu_dynticks_eqs_enter(); | 605 | rcu_dynticks_eqs_enter(); |
| 606 | // ... but is no longer watching here. | ||
| 600 | rcu_dynticks_task_enter(); | 607 | rcu_dynticks_task_enter(); |
| 601 | } | 608 | } |
| 602 | 609 | ||
| @@ -676,7 +683,9 @@ static __always_inline void rcu_nmi_exit_common(bool irq) · 73 unmodified lines | |||
| 676 | if (irq) | 683 | if (irq) |
| 677 | rcu_prepare_for_idle(); | 684 | rcu_prepare_for_idle(); |
| 678 | 685 | ||
| 686 | // RCU is watching here ... | ||
| 679 | rcu_dynticks_eqs_enter(); | 687 | rcu_dynticks_eqs_enter(); |
| 688 | // ... but is no longer watching here. | ||
| 680 | 689 | ||
| 681 | if (irq) | 690 | if (irq) |
| 682 | rcu_dynticks_task_enter(); | 691 | rcu_dynticks_task_enter(); |
| @@ -751,11 +760,14 @@ static void rcu_eqs_exit(bool user) · 68 unmodified lines | |||
| 751 | oldval = rdp->dynticks_nesting; | 760 | oldval = rdp->dynticks_nesting; |
| 752 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); | 761 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); |
| 753 | if (oldval) { | 762 | if (oldval) { |
| 763 | // RCU was already watching, so just do accounting and leave. | ||
| 754 | rdp->dynticks_nesting++; | 764 | rdp->dynticks_nesting++; |
| 755 | return; | 765 | return; |
| 756 | } | 766 | } |
| 757 | rcu_dynticks_task_exit(); | 767 | rcu_dynticks_task_exit(); |
| 768 | // RCU is not watching here ... | ||
| 758 | rcu_dynticks_eqs_exit(); | 769 | rcu_dynticks_eqs_exit(); |
| 770 | // ... but is watching here. | ||
| 759 | rcu_cleanup_after_idle(); | 771 | rcu_cleanup_after_idle(); |
| 760 | trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); | 772 | trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); |
| 761 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); | 773 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
| @@ -832,7 +844,9 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 70 unmodified lines | |||
| 832 | if (irq) | 844 | if (irq) |
| 833 | rcu_dynticks_task_exit(); | 845 | rcu_dynticks_task_exit(); |
| 834 | 846 | ||
| 847 | // RCU is not watching here ... | ||
| 835 | rcu_dynticks_eqs_exit(); | 848 | rcu_dynticks_eqs_exit(); |
| 849 | // ... but is watching here. | ||
| 836 | 850 | ||
| 837 | if (irq) | 851 | if (irq) |
| 838 | rcu_cleanup_after_idle(); | 852 | rcu_cleanup_after_idle(); |
| @@ -842,9 +856,16 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 3 unmodified lines | |||
| 842 | rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE && | 856 | rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE && |
| 843 | READ_ONCE(rdp->rcu_urgent_qs) && | 857 | READ_ONCE(rdp->rcu_urgent_qs) && |
| 844 | !READ_ONCE(rdp->rcu_forced_tick)) { | 858 | !READ_ONCE(rdp->rcu_forced_tick)) { |
| 859 | // We get here only if we had already exited the extended | ||
| 860 | // quiescent state and this was an interrupt (not an NMI). | ||
| 861 | // Therefore, (1) RCU is already watching and (2) The fact | ||
| 862 | // that we are in an interrupt handler and that the rcu_node | ||
| 863 | // lock is an irq-disabled lock prevents self-deadlock. | ||
| 864 | // So we can safely recheck under the lock. | ||
| 845 | raw_spin_lock_rcu_node(rdp->mynode); | 865 | raw_spin_lock_rcu_node(rdp->mynode); |
| 846 | // Recheck under lock. | ||
| 847 | if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { | 866 | if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { |
| 867 | // A nohz_full CPU is in the kernel and RCU | ||
| 868 | // needs a quiescent state. Turn on the tick! | ||
| 848 | WRITE_ONCE(rdp->rcu_forced_tick, true); | 869 | WRITE_ONCE(rdp->rcu_forced_tick, true); |
| 849 | tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU); | 870 | tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU); |
| 850 | } | 871 | } |