It is not as clear as it might be just where in RCU's idle entry/exit code RCU stops and starts watching the current CPU. This commit therefore adds comments calling out the transitions.
Unified · Split
| @@ -238,7 +238,9 @@ void rcu_softirq_qs(void) · 237 unmodified lines | ||
| 238 | 238 | |
| 239 | 239 | /* |
| 240 | 240 | * Record entry into an extended quiescent state. This is only to be |
| 241 | − * called when not already in an extended quiescent state. | |
| 241 | + * called when not already in an extended quiescent state, that is, | |
| 242 | + * RCU is watching prior to the call to this function and is no longer | |
| 243 | + * watching upon return. | |
| 242 | 244 | */ |
| 243 | 245 | static void rcu_dynticks_eqs_enter(void) |
| 244 | 246 | { |
| @@ -251,7 +253,7 @@ static void rcu_dynticks_eqs_enter(void) · 6 unmodified lines | ||
| 251 | 253 | * next idle sojourn. |
| 252 | 254 | */ |
| 253 | 255 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); |
| 254 | − /* Better be in an extended quiescent state! */ | |
| 256 | + // RCU is no longer watching. Better be in extended quiescent state! | |
| 255 | 257 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 256 | 258 | (seq & RCU_DYNTICK_CTRL_CTR)); |
| 257 | 259 | /* Better not have special action (TLB flush) pending! */ |
| @@ -261,7 +263,8 @@ static void rcu_dynticks_eqs_enter(void) · 3 unmodified lines | ||
| 261 | 263 | |
| 262 | 264 | /* |
| 263 | 265 | * Record exit from an extended quiescent state. This is only to be |
| 264 | − * called from an extended quiescent state. | |
| 266 | + * called from an extended quiescent state, that is, RCU is not watching | |
| 267 | + * prior to the call to this function and is watching upon return. | |
| 265 | 268 | */ |
| 266 | 269 | static void rcu_dynticks_eqs_exit(void) |
| 267 | 270 | { |
| @@ -274,6 +277,7 @@ static void rcu_dynticks_eqs_exit(void) · 6 unmodified lines | ||
| 274 | 277 | * critical section. |
| 275 | 278 | */ |
| 276 | 279 | seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); |
| 280 | + // RCU is now watching. Better not be in an extended quiescent state! | |
| 277 | 281 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 278 | 282 | !(seq & RCU_DYNTICK_CTRL_CTR)); |
| 279 | 283 | if (seq & RCU_DYNTICK_CTRL_MASK) { |
| @@ -584,6 +588,7 @@ static void rcu_eqs_enter(bool user) · 304 unmodified lines | ||
| 584 | 588 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && |
| 585 | 589 | rdp->dynticks_nesting == 0); |
| 586 | 590 | if (rdp->dynticks_nesting != 1) { |
| 591 | + // RCU will still be watching, so just do accounting and leave. | |
| 587 | 592 | rdp->dynticks_nesting--; |
| 588 | 593 | return; |
| 589 | 594 | } |
| @@ -596,7 +601,9 @@ static void rcu_eqs_enter(bool user) · 6 unmodified lines | ||
| 596 | 601 | rcu_prepare_for_idle(); |
| 597 | 602 | rcu_preempt_deferred_qs(current); |
| 598 | 603 | WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ |
| 604 | + // RCU is watching here ... | |
| 599 | 605 | rcu_dynticks_eqs_enter(); |
| 606 | + // ... but is no longer watching here. | |
| 600 | 607 | rcu_dynticks_task_enter(); |
| 601 | 608 | } |
| 602 | 609 | |
| @@ -676,7 +683,9 @@ static __always_inline void rcu_nmi_exit_common(bool irq) · 73 unmodified lines | ||
| 676 | 683 | if (irq) |
| 677 | 684 | rcu_prepare_for_idle(); |
| 678 | 685 | |
| 686 | + // RCU is watching here ... | |
| 679 | 687 | rcu_dynticks_eqs_enter(); |
| 688 | + // ... but is no longer watching here. | |
| 680 | 689 | |
| 681 | 690 | if (irq) |
| 682 | 691 | rcu_dynticks_task_enter(); |
| @@ -751,11 +760,14 @@ static void rcu_eqs_exit(bool user) · 68 unmodified lines | ||
| 751 | 760 | oldval = rdp->dynticks_nesting; |
| 752 | 761 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0); |
| 753 | 762 | if (oldval) { |
| 763 | + // RCU was already watching, so just do accounting and leave. | |
| 754 | 764 | rdp->dynticks_nesting++; |
| 755 | 765 | return; |
| 756 | 766 | } |
| 757 | 767 | rcu_dynticks_task_exit(); |
| 768 | + // RCU is not watching here ... | |
| 758 | 769 | rcu_dynticks_eqs_exit(); |
| 770 | + // ... but is watching here. | |
| 759 | 771 | rcu_cleanup_after_idle(); |
| 760 | 772 | trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); |
| 761 | 773 | WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); |
| @@ -832,7 +844,9 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 70 unmodified lines | ||
| 832 | 844 | if (irq) |
| 833 | 845 | rcu_dynticks_task_exit(); |
| 834 | 846 | |
| 847 | + // RCU is not watching here ... | |
| 835 | 848 | rcu_dynticks_eqs_exit(); |
| 849 | + // ... but is watching here. | |
| 836 | 850 | |
| 837 | 851 | if (irq) |
| 838 | 852 | rcu_cleanup_after_idle(); |
| @@ -842,9 +856,16 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 3 unmodified lines | ||
| 842 | 856 | rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE && |
| 843 | 857 | READ_ONCE(rdp->rcu_urgent_qs) && |
| 844 | 858 | !READ_ONCE(rdp->rcu_forced_tick)) { |
| 859 | + // We get here only if we had already exited the extended | |
| 860 | + // quiescent state and this was an interrupt (not an NMI). | |
| 861 | + // Therefore, (1) RCU is already watching and (2) The fact | |
| 862 | + // that we are in an interrupt handler and that the rcu_node | |
| 863 | + // lock is an irq-disabled lock prevents self-deadlock. | |
| 864 | + // So we can safely recheck under the lock. | |
| 845 | 865 | raw_spin_lock_rcu_node(rdp->mynode); |
| 846 | − // Recheck under lock. | |
| 847 | 866 | if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) { |
| 867 | + // A nohz_full CPU is in the kernel and RCU | |
| 868 | + // needs a quiescent state. Turn on the tick! | |
| 848 | 869 | WRITE_ONCE(rdp->rcu_forced_tick, true); |
| 849 | 870 | tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU); |
| 850 | 871 | } |