History

rcu: Add comments marking transitions between RCU watching and not

It is not as clear as it might be just where in RCU's idle entry/exit code RCU stops and starts watching the current CPU. This commit therefore adds comments calling out the transitions.

Paul E. McKenney committed 6 years ago

Commit ac3caf81 parent 52b1fc31 file · +25 −4Browse files

Unified · Split

kernel/rcu/tree.c+25−4
@@ -238,7 +238,9 @@ void rcu_softirq_qs(void) · 237 unmodified lines
238 238
239/*239/*
240 * Record entry into an extended quiescent state. This is only to be240 * Record entry into an extended quiescent state. This is only to be
241 * called when not already in an extended quiescent state.241 * called when not already in an extended quiescent state, that is,
242 * RCU is watching prior to the call to this function and is no longer
243 * watching upon return.
242 */244 */
243static void rcu_dynticks_eqs_enter(void)245static void rcu_dynticks_eqs_enter(void)
244{246{
@@ -251,7 +253,7 @@ static void rcu_dynticks_eqs_enter(void) · 6 unmodified lines
251 * next idle sojourn.253 * next idle sojourn.
252 */254 */
253 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);255 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
254 /* Better be in an extended quiescent state! */256 // RCU is no longer watching. Better be in extended quiescent state!
255 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&257 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
256 (seq & RCU_DYNTICK_CTRL_CTR));258 (seq & RCU_DYNTICK_CTRL_CTR));
257 /* Better not have special action (TLB flush) pending! */259 /* Better not have special action (TLB flush) pending! */
@@ -261,7 +263,8 @@ static void rcu_dynticks_eqs_enter(void) · 3 unmodified lines
261 263
262/*264/*
263 * Record exit from an extended quiescent state. This is only to be265 * Record exit from an extended quiescent state. This is only to be
264 * called from an extended quiescent state.266 * called from an extended quiescent state, that is, RCU is not watching
267 * prior to the call to this function and is watching upon return.
265 */268 */
266static void rcu_dynticks_eqs_exit(void)269static void rcu_dynticks_eqs_exit(void)
267{270{
@@ -274,6 +277,7 @@ static void rcu_dynticks_eqs_exit(void) · 6 unmodified lines
274 * critical section.277 * critical section.
275 */278 */
276 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);279 seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
280 // RCU is now watching. Better not be in an extended quiescent state!
277 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&281 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
278 !(seq & RCU_DYNTICK_CTRL_CTR));282 !(seq & RCU_DYNTICK_CTRL_CTR));
279 if (seq & RCU_DYNTICK_CTRL_MASK) {283 if (seq & RCU_DYNTICK_CTRL_MASK) {
@@ -584,6 +588,7 @@ static void rcu_eqs_enter(bool user) · 304 unmodified lines
584 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&588 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
585 rdp->dynticks_nesting == 0);589 rdp->dynticks_nesting == 0);
586 if (rdp->dynticks_nesting != 1) {590 if (rdp->dynticks_nesting != 1) {
591 // RCU will still be watching, so just do accounting and leave.
587 rdp->dynticks_nesting--;592 rdp->dynticks_nesting--;
588 return;593 return;
589 }594 }
@@ -596,7 +601,9 @@ static void rcu_eqs_enter(bool user) · 6 unmodified lines
596 rcu_prepare_for_idle();601 rcu_prepare_for_idle();
597 rcu_preempt_deferred_qs(current);602 rcu_preempt_deferred_qs(current);
598 WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */603 WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
604 // RCU is watching here ...
599 rcu_dynticks_eqs_enter();605 rcu_dynticks_eqs_enter();
606 // ... but is no longer watching here.
600 rcu_dynticks_task_enter();607 rcu_dynticks_task_enter();
601}608}
602 609
@@ -676,7 +683,9 @@ static __always_inline void rcu_nmi_exit_common(bool irq) · 73 unmodified lines
676 if (irq)683 if (irq)
677 rcu_prepare_for_idle();684 rcu_prepare_for_idle();
678 685
686 // RCU is watching here ...
679 rcu_dynticks_eqs_enter();687 rcu_dynticks_eqs_enter();
688 // ... but is no longer watching here.
680 689
681 if (irq)690 if (irq)
682 rcu_dynticks_task_enter();691 rcu_dynticks_task_enter();
@@ -751,11 +760,14 @@ static void rcu_eqs_exit(bool user) · 68 unmodified lines
751 oldval = rdp->dynticks_nesting;760 oldval = rdp->dynticks_nesting;
752 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);761 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
753 if (oldval) {762 if (oldval) {
763 // RCU was already watching, so just do accounting and leave.
754 rdp->dynticks_nesting++;764 rdp->dynticks_nesting++;
755 return;765 return;
756 }766 }
757 rcu_dynticks_task_exit();767 rcu_dynticks_task_exit();
768 // RCU is not watching here ...
758 rcu_dynticks_eqs_exit();769 rcu_dynticks_eqs_exit();
770 // ... but is watching here.
759 rcu_cleanup_after_idle();771 rcu_cleanup_after_idle();
760 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));772 trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
761 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));773 WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
@@ -832,7 +844,9 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 70 unmodified lines
832 if (irq)844 if (irq)
833 rcu_dynticks_task_exit();845 rcu_dynticks_task_exit();
834 846
847 // RCU is not watching here ...
835 rcu_dynticks_eqs_exit();848 rcu_dynticks_eqs_exit();
849 // ... but is watching here.
836 850
837 if (irq)851 if (irq)
838 rcu_cleanup_after_idle();852 rcu_cleanup_after_idle();
@@ -842,9 +856,16 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 3 unmodified lines
842 rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&856 rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
843 READ_ONCE(rdp->rcu_urgent_qs) &&857 READ_ONCE(rdp->rcu_urgent_qs) &&
844 !READ_ONCE(rdp->rcu_forced_tick)) {858 !READ_ONCE(rdp->rcu_forced_tick)) {
859 // We get here only if we had already exited the extended
860 // quiescent state and this was an interrupt (not an NMI).
861 // Therefore, (1) RCU is already watching and (2) The fact
862 // that we are in an interrupt handler and that the rcu_node
863 // lock is an irq-disabled lock prevents self-deadlock.
864 // So we can safely recheck under the lock.
845 raw_spin_lock_rcu_node(rdp->mynode);865 raw_spin_lock_rcu_node(rdp->mynode);
846 // Recheck under lock.
847 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {866 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
867 // A nohz_full CPU is in the kernel and RCU
868 // needs a quiescent state. Turn on the tick!
848 WRITE_ONCE(rdp->rcu_forced_tick, true);869 WRITE_ONCE(rdp->rcu_forced_tick, true);
849 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);870 tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
850 }871 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t