/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <asm/atomic.h>#include <linux/bitops.h>#include <linux/module.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h> #include "rcutree.h" /* Data structures. */ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; #define RCU_STATE_INITIALIZER(structname) { \ .level = { &structname.node[0] }, \ .levelcnt = { \ NUM_RCU_LVL_0, /* root of hierarchy. */ \ NUM_RCU_LVL_1, \ NUM_RCU_LVL_2, \ NUM_RCU_LVL_3, \ NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \ }, \ .signaled = RCU_GP_IDLE, \ .gpnum = -300, \ .completed = -300, \ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \ .n_force_qs = 0, \ .n_force_qs_ngp = 0, \ .name = #structname, \} struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(struct rcu_state *rsp){ return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum);} /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. */void rcu_sched_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1;} void rcu_bh_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1;} /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. */void rcu_note_context_switch(int cpu){ rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu);} #ifdef CONFIG_NO_HZDEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = 1, .dynticks = 1,};#endif /* #ifdef CONFIG_NO_HZ */ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark = 10000; /* If this many pending, ignore blimit. */static int qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, int, 0);module_param(qhimark, int, 0);module_param(qlowmark, int, 0); #ifdef CONFIG_RCU_CPU_STALL_DETECTORint rcu_cpu_stall_suppress __read_mostly = RCU_CPU_STALL_SUPPRESS_INIT;module_param(rcu_cpu_stall_suppress, int, 0644);#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ static void force_quiescent_state(struct rcu_state *rsp, int relaxed);static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */long rcu_batches_completed_sched(void){ return rcu_sched_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */long rcu_batches_completed_bh(void){ return rcu_bh_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */void rcu_bh_force_quiescent_state(void){ force_quiescent_state(&rcu_bh_state, 0);}EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Force a quiescent state for RCU-sched. */void rcu_sched_force_quiescent_state(void){ force_quiescent_state(&rcu_sched_state, 0);}EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */static intcpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp){ return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL];} /* * Does the current CPU require a yet-as-unscheduled grace period? */static intcpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp){ return *rdp->nxttail[RCU_DONE_TAIL] && !rcu_gp_in_progress(rsp);} /* * Return the root node of the specified rcu_state structure. */static struct rcu_node *rcu_get_root(struct rcu_state *rsp){ return &rsp->node[0];} #ifdef CONFIG_SMP /* * If the specified CPU is offline, tell the caller that it is in * a quiescent state. Otherwise, whack it with a reschedule IPI. * Grace periods can end up waiting on an offline CPU when that * CPU is in the process of coming online -- it will be added to the * rcu_node bitmasks before it actually makes it online. The same thing * can happen while a CPU is in the process of coming online. Because this * race is quite rare, we check for it after detecting that the grace * period has been delayed rather than checking each and every CPU * each and every time we start a new grace period. */static int rcu_implicit_offline_qs(struct rcu_data *rdp){ /* * If the CPU is offline, it is in a quiescent state. We can * trust its state not to change because interrupts are disabled. */ if (cpu_is_offline(rdp->cpu)) { rdp->offline_fqs++; return 1; } /* If preemptable RCU, no point in sending reschedule IPI. */ if (rdp->preemptable) return 0; /* The CPU is online, so send it a reschedule IPI. */ if (rdp->cpu != smp_processor_id()) smp_send_reschedule(rdp->cpu); else set_need_resched(); rdp->resched_ipi++; return 0;} #endif /* #ifdef CONFIG_SMP */ #ifdef CONFIG_NO_HZ /** * rcu_enter_nohz - inform RCU that current CPU is entering nohz * * Enter nohz mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in nohz mode, a possibility * handled by rcu_irq_enter() and rcu_irq_exit()). */void rcu_enter_nohz(void){ unsigned long flags; struct rcu_dynticks *rdtp; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); rdtp->dynticks++; rdtp->dynticks_nesting--; WARN_ON_ONCE(rdtp->dynticks & 0x1); local_irq_restore(flags);} /* * rcu_exit_nohz - inform RCU that current CPU is leaving nohz * * Exit nohz mode, in other words, -enter- the mode in which RCU * read-side critical sections normally occur. */void rcu_exit_nohz(void){ unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); rdtp->dynticks++; rdtp->dynticks_nesting++; WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); local_irq_restore(flags); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */} /** * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is active. */void rcu_nmi_enter(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks & 0x1) return; rdtp->dynticks_nmi++; WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1)); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */} /** * rcu_nmi_exit - inform RCU of exit from NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is no longer active. */void rcu_nmi_exit(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks & 0x1) return; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ rdtp->dynticks_nmi++; WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);} /** * rcu_irq_enter - inform RCU of entry to hard irq context * * If the CPU was idle with dynamic ticks active, this updates the * rdtp->dynticks to let the RCU handling know that the CPU is active. */void rcu_irq_enter(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nesting++) return; rdtp->dynticks++; WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */} /** * rcu_irq_exit - inform RCU of exit from hard irq context * * If the CPU was idle with dynamic ticks active, update the rdp->dynticks * to put let the RCU handling be aware that the CPU is going back to idle * with no ticks. */void rcu_irq_exit(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (--rdtp->dynticks_nesting) return; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ rdtp->dynticks++; WARN_ON_ONCE(rdtp->dynticks & 0x1); /* If the interrupt queued a callback, get out of dyntick mode. */ if (__this_cpu_read(rcu_sched_data.nxtlist) || __this_cpu_read(rcu_bh_data.nxtlist)) set_need_resched();} #ifdef CONFIG_SMP /* * Snapshot the specified CPU's dynticks counter so that we can later * credit them with an implicit quiescent state. Return 1 if this CPU * is in dynticks idle mode, which is an extended quiescent state. */static int dyntick_save_progress_counter(struct rcu_data *rdp){ int ret; int snap; int snap_nmi; snap = rdp->dynticks->dynticks; snap_nmi = rdp->dynticks->dynticks_nmi; smp_mb(); /* Order sampling of snap with end of grace period. */ rdp->dynticks_snap = snap; rdp->dynticks_nmi_snap = snap_nmi; ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0); if (ret) rdp->dynticks_fqs++; return ret;} /* * Return true if the specified CPU has passed through a quiescent * state by virtue of being in or having passed through an dynticks * idle state since the last call to dyntick_save_progress_counter() * for this same CPU. */static int rcu_implicit_dynticks_qs(struct rcu_data *rdp){ long curr; long curr_nmi; long snap; long snap_nmi; curr = rdp->dynticks->dynticks; snap = rdp->dynticks_snap; curr_nmi = rdp->dynticks->dynticks_nmi; snap_nmi = rdp->dynticks_nmi_snap; smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ /* * If the CPU passed through or entered a dynticks idle phase with * no active irq/NMI handlers, then we can safely pretend that the CPU * already acknowledged the request to pass through a quiescent * state. Either way, that CPU cannot possibly be in an RCU * read-side critical section that started before the beginning * of the current RCU grace period. */ if ((curr != snap || (curr & 0x1) == 0) && (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) { rdp->dynticks_fqs++; return 1; } /* Go check for the CPU being offline. */ return rcu_implicit_offline_qs(rdp);} #endif /* #ifdef CONFIG_SMP */ #else /* #ifdef CONFIG_NO_HZ */ #ifdef CONFIG_SMP static int dyntick_save_progress_counter(struct rcu_data *rdp){ return 0;} static int rcu_implicit_dynticks_qs(struct rcu_data *rdp){ return rcu_implicit_offline_qs(rdp);} #endif /* #ifdef CONFIG_SMP */ #endif /* #else #ifdef CONFIG_NO_HZ */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR int rcu_cpu_stall_suppress __read_mostly; static void record_gp_stall_check_time(struct rcu_state *rsp){ rsp->gp_start = jiffies; rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;} static void print_other_cpu_stall(struct rcu_state *rsp){ int cpu; long delta; unsigned long flags; struct rcu_node *rnp = rcu_get_root(rsp); /* Only let one CPU complain about others per time interval. */ raw_spin_lock_irqsave(&rnp->lock, flags); delta = jiffies - rsp->jiffies_stall; if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { raw_spin_unlock_irqrestore(&rnp->lock, flags); return; } rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK; /* * Now rat on any tasks that got kicked up to the root rcu_node * due to CPU offlining. */ rcu_print_task_stall(rnp); raw_spin_unlock_irqrestore(&rnp->lock, flags); /* * OK, time to rat on our buddy... * See Documentation/RCU/stallwarn.txt for info on how to debug * RCU CPU stall warnings. */ printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {", rsp->name); rcu_for_each_leaf_node(rsp, rnp) { raw_spin_lock_irqsave(&rnp->lock, flags); rcu_print_task_stall(rnp); raw_spin_unlock_irqrestore(&rnp->lock, flags); if (rnp->qsmask == 0) continue; for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)Lines 1–500 of 1984. Load more
/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h> #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include "rcutree.h" /* Data structures. */ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; #define RCU_STATE_INITIALIZER(structname) { \ .level = { &structname.node[0] }, \ .levelcnt = { \ NUM_RCU_LVL_0, /* root of hierarchy. */ \ NUM_RCU_LVL_1, \ NUM_RCU_LVL_2, \ NUM_RCU_LVL_3, \ NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \ }, \ .signaled = RCU_GP_IDLE, \ .gpnum = -300, \ .completed = -300, \ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \ .n_force_qs = 0, \ .n_force_qs_ngp = 0, \ .name = #structname, \ } struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state); DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(struct rcu_state *rsp) { return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); } /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. */ void rcu_sched_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1; } void rcu_bh_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1; } /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. */ void rcu_note_context_switch(int cpu) { rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu); } #ifdef CONFIG_NO_HZ DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = 1, .dynticks = 1, }; #endif /* #ifdef CONFIG_NO_HZ */ static int blimit = 10; /* Maximum callbacks per softirq. */ static int qhimark = 10000; /* If this many pending, ignore blimit. */ static int qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, int, 0); module_param(qhimark, int, 0); module_param(qlowmark, int, 0); #ifdef CONFIG_RCU_CPU_STALL_DETECTOR int rcu_cpu_stall_suppress __read_mostly = RCU_CPU_STALL_SUPPRESS_INIT; module_param(rcu_cpu_stall_suppress, int, 0644); #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ static void force_quiescent_state(struct rcu_state *rsp, int relaxed); static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */ long rcu_batches_completed_sched(void) { return rcu_sched_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */ long rcu_batches_completed_bh(void) { return rcu_bh_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */ void rcu_bh_force_quiescent_state(void) { force_quiescent_state(&rcu_bh_state, 0); } EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Force a quiescent state for RCU-sched. */ void rcu_sched_force_quiescent_state(void) { force_quiescent_state(&rcu_sched_state, 0); } EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */ static int cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) { return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]; } /* * Does the current CPU require a yet-as-unscheduled grace period? */ static int cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) { return *rdp->nxttail[RCU_DONE_TAIL] && !rcu_gp_in_progress(rsp); } /* * Return the root node of the specified rcu_state structure. */ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) { return &rsp->node[0]; } #ifdef CONFIG_SMP /* * If the specified CPU is offline, tell the caller that it is in * a quiescent state. Otherwise, whack it with a reschedule IPI. * Grace periods can end up waiting on an offline CPU when that * CPU is in the process of coming online -- it will be added to the * rcu_node bitmasks before it actually makes it online. The same thing * can happen while a CPU is in the process of coming online. Because this * race is quite rare, we check for it after detecting that the grace * period has been delayed rather than checking each and every CPU * each and every time we start a new grace period. */ static int rcu_implicit_offline_qs(struct rcu_data *rdp) { /* * If the CPU is offline, it is in a quiescent state. We can * trust its state not to change because interrupts are disabled. */ if (cpu_is_offline(rdp->cpu)) { rdp->offline_fqs++; return 1; } /* If preemptable RCU, no point in sending reschedule IPI. */ if (rdp->preemptable) return 0; /* The CPU is online, so send it a reschedule IPI. */ if (rdp->cpu != smp_processor_id()) smp_send_reschedule(rdp->cpu); else set_need_resched(); rdp->resched_ipi++; return 0; } #endif /* #ifdef CONFIG_SMP */ #ifdef CONFIG_NO_HZ /** * rcu_enter_nohz - inform RCU that current CPU is entering nohz * * Enter nohz mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in nohz mode, a possibility * handled by rcu_irq_enter() and rcu_irq_exit()). */ void rcu_enter_nohz(void) { unsigned long flags; struct rcu_dynticks *rdtp; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); rdtp->dynticks++; rdtp->dynticks_nesting--; WARN_ON_ONCE(rdtp->dynticks & 0x1); local_irq_restore(flags); } /* * rcu_exit_nohz - inform RCU that current CPU is leaving nohz * * Exit nohz mode, in other words, -enter- the mode in which RCU * read-side critical sections normally occur. */ void rcu_exit_nohz(void) { unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); rdtp->dynticks++; rdtp->dynticks_nesting++; WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); local_irq_restore(flags); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ } /** * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is active. */ void rcu_nmi_enter(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks & 0x1) return; rdtp->dynticks_nmi++; WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1)); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ } /** * rcu_nmi_exit - inform RCU of exit from NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is no longer active. */ void rcu_nmi_exit(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks & 0x1) return; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ rdtp->dynticks_nmi++; WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1); } /** * rcu_irq_enter - inform RCU of entry to hard irq context * * If the CPU was idle with dynamic ticks active, this updates the * rdtp->dynticks to let the RCU handling know that the CPU is active. */ void rcu_irq_enter(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nesting++) return; rdtp->dynticks++; WARN_ON_ONCE(!(rdtp->dynticks & 0x1)); smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ } /** * rcu_irq_exit - inform RCU of exit from hard irq context * * If the CPU was idle with dynamic ticks active, update the rdp->dynticks * to put let the RCU handling be aware that the CPU is going back to idle * with no ticks. */ void rcu_irq_exit(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (--rdtp->dynticks_nesting) return; smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ rdtp->dynticks++; WARN_ON_ONCE(rdtp->dynticks & 0x1); /* If the interrupt queued a callback, get out of dyntick mode. */ if (__this_cpu_read(rcu_sched_data.nxtlist) || __this_cpu_read(rcu_bh_data.nxtlist)) set_need_resched(); } #ifdef CONFIG_SMP /* * Snapshot the specified CPU's dynticks counter so that we can later * credit them with an implicit quiescent state. Return 1 if this CPU * is in dynticks idle mode, which is an extended quiescent state. */ static int dyntick_save_progress_counter(struct rcu_data *rdp) { int ret; int snap; int snap_nmi; snap = rdp->dynticks->dynticks; snap_nmi = rdp->dynticks->dynticks_nmi; smp_mb(); /* Order sampling of snap with end of grace period. */ rdp->dynticks_snap = snap; rdp->dynticks_nmi_snap = snap_nmi; ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0); if (ret) rdp->dynticks_fqs++; return ret; } /* * Return true if the specified CPU has passed through a quiescent * state by virtue of being in or having passed through an dynticks * idle state since the last call to dyntick_save_progress_counter() * for this same CPU. */ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) { long curr; long curr_nmi; long snap; long snap_nmi; curr = rdp->dynticks->dynticks; snap = rdp->dynticks_snap; curr_nmi = rdp->dynticks->dynticks_nmi; snap_nmi = rdp->dynticks_nmi_snap; smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ /* * If the CPU passed through or entered a dynticks idle phase with * no active irq/NMI handlers, then we can safely pretend that the CPU * already acknowledged the request to pass through a quiescent * state. Either way, that CPU cannot possibly be in an RCU * read-side critical section that started before the beginning * of the current RCU grace period. */ if ((curr != snap || (curr & 0x1) == 0) && (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) { rdp->dynticks_fqs++; return 1; } /* Go check for the CPU being offline. */ return rcu_implicit_offline_qs(rdp); } #endif /* #ifdef CONFIG_SMP */ #else /* #ifdef CONFIG_NO_HZ */ #ifdef CONFIG_SMP static int dyntick_save_progress_counter(struct rcu_data *rdp) { return 0; } static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) { return rcu_implicit_offline_qs(rdp); } #endif /* #ifdef CONFIG_SMP */ #endif /* #else #ifdef CONFIG_NO_HZ */ #ifdef CONFIG_RCU_CPU_STALL_DETECTOR int rcu_cpu_stall_suppress __read_mostly; static void record_gp_stall_check_time(struct rcu_state *rsp) { rsp->gp_start = jiffies; rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK; } static void print_other_cpu_stall(struct rcu_state *rsp) { int cpu; long delta; unsigned long flags; struct rcu_node *rnp = rcu_get_root(rsp); /* Only let one CPU complain about others per time interval. */ raw_spin_lock_irqsave(&rnp->lock, flags); delta = jiffies - rsp->jiffies_stall; if (delta < RCU_STALL_RAT_DELAY || !rcu_gp_in_progress(rsp)) { raw_spin_unlock_irqrestore(&rnp->lock, flags); return; } rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK; /* * Now rat on any tasks that got kicked up to the root rcu_node * due to CPU offlining. */ rcu_print_task_stall(rnp); raw_spin_unlock_irqrestore(&rnp->lock, flags); /* * OK, time to rat on our buddy... * See Documentation/RCU/stallwarn.txt for info on how to debug * RCU CPU stall warnings. */ printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {", rsp->name); rcu_for_each_leaf_node(rsp, rnp) { raw_spin_lock_irqsave(&rnp->lock, flags); rcu_print_task_stall(rnp); raw_spin_unlock_irqrestore(&rnp->lock, flags); if (rnp->qsmask == 0) continue; for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)