/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/module.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <linux/prefetch.h> #include "rcutree.h" /* Data structures. */ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; #define RCU_STATE_INITIALIZER(structname) { \ .level = { &structname.node[0] }, \ .levelcnt = { \ NUM_RCU_LVL_0, /* root of hierarchy. */ \ NUM_RCU_LVL_1, \ NUM_RCU_LVL_2, \ NUM_RCU_LVL_3, \ NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \ }, \ .signaled = RCU_GP_IDLE, \ .gpnum = -300, \ .completed = -300, \ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \ .n_force_qs = 0, \ .n_force_qs_ngp = 0, \ .name = #structname, \} struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); static struct rcu_state *rcu_state; /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimized synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; #ifdef CONFIG_RCU_BOOST /* * Control variables for per-CPU and per-rcu_node kthreads. These * handle all flavors of RCU. */static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);DEFINE_PER_CPU(char, rcu_cpu_has_work); #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);static void invoke_rcu_core(void);static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); #define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */ /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */unsigned long rcutorture_testseq;unsigned long rcutorture_vernum; /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(struct rcu_state *rsp){ return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum);} /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. */void rcu_sched_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1;} void rcu_bh_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1;} /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. */void rcu_note_context_switch(int cpu){ rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu);}EXPORT_SYMBOL_GPL(rcu_note_context_switch); #ifdef CONFIG_NO_HZDEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = 1, .dynticks = ATOMIC_INIT(1),};#endif /* #ifdef CONFIG_NO_HZ */ static int blimit = 10; /* Maximum callbacks per softirq. */static int qhimark = 10000; /* If this many pending, ignore blimit. */static int qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, int, 0);module_param(qhimark, int, 0);module_param(qlowmark, int, 0); int rcu_cpu_stall_suppress __read_mostly;module_param(rcu_cpu_stall_suppress, int, 0644); static void force_quiescent_state(struct rcu_state *rsp, int relaxed);static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */long rcu_batches_completed_sched(void){ return rcu_sched_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */long rcu_batches_completed_bh(void){ return rcu_bh_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */void rcu_bh_force_quiescent_state(void){ force_quiescent_state(&rcu_bh_state, 0);}EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */void rcutorture_record_test_transition(void){ rcutorture_testseq++; rcutorture_vernum = 0;}EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); /* * Record the number of writer passes through the current rcutorture test. * This is also used to correlate debugfs tracing stats with the rcutorture * messages. */void rcutorture_record_progress(unsigned long vernum){ rcutorture_vernum++;}EXPORT_SYMBOL_GPL(rcutorture_record_progress); /* * Force a quiescent state for RCU-sched. */void rcu_sched_force_quiescent_state(void){ force_quiescent_state(&rcu_sched_state, 0);}EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */static intcpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp){ return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL];} /* * Does the current CPU require a yet-as-unscheduled grace period? */static intcpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp){ return *rdp->nxttail[RCU_DONE_TAIL] && !rcu_gp_in_progress(rsp);} /* * Return the root node of the specified rcu_state structure. */static struct rcu_node *rcu_get_root(struct rcu_state *rsp){ return &rsp->node[0];} #ifdef CONFIG_SMP /* * If the specified CPU is offline, tell the caller that it is in * a quiescent state. Otherwise, whack it with a reschedule IPI. * Grace periods can end up waiting on an offline CPU when that * CPU is in the process of coming online -- it will be added to the * rcu_node bitmasks before it actually makes it online. The same thing * can happen while a CPU is in the process of coming online. Because this * race is quite rare, we check for it after detecting that the grace * period has been delayed rather than checking each and every CPU * each and every time we start a new grace period. */static int rcu_implicit_offline_qs(struct rcu_data *rdp){ /* * If the CPU is offline, it is in a quiescent state. We can * trust its state not to change because interrupts are disabled. */ if (cpu_is_offline(rdp->cpu)) { rdp->offline_fqs++; return 1; } /* If preemptible RCU, no point in sending reschedule IPI. */ if (rdp->preemptible) return 0; /* The CPU is online, so send it a reschedule IPI. */ if (rdp->cpu != smp_processor_id()) smp_send_reschedule(rdp->cpu); else set_need_resched(); rdp->resched_ipi++; return 0;} #endif /* #ifdef CONFIG_SMP */ #ifdef CONFIG_NO_HZ /** * rcu_enter_nohz - inform RCU that current CPU is entering nohz * * Enter nohz mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in nohz mode, a possibility * handled by rcu_irq_enter() and rcu_irq_exit()). */void rcu_enter_nohz(void){ unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); if (--rdtp->dynticks_nesting) { local_irq_restore(flags); return; } /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); local_irq_restore(flags); /* If the interrupt queued a callback, get out of dyntick mode. */ if (in_irq() && (__get_cpu_var(rcu_sched_data).nxtlist || __get_cpu_var(rcu_bh_data).nxtlist || rcu_preempt_needs_cpu(smp_processor_id()))) set_need_resched();} /* * rcu_exit_nohz - inform RCU that current CPU is leaving nohz * * Exit nohz mode, in other words, -enter- the mode in which RCU * read-side critical sections normally occur. */void rcu_exit_nohz(void){ unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nesting++) { local_irq_restore(flags); return; } smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); local_irq_restore(flags);} /** * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is active. */void rcu_nmi_enter(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 && (atomic_read(&rdtp->dynticks) & 0x1)) return; rdtp->dynticks_nmi_nesting++; smp_mb__before_atomic_inc(); /* Force delay from prior write. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));} /** * rcu_nmi_exit - inform RCU of exit from NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is no longer active. */void rcu_nmi_exit(void){ struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 || --rdtp->dynticks_nmi_nesting != 0) return; /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force delay to next write. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);} /** * rcu_irq_enter - inform RCU of entry to hard irq context * * If the CPU was idle with dynamic ticks active, this updates the * rdtp->dynticks to let the RCU handling know that the CPU is active. */void rcu_irq_enter(void){ rcu_exit_nohz();} /** * rcu_irq_exit - inform RCU of exit from hard irq context * * If the CPU was idle with dynamic ticks active, update the rdp->dynticks * to put let the RCU handling be aware that the CPU is going back to idle * with no ticks. */void rcu_irq_exit(void){ rcu_enter_nohz();} #ifdef CONFIG_SMP /* * Snapshot the specified CPU's dynticks counter so that we can later * credit them with an implicit quiescent state. Return 1 if this CPU * is in dynticks idle mode, which is an extended quiescent state. */static int dyntick_save_progress_counter(struct rcu_data *rdp){ rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); return 0;} /* * Return true if the specified CPU has passed through a quiescent * state by virtue of being in or having passed through an dynticks * idle state since the last call to dyntick_save_progress_counter() * for this same CPU. */static int rcu_implicit_dynticks_qs(struct rcu_data *rdp){ unsigned long curr; unsigned long snap; curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks); snap = (unsigned long)rdp->dynticks_snap; /* * If the CPU passed through or entered a dynticks idle phase with * no active irq/NMI handlers, then we can safely pretend that the CPU * already acknowledged the request to pass through a quiescent * state. Either way, that CPU cannot possibly be in an RCU * read-side critical section that started before the beginning * of the current RCU grace period. */ if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) { rdp->dynticks_fqs++; return 1;Lines 1–500 of 2089. Load more
/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/module.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include <linux/wait.h> #include <linux/kthread.h> #include <linux/prefetch.h> #include "rcutree.h" /* Data structures. */ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; #define RCU_STATE_INITIALIZER(structname) { \ .level = { &structname.node[0] }, \ .levelcnt = { \ NUM_RCU_LVL_0, /* root of hierarchy. */ \ NUM_RCU_LVL_1, \ NUM_RCU_LVL_2, \ NUM_RCU_LVL_3, \ NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \ }, \ .signaled = RCU_GP_IDLE, \ .gpnum = -300, \ .completed = -300, \ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \ .n_force_qs = 0, \ .n_force_qs_ngp = 0, \ .name = #structname, \ } struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state); DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); static struct rcu_state *rcu_state; /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimized synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */ static int rcu_scheduler_fully_active __read_mostly; #ifdef CONFIG_RCU_BOOST /* * Control variables for per-CPU and per-rcu_node kthreads. These * handle all flavors of RCU. */ static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); DEFINE_PER_CPU(char, rcu_cpu_has_work); #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); #define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */ /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */ unsigned long rcutorture_testseq; unsigned long rcutorture_vernum; /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(struct rcu_state *rsp) { return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); } /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. */ void rcu_sched_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1; } void rcu_bh_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); rdp->passed_quiesc_completed = rdp->gpnum - 1; barrier(); rdp->passed_quiesc = 1; } /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. */ void rcu_note_context_switch(int cpu) { rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu); } EXPORT_SYMBOL_GPL(rcu_note_context_switch); #ifdef CONFIG_NO_HZ DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = 1, .dynticks = ATOMIC_INIT(1), }; #endif /* #ifdef CONFIG_NO_HZ */ static int blimit = 10; /* Maximum callbacks per softirq. */ static int qhimark = 10000; /* If this many pending, ignore blimit. */ static int qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, int, 0); module_param(qhimark, int, 0); module_param(qlowmark, int, 0); int rcu_cpu_stall_suppress __read_mostly; module_param(rcu_cpu_stall_suppress, int, 0644); static void force_quiescent_state(struct rcu_state *rsp, int relaxed); static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */ long rcu_batches_completed_sched(void) { return rcu_sched_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */ long rcu_batches_completed_bh(void) { return rcu_bh_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */ void rcu_bh_force_quiescent_state(void) { force_quiescent_state(&rcu_bh_state, 0); } EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */ void rcutorture_record_test_transition(void) { rcutorture_testseq++; rcutorture_vernum = 0; } EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); /* * Record the number of writer passes through the current rcutorture test. * This is also used to correlate debugfs tracing stats with the rcutorture * messages. */ void rcutorture_record_progress(unsigned long vernum) { rcutorture_vernum++; } EXPORT_SYMBOL_GPL(rcutorture_record_progress); /* * Force a quiescent state for RCU-sched. */ void rcu_sched_force_quiescent_state(void) { force_quiescent_state(&rcu_sched_state, 0); } EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */ static int cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) { return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]; } /* * Does the current CPU require a yet-as-unscheduled grace period? */ static int cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) { return *rdp->nxttail[RCU_DONE_TAIL] && !rcu_gp_in_progress(rsp); } /* * Return the root node of the specified rcu_state structure. */ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) { return &rsp->node[0]; } #ifdef CONFIG_SMP /* * If the specified CPU is offline, tell the caller that it is in * a quiescent state. Otherwise, whack it with a reschedule IPI. * Grace periods can end up waiting on an offline CPU when that * CPU is in the process of coming online -- it will be added to the * rcu_node bitmasks before it actually makes it online. The same thing * can happen while a CPU is in the process of coming online. Because this * race is quite rare, we check for it after detecting that the grace * period has been delayed rather than checking each and every CPU * each and every time we start a new grace period. */ static int rcu_implicit_offline_qs(struct rcu_data *rdp) { /* * If the CPU is offline, it is in a quiescent state. We can * trust its state not to change because interrupts are disabled. */ if (cpu_is_offline(rdp->cpu)) { rdp->offline_fqs++; return 1; } /* If preemptible RCU, no point in sending reschedule IPI. */ if (rdp->preemptible) return 0; /* The CPU is online, so send it a reschedule IPI. */ if (rdp->cpu != smp_processor_id()) smp_send_reschedule(rdp->cpu); else set_need_resched(); rdp->resched_ipi++; return 0; } #endif /* #ifdef CONFIG_SMP */ #ifdef CONFIG_NO_HZ /** * rcu_enter_nohz - inform RCU that current CPU is entering nohz * * Enter nohz mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in nohz mode, a possibility * handled by rcu_irq_enter() and rcu_irq_exit()). */ void rcu_enter_nohz(void) { unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); if (--rdtp->dynticks_nesting) { local_irq_restore(flags); return; } /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); local_irq_restore(flags); /* If the interrupt queued a callback, get out of dyntick mode. */ if (in_irq() && (__get_cpu_var(rcu_sched_data).nxtlist || __get_cpu_var(rcu_bh_data).nxtlist || rcu_preempt_needs_cpu(smp_processor_id()))) set_need_resched(); } /* * rcu_exit_nohz - inform RCU that current CPU is leaving nohz * * Exit nohz mode, in other words, -enter- the mode in which RCU * read-side critical sections normally occur. */ void rcu_exit_nohz(void) { unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nesting++) { local_irq_restore(flags); return; } smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); local_irq_restore(flags); } /** * rcu_nmi_enter - inform RCU of entry to NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is active. */ void rcu_nmi_enter(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 && (atomic_read(&rdtp->dynticks) & 0x1)) return; rdtp->dynticks_nmi_nesting++; smp_mb__before_atomic_inc(); /* Force delay from prior write. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); } /** * rcu_nmi_exit - inform RCU of exit from NMI context * * If the CPU was idle with dynamic ticks active, and there is no * irq handler running, this updates rdtp->dynticks_nmi to let the * RCU grace-period handling know that the CPU is no longer active. */ void rcu_nmi_exit(void) { struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks); if (rdtp->dynticks_nmi_nesting == 0 || --rdtp->dynticks_nmi_nesting != 0) return; /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force delay to next write. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); } /** * rcu_irq_enter - inform RCU of entry to hard irq context * * If the CPU was idle with dynamic ticks active, this updates the * rdtp->dynticks to let the RCU handling know that the CPU is active. */ void rcu_irq_enter(void) { rcu_exit_nohz(); } /** * rcu_irq_exit - inform RCU of exit from hard irq context * * If the CPU was idle with dynamic ticks active, update the rdp->dynticks * to put let the RCU handling be aware that the CPU is going back to idle * with no ticks. */ void rcu_irq_exit(void) { rcu_enter_nohz(); } #ifdef CONFIG_SMP /* * Snapshot the specified CPU's dynticks counter so that we can later * credit them with an implicit quiescent state. Return 1 if this CPU * is in dynticks idle mode, which is an extended quiescent state. */ static int dyntick_save_progress_counter(struct rcu_data *rdp) { rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks); return 0; } /* * Return true if the specified CPU has passed through a quiescent * state by virtue of being in or having passed through an dynticks * idle state since the last call to dyntick_save_progress_counter() * for this same CPU. */ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) { unsigned long curr; unsigned long snap; curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks); snap = (unsigned long)rdp->dynticks_snap; /* * If the CPU passed through or entered a dynticks idle phase with * no active irq/NMI handlers, then we can safely pretend that the CPU * already acknowledged the request to pass through a quiescent * state. Either way, that CPU cannot possibly be in an RCU * read-side critical section that started before the beginning * of the current RCU grace period. */ if ((curr & 0x1) == 0 || ULONG_CMP_GE(curr, snap + 2)) { rdp->dynticks_fqs++; return 1;