/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/stop_machine.h>#include <linux/random.h> #include "rcutree.h"#include <trace/events/rcu.h> #include "rcu.h" /* Data structures. */ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; #define RCU_STATE_INITIALIZER(sname, cr) { \ .level = { &sname##_state.node[0] }, \ .call = cr, \ .fqs_state = RCU_GP_IDLE, \ .gpnum = 0UL - 300UL, \ .completed = 0UL - 300UL, \ .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \ .orphan_nxttail = &sname##_state.orphan_nxtlist, \ .orphan_donetail = &sname##_state.orphan_donelist, \ .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \ .name = #sname, \} struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched, call_rcu_sched);DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh);DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); static struct rcu_state *rcu_state;LIST_HEAD(rcu_struct_flavors); /* Increase (but not decrease) the CONFIG_RCU_FANOUT_LEAF at boot time. */static int rcu_fanout_leaf = CONFIG_RCU_FANOUT_LEAF;module_param(rcu_fanout_leaf, int, 0444);int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;static int num_rcu_lvl[] = { /* Number of rcu_nodes at specified level. */ NUM_RCU_LVL_0, NUM_RCU_LVL_1, NUM_RCU_LVL_2, NUM_RCU_LVL_3, NUM_RCU_LVL_4,};int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimized synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; #ifdef CONFIG_RCU_BOOST /* * Control variables for per-CPU and per-rcu_node kthreads. These * handle all flavors of RCU. */static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);DEFINE_PER_CPU(char, rcu_cpu_has_work); #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);static void invoke_rcu_core(void);static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */unsigned long rcutorture_testseq;unsigned long rcutorture_vernum; /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(struct rcu_state *rsp){ return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum);} /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. * The caller must have disabled preemption. */void rcu_sched_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); if (rdp->passed_quiesce == 0) trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs"); rdp->passed_quiesce = 1;} void rcu_bh_qs(int cpu){ struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); if (rdp->passed_quiesce == 0) trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs"); rdp->passed_quiesce = 1;} /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */void rcu_note_context_switch(int cpu){ trace_rcu_utilization("Start context switch"); rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu); trace_rcu_utilization("End context switch");}EXPORT_SYMBOL_GPL(rcu_note_context_switch); DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE, .dynticks = ATOMIC_INIT(1),}; static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */static long qhimark = 10000; /* If this many pending, ignore blimit. */static long qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, long, 0444);module_param(qhimark, long, 0444);module_param(qlowmark, long, 0444); int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT; module_param(rcu_cpu_stall_suppress, int, 0644);module_param(rcu_cpu_stall_timeout, int, 0644); static ulong jiffies_till_first_fqs = RCU_JIFFIES_TILL_FORCE_QS;static ulong jiffies_till_next_fqs = RCU_JIFFIES_TILL_FORCE_QS; module_param(jiffies_till_first_fqs, ulong, 0644);module_param(jiffies_till_next_fqs, ulong, 0644); static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *));static void force_quiescent_state(struct rcu_state *rsp);static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */long rcu_batches_completed_sched(void){ return rcu_sched_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */long rcu_batches_completed_bh(void){ return rcu_bh_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */void rcu_bh_force_quiescent_state(void){ force_quiescent_state(&rcu_bh_state);}EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */void rcutorture_record_test_transition(void){ rcutorture_testseq++; rcutorture_vernum = 0;}EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); /* * Record the number of writer passes through the current rcutorture test. * This is also used to correlate debugfs tracing stats with the rcutorture * messages. */void rcutorture_record_progress(unsigned long vernum){ rcutorture_vernum++;}EXPORT_SYMBOL_GPL(rcutorture_record_progress); /* * Force a quiescent state for RCU-sched. */void rcu_sched_force_quiescent_state(void){ force_quiescent_state(&rcu_sched_state);}EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */static intcpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp){ return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] && rdp->nxttail[RCU_DONE_TAIL] != NULL;} /* * Does the current CPU require a yet-as-unscheduled grace period? */static intcpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp){ struct rcu_head **ntp; ntp = rdp->nxttail[RCU_DONE_TAIL + (ACCESS_ONCE(rsp->completed) != rdp->completed)]; return rdp->nxttail[RCU_DONE_TAIL] && ntp && *ntp && !rcu_gp_in_progress(rsp);} /* * Return the root node of the specified rcu_state structure. */static struct rcu_node *rcu_get_root(struct rcu_state *rsp){ return &rsp->node[0];} /* * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state * * If the new value of the ->dynticks_nesting counter now is zero, * we really have entered idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, bool user){ trace_rcu_dyntick("Start", oldval, 0); if (!user && !is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on entry: not idle task", oldval, 0); ftrace_dump(DUMP_ORIG); WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", current->pid, current->comm, idle->pid, idle->comm); /* must be idle task! */ } rcu_prepare_for_idle(smp_processor_id()); /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); /* * It is illegal to enter an extended quiescent state while * in an RCU read-side critical section. */ rcu_lockdep_assert(!lock_is_held(&rcu_lock_map), "Illegal idle entry in RCU read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), "Illegal idle entry in RCU-bh read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), "Illegal idle entry in RCU-sched read-side critical section.");} /* * Enter an RCU extended quiescent state, which can be either the * idle loop or adaptive-tickless usermode execution. */static void rcu_eqs_enter(bool user){ long long oldval; struct rcu_dynticks *rdtp; rdtp = &__get_cpu_var(rcu_dynticks); oldval = rdtp->dynticks_nesting; WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) rdtp->dynticks_nesting = 0; else rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE; rcu_eqs_enter_common(rdtp, oldval, user);} /** * rcu_idle_enter - inform RCU that current CPU is entering idle * * Enter idle mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in idle, a possibility * handled by irq_enter() and irq_exit().) * * We crowbar the ->dynticks_nesting field to zero to allow for * the possibility of usermode upcalls having messed up our count * of interrupt nesting level during the prior busy period. */void rcu_idle_enter(void){ unsigned long flags; local_irq_save(flags); rcu_eqs_enter(false); local_irq_restore(flags);}EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_RCU_USER_QS/** * rcu_user_enter - inform RCU that we are resuming userspace. * * Enter RCU idle mode right before resuming userspace. No use of RCU * is permitted between this call and rcu_user_exit(). This way the * CPU doesn't need to maintain the tick for RCU maintenance purposes * when the CPU runs in userspace. */void rcu_user_enter(void){ rcu_eqs_enter(1);} /** * rcu_user_enter_after_irq - inform RCU that we are going to resume userspace * after the current irq returns. * * This is similar to rcu_user_enter() but in the context of a non-nesting * irq. After this call, RCU enters into idle mode when the interrupt * returns. */void rcu_user_enter_after_irq(void){ unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); /* Ensure this irq is interrupting a non-idle RCU state. */ WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK)); rdtp->dynticks_nesting = 1; local_irq_restore(flags);}#endif /* CONFIG_RCU_USER_QS */ /** * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle * * Exit from an interrupt handler, which might possibly result in entering * idle mode, in other words, leaving the mode in which read-side critical * sections can occur. * * This code assumes that the idle loop never does anything that might * result in unbalanced calls to irq_enter() and irq_exit(). If your * architecture violates this assumption, RCU will give you what you * deserve, good and hard. But very infrequently and irreproducibly. * * Use things like work queues to work around this limitation. * * You have been warned. */void rcu_irq_exit(void){ unsigned long flags; long long oldval; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); oldval = rdtp->dynticks_nesting; rdtp->dynticks_nesting--; WARN_ON_ONCE(rdtp->dynticks_nesting < 0); if (rdtp->dynticks_nesting) trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting); else rcu_eqs_enter_common(rdtp, oldval, true); local_irq_restore(flags);} /* * rcu_eqs_exit_common - current CPU moving away from extended quiescent state * * If the new value of the ->dynticks_nesting counter was previously zero, * we really have exited idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, int user){ smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); rcu_cleanup_after_idle(smp_processor_id()); trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting); if (!user && !is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on exit: not idle task", oldval, rdtp->dynticks_nesting);Lines 1–500 of 3080. Load more
/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/export.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include <linux/wait.h> #include <linux/kthread.h> #include <linux/prefetch.h> #include <linux/delay.h> #include <linux/stop_machine.h> #include <linux/random.h> #include "rcutree.h" #include <trace/events/rcu.h> #include "rcu.h" /* Data structures. */ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS]; static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; #define RCU_STATE_INITIALIZER(sname, cr) { \ .level = { &sname##_state.node[0] }, \ .call = cr, \ .fqs_state = RCU_GP_IDLE, \ .gpnum = 0UL - 300UL, \ .completed = 0UL - 300UL, \ .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \ .orphan_nxttail = &sname##_state.orphan_nxtlist, \ .orphan_donetail = &sname##_state.orphan_donelist, \ .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ .onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \ .name = #sname, \ } struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched, call_rcu_sched); DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh); DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); static struct rcu_state *rcu_state; LIST_HEAD(rcu_struct_flavors); /* Increase (but not decrease) the CONFIG_RCU_FANOUT_LEAF at boot time. */ static int rcu_fanout_leaf = CONFIG_RCU_FANOUT_LEAF; module_param(rcu_fanout_leaf, int, 0444); int rcu_num_lvls __read_mostly = RCU_NUM_LVLS; static int num_rcu_lvl[] = { /* Number of rcu_nodes at specified level. */ NUM_RCU_LVL_0, NUM_RCU_LVL_1, NUM_RCU_LVL_2, NUM_RCU_LVL_3, NUM_RCU_LVL_4, }; int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimized synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */ static int rcu_scheduler_fully_active __read_mostly; #ifdef CONFIG_RCU_BOOST /* * Control variables for per-CPU and per-rcu_node kthreads. These * handle all flavors of RCU. */ static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); DEFINE_PER_CPU(char, rcu_cpu_has_work); #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */ unsigned long rcutorture_testseq; unsigned long rcutorture_vernum; /* * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(struct rcu_state *rsp) { return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum); } /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. * The caller must have disabled preemption. */ void rcu_sched_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); if (rdp->passed_quiesce == 0) trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs"); rdp->passed_quiesce = 1; } void rcu_bh_qs(int cpu) { struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); if (rdp->passed_quiesce == 0) trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs"); rdp->passed_quiesce = 1; } /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */ void rcu_note_context_switch(int cpu) { trace_rcu_utilization("Start context switch"); rcu_sched_qs(cpu); rcu_preempt_note_context_switch(cpu); trace_rcu_utilization("End context switch"); } EXPORT_SYMBOL_GPL(rcu_note_context_switch); DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE, .dynticks = ATOMIC_INIT(1), }; static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */ static long qhimark = 10000; /* If this many pending, ignore blimit. */ static long qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, long, 0444); module_param(qhimark, long, 0444); module_param(qlowmark, long, 0444); int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */ int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT; module_param(rcu_cpu_stall_suppress, int, 0644); module_param(rcu_cpu_stall_timeout, int, 0644); static ulong jiffies_till_first_fqs = RCU_JIFFIES_TILL_FORCE_QS; static ulong jiffies_till_next_fqs = RCU_JIFFIES_TILL_FORCE_QS; module_param(jiffies_till_first_fqs, ulong, 0644); module_param(jiffies_till_next_fqs, ulong, 0644); static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *)); static void force_quiescent_state(struct rcu_state *rsp); static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */ long rcu_batches_completed_sched(void) { return rcu_sched_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */ long rcu_batches_completed_bh(void) { return rcu_bh_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */ void rcu_bh_force_quiescent_state(void) { force_quiescent_state(&rcu_bh_state); } EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */ void rcutorture_record_test_transition(void) { rcutorture_testseq++; rcutorture_vernum = 0; } EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); /* * Record the number of writer passes through the current rcutorture test. * This is also used to correlate debugfs tracing stats with the rcutorture * messages. */ void rcutorture_record_progress(unsigned long vernum) { rcutorture_vernum++; } EXPORT_SYMBOL_GPL(rcutorture_record_progress); /* * Force a quiescent state for RCU-sched. */ void rcu_sched_force_quiescent_state(void) { force_quiescent_state(&rcu_sched_state); } EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */ static int cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) { return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] && rdp->nxttail[RCU_DONE_TAIL] != NULL; } /* * Does the current CPU require a yet-as-unscheduled grace period? */ static int cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) { struct rcu_head **ntp; ntp = rdp->nxttail[RCU_DONE_TAIL + (ACCESS_ONCE(rsp->completed) != rdp->completed)]; return rdp->nxttail[RCU_DONE_TAIL] && ntp && *ntp && !rcu_gp_in_progress(rsp); } /* * Return the root node of the specified rcu_state structure. */ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) { return &rsp->node[0]; } /* * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state * * If the new value of the ->dynticks_nesting counter now is zero, * we really have entered idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, bool user) { trace_rcu_dyntick("Start", oldval, 0); if (!user && !is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on entry: not idle task", oldval, 0); ftrace_dump(DUMP_ORIG); WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s", current->pid, current->comm, idle->pid, idle->comm); /* must be idle task! */ } rcu_prepare_for_idle(smp_processor_id()); /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic_inc(); /* See above. */ atomic_inc(&rdtp->dynticks); smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */ WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); /* * It is illegal to enter an extended quiescent state while * in an RCU read-side critical section. */ rcu_lockdep_assert(!lock_is_held(&rcu_lock_map), "Illegal idle entry in RCU read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), "Illegal idle entry in RCU-bh read-side critical section."); rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), "Illegal idle entry in RCU-sched read-side critical section."); } /* * Enter an RCU extended quiescent state, which can be either the * idle loop or adaptive-tickless usermode execution. */ static void rcu_eqs_enter(bool user) { long long oldval; struct rcu_dynticks *rdtp; rdtp = &__get_cpu_var(rcu_dynticks); oldval = rdtp->dynticks_nesting; WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) rdtp->dynticks_nesting = 0; else rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE; rcu_eqs_enter_common(rdtp, oldval, user); } /** * rcu_idle_enter - inform RCU that current CPU is entering idle * * Enter idle mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur. (Though RCU read-side * critical sections can occur in irq handlers in idle, a possibility * handled by irq_enter() and irq_exit().) * * We crowbar the ->dynticks_nesting field to zero to allow for * the possibility of usermode upcalls having messed up our count * of interrupt nesting level during the prior busy period. */ void rcu_idle_enter(void) { unsigned long flags; local_irq_save(flags); rcu_eqs_enter(false); local_irq_restore(flags); } EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_RCU_USER_QS /** * rcu_user_enter - inform RCU that we are resuming userspace. * * Enter RCU idle mode right before resuming userspace. No use of RCU * is permitted between this call and rcu_user_exit(). This way the * CPU doesn't need to maintain the tick for RCU maintenance purposes * when the CPU runs in userspace. */ void rcu_user_enter(void) { rcu_eqs_enter(1); } /** * rcu_user_enter_after_irq - inform RCU that we are going to resume userspace * after the current irq returns. * * This is similar to rcu_user_enter() but in the context of a non-nesting * irq. After this call, RCU enters into idle mode when the interrupt * returns. */ void rcu_user_enter_after_irq(void) { unsigned long flags; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); /* Ensure this irq is interrupting a non-idle RCU state. */ WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK)); rdtp->dynticks_nesting = 1; local_irq_restore(flags); } #endif /* CONFIG_RCU_USER_QS */ /** * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle * * Exit from an interrupt handler, which might possibly result in entering * idle mode, in other words, leaving the mode in which read-side critical * sections can occur. * * This code assumes that the idle loop never does anything that might * result in unbalanced calls to irq_enter() and irq_exit(). If your * architecture violates this assumption, RCU will give you what you * deserve, good and hard. But very infrequently and irreproducibly. * * Use things like work queues to work around this limitation. * * You have been warned. */ void rcu_irq_exit(void) { unsigned long flags; long long oldval; struct rcu_dynticks *rdtp; local_irq_save(flags); rdtp = &__get_cpu_var(rcu_dynticks); oldval = rdtp->dynticks_nesting; rdtp->dynticks_nesting--; WARN_ON_ONCE(rdtp->dynticks_nesting < 0); if (rdtp->dynticks_nesting) trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting); else rcu_eqs_enter_common(rdtp, oldval, true); local_irq_restore(flags); } /* * rcu_eqs_exit_common - current CPU moving away from extended quiescent state * * If the new value of the ->dynticks_nesting counter was previously zero, * we really have exited idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */ static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, int user) { smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ atomic_inc(&rdtp->dynticks); /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic_inc(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); rcu_cleanup_after_idle(smp_processor_id()); trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting); if (!user && !is_idle_task(current)) { struct task_struct *idle = idle_task(smp_processor_id()); trace_rcu_dyntick("Error on exit: not idle task", oldval, rdtp->dynticks_nesting);