/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> *	    Manfred Spraul <manfred@colorfullife.com> *	    Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - *	Documentation/RCU */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/stop_machine.h>#include <linux/random.h> #include "rcutree.h"#include <trace/events/rcu.h> #include "rcu.h" /* Data structures. */ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; #define RCU_STATE_INITIALIZER(sname, cr) { \	.level = { &sname##_state.node[0] }, \	.call = cr, \	.fqs_state = RCU_GP_IDLE, \	.gpnum = 0UL - 300UL, \	.completed = 0UL - 300UL, \	.orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \	.orphan_nxttail = &sname##_state.orphan_nxtlist, \	.orphan_donetail = &sname##_state.orphan_donelist, \	.barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \	.onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \	.name = #sname, \} struct rcu_state rcu_sched_state =	RCU_STATE_INITIALIZER(rcu_sched, call_rcu_sched);DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh, call_rcu_bh);DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); static struct rcu_state *rcu_state;LIST_HEAD(rcu_struct_flavors); /* Increase (but not decrease) the CONFIG_RCU_FANOUT_LEAF at boot time. */static int rcu_fanout_leaf = CONFIG_RCU_FANOUT_LEAF;module_param(rcu_fanout_leaf, int, 0444);int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;static int num_rcu_lvl[] = {  /* Number of rcu_nodes at specified level. */	NUM_RCU_LVL_0,	NUM_RCU_LVL_1,	NUM_RCU_LVL_2,	NUM_RCU_LVL_3,	NUM_RCU_LVL_4,};int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned.  So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_sched() to a simple barrier().  When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods.  This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks.  So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one.  We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; #ifdef CONFIG_RCU_BOOST /* * Control variables for per-CPU and per-rcu_node kthreads.  These * handle all flavors of RCU. */static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);DEFINE_PER_CPU(char, rcu_cpu_has_work); #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);static void invoke_rcu_core(void);static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); /* * Track the rcutorture test sequence number and the update version * number within a given test.  The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running.  The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */unsigned long rcutorture_testseq;unsigned long rcutorture_vernum; /* * Return true if an RCU grace period is in progress.  The ACCESS_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(struct rcu_state *rsp){	return ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum);} /* * Note a quiescent state.  Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. * The caller must have disabled preemption. */void rcu_sched_qs(int cpu){	struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu); 	if (rdp->passed_quiesce == 0)		trace_rcu_grace_period("rcu_sched", rdp->gpnum, "cpuqs");	rdp->passed_quiesce = 1;} void rcu_bh_qs(int cpu){	struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); 	if (rdp->passed_quiesce == 0)		trace_rcu_grace_period("rcu_bh", rdp->gpnum, "cpuqs");	rdp->passed_quiesce = 1;} /* * Note a context switch.  This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */void rcu_note_context_switch(int cpu){	trace_rcu_utilization("Start context switch");	rcu_sched_qs(cpu);	rcu_preempt_note_context_switch(cpu);	trace_rcu_utilization("End context switch");}EXPORT_SYMBOL_GPL(rcu_note_context_switch); DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {	.dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,	.dynticks = ATOMIC_INIT(1),}; static long blimit = 10;	/* Maximum callbacks per rcu_do_batch. */static long qhimark = 10000;	/* If this many pending, ignore blimit. */static long qlowmark = 100;	/* Once only this many pending, use blimit. */ module_param(blimit, long, 0444);module_param(qhimark, long, 0444);module_param(qlowmark, long, 0444); static ulong jiffies_till_first_fqs = RCU_JIFFIES_TILL_FORCE_QS;static ulong jiffies_till_next_fqs = RCU_JIFFIES_TILL_FORCE_QS; module_param(jiffies_till_first_fqs, ulong, 0644);module_param(jiffies_till_next_fqs, ulong, 0644); static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *));static void force_quiescent_state(struct rcu_state *rsp);static int rcu_pending(int cpu); /* * Return the number of RCU-sched batches processed thus far for debug & stats. */long rcu_batches_completed_sched(void){	return rcu_sched_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches processed thus far for debug & stats. */long rcu_batches_completed_bh(void){	return rcu_bh_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state for RCU BH. */void rcu_bh_force_quiescent_state(void){	force_quiescent_state(&rcu_bh_state);}EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Record the number of times rcutorture tests have been initiated and * terminated.  This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded.  In other words, we cannot * store this state in rcutorture itself. */void rcutorture_record_test_transition(void){	rcutorture_testseq++;	rcutorture_vernum = 0;}EXPORT_SYMBOL_GPL(rcutorture_record_test_transition); /* * Record the number of writer passes through the current rcutorture test. * This is also used to correlate debugfs tracing stats with the rcutorture * messages. */void rcutorture_record_progress(unsigned long vernum){	rcutorture_vernum++;}EXPORT_SYMBOL_GPL(rcutorture_record_progress); /* * Force a quiescent state for RCU-sched. */void rcu_sched_force_quiescent_state(void){	force_quiescent_state(&rcu_sched_state);}EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Does the CPU have callbacks ready to be invoked? */static intcpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp){	return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL] &&	       rdp->nxttail[RCU_DONE_TAIL] != NULL;} /* * Does the current CPU require a not-yet-started grace period? * The caller must have disabled interrupts to prevent races with * normal callback registry. */static intcpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp){	int i; 	if (rcu_gp_in_progress(rsp))		return 0;  /* No, a grace period is already in progress. */	if (!rdp->nxttail[RCU_NEXT_TAIL])		return 0;  /* No, this is a no-CBs (or offline) CPU. */	if (*rdp->nxttail[RCU_NEXT_READY_TAIL])		return 1;  /* Yes, this CPU has newly registered callbacks. */	for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)		if (rdp->nxttail[i - 1] != rdp->nxttail[i] &&		    ULONG_CMP_LT(ACCESS_ONCE(rsp->completed),				 rdp->nxtcompleted[i]))			return 1;  /* Yes, CBs for future grace period. */	return 0; /* No grace period needed. */} /* * Return the root node of the specified rcu_state structure. */static struct rcu_node *rcu_get_root(struct rcu_state *rsp){	return &rsp->node[0];} /* * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state * * If the new value of the ->dynticks_nesting counter now is zero, * we really have entered idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,				bool user){	trace_rcu_dyntick("Start", oldval, rdtp->dynticks_nesting);	if (!user && !is_idle_task(current)) {		struct task_struct *idle = idle_task(smp_processor_id()); 		trace_rcu_dyntick("Error on entry: not idle task", oldval, 0);		ftrace_dump(DUMP_ORIG);		WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",			  current->pid, current->comm,			  idle->pid, idle->comm); /* must be idle task! */	}	rcu_prepare_for_idle(smp_processor_id());	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */	smp_mb__before_atomic_inc();  /* See above. */	atomic_inc(&rdtp->dynticks);	smp_mb__after_atomic_inc();  /* Force ordering with next sojourn. */	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1); 	/*	 * It is illegal to enter an extended quiescent state while	 * in an RCU read-side critical section.	 */	rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),			   "Illegal idle entry in RCU read-side critical section.");	rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map),			   "Illegal idle entry in RCU-bh read-side critical section.");	rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map),			   "Illegal idle entry in RCU-sched read-side critical section.");} /* * Enter an RCU extended quiescent state, which can be either the * idle loop or adaptive-tickless usermode execution. */static void rcu_eqs_enter(bool user){	long long oldval;	struct rcu_dynticks *rdtp; 	rdtp = &__get_cpu_var(rcu_dynticks);	oldval = rdtp->dynticks_nesting;	WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);	if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE)		rdtp->dynticks_nesting = 0;	else		rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;	rcu_eqs_enter_common(rdtp, oldval, user);} /** * rcu_idle_enter - inform RCU that current CPU is entering idle * * Enter idle mode, in other words, -leave- the mode in which RCU * read-side critical sections can occur.  (Though RCU read-side * critical sections can occur in irq handlers in idle, a possibility * handled by irq_enter() and irq_exit().) * * We crowbar the ->dynticks_nesting field to zero to allow for * the possibility of usermode upcalls having messed up our count * of interrupt nesting level during the prior busy period. */void rcu_idle_enter(void){	unsigned long flags; 	local_irq_save(flags);	rcu_eqs_enter(false);	local_irq_restore(flags);}EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_RCU_USER_QS/** * rcu_user_enter - inform RCU that we are resuming userspace. * * Enter RCU idle mode right before resuming userspace.  No use of RCU * is permitted between this call and rcu_user_exit(). This way the * CPU doesn't need to maintain the tick for RCU maintenance purposes * when the CPU runs in userspace. */void rcu_user_enter(void){	rcu_eqs_enter(1);} /** * rcu_user_enter_after_irq - inform RCU that we are going to resume userspace * after the current irq returns. * * This is similar to rcu_user_enter() but in the context of a non-nesting * irq. After this call, RCU enters into idle mode when the interrupt * returns. */void rcu_user_enter_after_irq(void){	unsigned long flags;	struct rcu_dynticks *rdtp; 	local_irq_save(flags);	rdtp = &__get_cpu_var(rcu_dynticks);	/* Ensure this irq is interrupting a non-idle RCU state.  */	WARN_ON_ONCE(!(rdtp->dynticks_nesting & DYNTICK_TASK_MASK));	rdtp->dynticks_nesting = 1;	local_irq_restore(flags);}#endif /* CONFIG_RCU_USER_QS */ /** * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle * * Exit from an interrupt handler, which might possibly result in entering * idle mode, in other words, leaving the mode in which read-side critical * sections can occur. * * This code assumes that the idle loop never does anything that might * result in unbalanced calls to irq_enter() and irq_exit().  If your * architecture violates this assumption, RCU will give you what you * deserve, good and hard.  But very infrequently and irreproducibly. * * Use things like work queues to work around this limitation. * * You have been warned. */void rcu_irq_exit(void){	unsigned long flags;	long long oldval;	struct rcu_dynticks *rdtp; 	local_irq_save(flags);	rdtp = &__get_cpu_var(rcu_dynticks);	oldval = rdtp->dynticks_nesting;	rdtp->dynticks_nesting--;	WARN_ON_ONCE(rdtp->dynticks_nesting < 0);	if (rdtp->dynticks_nesting)		trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting);	else		rcu_eqs_enter_common(rdtp, oldval, true);	local_irq_restore(flags);} /* * rcu_eqs_exit_common - current CPU moving away from extended quiescent state * * If the new value of the ->dynticks_nesting counter was previously zero, * we really have exited idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,			       int user){	smp_mb__before_atomic_inc();  /* Force ordering w/previous sojourn. */	atomic_inc(&rdtp->dynticks);	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */	smp_mb__after_atomic_inc();  /* See above. */	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));	rcu_cleanup_after_idle(smp_processor_id());	trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);	if (!user && !is_idle_task(current)) {

Lines 1–500 of 3186. Load more