/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/module.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/stop_machine.h>#include <linux/random.h>#include <linux/trace_events.h>#include <linux/suspend.h> #include "tree.h"#include "rcu.h" MODULE_ALIAS("rcutree");#ifdef MODULE_PARAM_PREFIX#undef MODULE_PARAM_PREFIX#endif#define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; /* * In order to export the rcu_state name to the tracing tools, it * needs to be added in the __tracepoint_string section. * This requires defining a separate variable tp_<sname>_varname * that points to the string being used, and this will allow * the tracing userspace tools to be able to decipher the string * address to the matching string. */#ifdef CONFIG_TRACING# define DEFINE_RCU_TPS(sname) \static char sname##_varname[] = #sname; \static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname;# define RCU_STATE_NAME(sname) sname##_varname#else# define DEFINE_RCU_TPS(sname)# define RCU_STATE_NAME(sname) __stringify(sname)#endif #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \DEFINE_RCU_TPS(sname) \static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \struct rcu_state sname##_state = { \ .level = { &sname##_state.node[0] }, \ .rda = &sname##_data, \ .call = cr, \ .fqs_state = RCU_GP_IDLE, \ .gpnum = 0UL - 300UL, \ .completed = 0UL - 300UL, \ .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \ .orphan_nxttail = &sname##_state.orphan_nxtlist, \ .orphan_donetail = &sname##_state.orphan_donelist, \ .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ .name = RCU_STATE_NAME(sname), \ .abbr = sabbr, \} RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); static struct rcu_state *const rcu_state_p;static struct rcu_data __percpu *const rcu_data_p;LIST_HEAD(rcu_struct_flavors); /* Dump rcu_node combining tree at boot to verify correct setup. */static bool dump_tree;module_param(dump_tree, bool, 0444);/* Control rcu_node-tree auto-balancing at boot time. */static bool rcu_fanout_exact;module_param(rcu_fanout_exact, bool, 0444);/* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */static int rcu_fanout_leaf = RCU_FANOUT_LEAF;module_param(rcu_fanout_leaf, int, 0444);int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;static int num_rcu_lvl[] = { /* Number of rcu_nodes at specified level. */ NUM_RCU_LVL_0, NUM_RCU_LVL_1, NUM_RCU_LVL_2, NUM_RCU_LVL_3, NUM_RCU_LVL_4,};int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);static void invoke_rcu_core(void);static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); /* rcuc/rcub kthread realtime priority */#ifdef CONFIG_RCU_KTHREAD_PRIOstatic int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;#else /* #ifdef CONFIG_RCU_KTHREAD_PRIO */static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;#endif /* #else #ifdef CONFIG_RCU_KTHREAD_PRIO */module_param(kthread_prio, int, 0644); /* Delay in jiffies for grace-period initialization delays, debug only. */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINITstatic int gp_preinit_delay = CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT_DELAY;module_param(gp_preinit_delay, int, 0644);#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */static const int gp_preinit_delay;#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INITstatic int gp_init_delay = CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY;module_param(gp_init_delay, int, 0644);#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */static const int gp_init_delay;#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUPstatic int gp_cleanup_delay = CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY;module_param(gp_cleanup_delay, int, 0644);#else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */static const int gp_cleanup_delay;#endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */ /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */#define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */ /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */unsigned long rcutorture_testseq;unsigned long rcutorture_vernum; /* * Compute the mask of online CPUs for the specified rcu_node structure. * This will not be stable unless the rcu_node structure's ->lock is * held, but the bit corresponding to the current CPU will be stable * in most contexts. */unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp){ return READ_ONCE(rnp->qsmaskinitnext);} /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(struct rcu_state *rsp){ return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum);} /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. * The caller must have disabled preemption. */void rcu_sched_qs(void){ if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) { trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_sched_data.gpnum), TPS("cpuqs")); __this_cpu_write(rcu_sched_data.passed_quiesce, 1); }} void rcu_bh_qs(void){ if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { trace_rcu_grace_period(TPS("rcu_bh"), __this_cpu_read(rcu_bh_data.gpnum), TPS("cpuqs")); __this_cpu_write(rcu_bh_data.passed_quiesce, 1); }} static DEFINE_PER_CPU(int, rcu_sched_qs_mask); static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE, .dynticks = ATOMIC_INIT(1),#ifdef CONFIG_NO_HZ_FULL_SYSIDLE .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE, .dynticks_idle = ATOMIC_INIT(1),#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */}; DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr); /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle * period, which we in turn do by incrementing the ->dynticks counter * by two. */static void rcu_momentary_dyntick_idle(void){ unsigned long flags; struct rcu_data *rdp; struct rcu_dynticks *rdtp; int resched_mask; struct rcu_state *rsp; local_irq_save(flags); /* * Yes, we can lose flag-setting operations. This is OK, because * the flag will be set again after some delay. */ resched_mask = raw_cpu_read(rcu_sched_qs_mask); raw_cpu_write(rcu_sched_qs_mask, 0); /* Find the flavor that needs a quiescent state. */ for_each_rcu_flavor(rsp) { rdp = raw_cpu_ptr(rsp->rda); if (!(resched_mask & rsp->flavor_mask)) continue; smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */ if (READ_ONCE(rdp->mynode->completed) != READ_ONCE(rdp->cond_resched_completed)) continue; /* * Pretend to be momentarily idle for the quiescent state. * This allows the grace-period kthread to record the * quiescent state, with no need for this CPU to do anything * further. */ rdtp = this_cpu_ptr(&rcu_dynticks); smp_mb__before_atomic(); /* Earlier stuff before QS. */ atomic_add(2, &rdtp->dynticks); /* QS. */ smp_mb__after_atomic(); /* Later stuff after QS. */ break; } local_irq_restore(flags);} /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */void rcu_note_context_switch(void){ trace_rcu_utilization(TPS("Start context switch")); rcu_sched_qs(); rcu_preempt_note_context_switch(); if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) rcu_momentary_dyntick_idle(); trace_rcu_utilization(TPS("End context switch"));}EXPORT_SYMBOL_GPL(rcu_note_context_switch); /* * Register a quiescent state for all RCU flavors. If there is an * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight * dyntick-idle quiescent state visible to other CPUs (but only for those * RCU flavors in desperate need of a quiescent state, which will normally * be none of them). Either way, do a lightweight quiescent state for * all RCU flavors. */void rcu_all_qs(void){ if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) rcu_momentary_dyntick_idle(); this_cpu_inc(rcu_qs_ctr);}EXPORT_SYMBOL_GPL(rcu_all_qs); static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */static long qhimark = 10000; /* If this many pending, ignore blimit. */static long qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, long, 0444);module_param(qhimark, long, 0444);module_param(qlowmark, long, 0444); static ulong jiffies_till_first_fqs = ULONG_MAX;static ulong jiffies_till_next_fqs = ULONG_MAX; module_param(jiffies_till_first_fqs, ulong, 0644);module_param(jiffies_till_next_fqs, ulong, 0644); /* * How long the grace period must be before we start recruiting * quiescent-state help from rcu_note_context_switch(). */static ulong jiffies_till_sched_qs = HZ / 20;module_param(jiffies_till_sched_qs, ulong, 0644); static bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp);static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp, bool *isidle, unsigned long *maxj), bool *isidle, unsigned long *maxj);static void force_quiescent_state(struct rcu_state *rsp);static int rcu_pending(void); /* * Return the number of RCU batches started thus far for debug & stats. */unsigned long rcu_batches_started(void){ return rcu_state_p->gpnum;}EXPORT_SYMBOL_GPL(rcu_batches_started); /* * Return the number of RCU-sched batches started thus far for debug & stats. */unsigned long rcu_batches_started_sched(void){ return rcu_sched_state.gpnum;}EXPORT_SYMBOL_GPL(rcu_batches_started_sched); /* * Return the number of RCU BH batches started thus far for debug & stats. */unsigned long rcu_batches_started_bh(void){ return rcu_bh_state.gpnum;}EXPORT_SYMBOL_GPL(rcu_batches_started_bh); /* * Return the number of RCU batches completed thus far for debug & stats. */unsigned long rcu_batches_completed(void){ return rcu_state_p->completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed); /* * Return the number of RCU-sched batches completed thus far for debug & stats. */unsigned long rcu_batches_completed_sched(void){ return rcu_sched_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches completed thus far for debug & stats. */unsigned long rcu_batches_completed_bh(void){ return rcu_bh_state.completed;}EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state. */void rcu_force_quiescent_state(void){ force_quiescent_state(rcu_state_p);}EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); /* * Force a quiescent state for RCU BH. */void rcu_bh_force_quiescent_state(void){ force_quiescent_state(&rcu_bh_state);}EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Force a quiescent state for RCU-sched. */void rcu_sched_force_quiescent_state(void){ force_quiescent_state(&rcu_sched_state);}EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Show the state of the grace-period kthreads. */void show_rcu_gp_kthreads(void){ struct rcu_state *rsp; for_each_rcu_flavor(rsp) { pr_info("%s: wait state: %d ->state: %#lx\n", rsp->name, rsp->gp_state, rsp->gp_kthread->state); /* sched_show_task(rsp->gp_kthread); */ }}EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */Lines 1–500 of 4195. Load more
/* * Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version * * Based on the original work by Paul McKenney <paulmck@us.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/export.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/module.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include <linux/wait.h> #include <linux/kthread.h> #include <linux/prefetch.h> #include <linux/delay.h> #include <linux/stop_machine.h> #include <linux/random.h> #include <linux/trace_events.h> #include <linux/suspend.h> #include "tree.h" #include "rcu.h" MODULE_ALIAS("rcutree"); #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif #define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */ static struct lock_class_key rcu_node_class[RCU_NUM_LVLS]; static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS]; /* * In order to export the rcu_state name to the tracing tools, it * needs to be added in the __tracepoint_string section. * This requires defining a separate variable tp_<sname>_varname * that points to the string being used, and this will allow * the tracing userspace tools to be able to decipher the string * address to the matching string. */ #ifdef CONFIG_TRACING # define DEFINE_RCU_TPS(sname) \ static char sname##_varname[] = #sname; \ static const char *tp_##sname##_varname __used __tracepoint_string = sname##_varname; # define RCU_STATE_NAME(sname) sname##_varname #else # define DEFINE_RCU_TPS(sname) # define RCU_STATE_NAME(sname) __stringify(sname) #endif #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \ DEFINE_RCU_TPS(sname) \ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \ struct rcu_state sname##_state = { \ .level = { &sname##_state.node[0] }, \ .rda = &sname##_data, \ .call = cr, \ .fqs_state = RCU_GP_IDLE, \ .gpnum = 0UL - 300UL, \ .completed = 0UL - 300UL, \ .orphan_lock = __RAW_SPIN_LOCK_UNLOCKED(&sname##_state.orphan_lock), \ .orphan_nxttail = &sname##_state.orphan_nxtlist, \ .orphan_donetail = &sname##_state.orphan_donelist, \ .barrier_mutex = __MUTEX_INITIALIZER(sname##_state.barrier_mutex), \ .name = RCU_STATE_NAME(sname), \ .abbr = sabbr, \ } RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); static struct rcu_state *const rcu_state_p; static struct rcu_data __percpu *const rcu_data_p; LIST_HEAD(rcu_struct_flavors); /* Dump rcu_node combining tree at boot to verify correct setup. */ static bool dump_tree; module_param(dump_tree, bool, 0444); /* Control rcu_node-tree auto-balancing at boot time. */ static bool rcu_fanout_exact; module_param(rcu_fanout_exact, bool, 0444); /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */ static int rcu_fanout_leaf = RCU_FANOUT_LEAF; module_param(rcu_fanout_leaf, int, 0444); int rcu_num_lvls __read_mostly = RCU_NUM_LVLS; static int num_rcu_lvl[] = { /* Number of rcu_nodes at specified level. */ NUM_RCU_LVL_0, NUM_RCU_LVL_1, NUM_RCU_LVL_2, NUM_RCU_LVL_3, NUM_RCU_LVL_4, }; int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable transitions from zero to one just * before the first task is spawned. So when this variable is zero, RCU * can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_sched() to a simple barrier(). When this variable * is one, RCU must actually do all the hard work required to detect real * grace periods. This variable is also used to suppress boot-time false * positives from lockdep-RCU error checking. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */ static int rcu_scheduler_fully_active __read_mostly; static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); /* rcuc/rcub kthread realtime priority */ #ifdef CONFIG_RCU_KTHREAD_PRIO static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO; #else /* #ifdef CONFIG_RCU_KTHREAD_PRIO */ static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; #endif /* #else #ifdef CONFIG_RCU_KTHREAD_PRIO */ module_param(kthread_prio, int, 0644); /* Delay in jiffies for grace-period initialization delays, debug only. */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT static int gp_preinit_delay = CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT_DELAY; module_param(gp_preinit_delay, int, 0644); #else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */ static const int gp_preinit_delay; #endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_PREINIT */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT static int gp_init_delay = CONFIG_RCU_TORTURE_TEST_SLOW_INIT_DELAY; module_param(gp_init_delay, int, 0644); #else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */ static const int gp_init_delay; #endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_INIT */ #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP static int gp_cleanup_delay = CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP_DELAY; module_param(gp_cleanup_delay, int, 0644); #else /* #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */ static const int gp_cleanup_delay; #endif /* #else #ifdef CONFIG_RCU_TORTURE_TEST_SLOW_CLEANUP */ /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */ #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */ /* * Track the rcutorture test sequence number and the update version * number within a given test. The rcutorture_testseq is incremented * on every rcutorture module load and unload, so has an odd value * when a test is running. The rcutorture_vernum is set to zero * when rcutorture starts and is incremented on each rcutorture update. * These variables enable correlating rcutorture output with the * RCU tracing information. */ unsigned long rcutorture_testseq; unsigned long rcutorture_vernum; /* * Compute the mask of online CPUs for the specified rcu_node structure. * This will not be stable unless the rcu_node structure's ->lock is * held, but the bit corresponding to the current CPU will be stable * in most contexts. */ unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) { return READ_ONCE(rnp->qsmaskinitnext); } /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(struct rcu_state *rsp) { return READ_ONCE(rsp->completed) != READ_ONCE(rsp->gpnum); } /* * Note a quiescent state. Because we do not need to know * how many quiescent states passed, just if there was at least * one since the start of the grace period, this just sets a flag. * The caller must have disabled preemption. */ void rcu_sched_qs(void) { if (!__this_cpu_read(rcu_sched_data.passed_quiesce)) { trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_sched_data.gpnum), TPS("cpuqs")); __this_cpu_write(rcu_sched_data.passed_quiesce, 1); } } void rcu_bh_qs(void) { if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { trace_rcu_grace_period(TPS("rcu_bh"), __this_cpu_read(rcu_bh_data.gpnum), TPS("cpuqs")); __this_cpu_write(rcu_bh_data.passed_quiesce, 1); } } static DEFINE_PER_CPU(int, rcu_sched_qs_mask); static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = { .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE, .dynticks = ATOMIC_INIT(1), #ifdef CONFIG_NO_HZ_FULL_SYSIDLE .dynticks_idle_nesting = DYNTICK_TASK_NEST_VALUE, .dynticks_idle = ATOMIC_INIT(1), #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ }; DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr); EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr); /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle * period, which we in turn do by incrementing the ->dynticks counter * by two. */ static void rcu_momentary_dyntick_idle(void) { unsigned long flags; struct rcu_data *rdp; struct rcu_dynticks *rdtp; int resched_mask; struct rcu_state *rsp; local_irq_save(flags); /* * Yes, we can lose flag-setting operations. This is OK, because * the flag will be set again after some delay. */ resched_mask = raw_cpu_read(rcu_sched_qs_mask); raw_cpu_write(rcu_sched_qs_mask, 0); /* Find the flavor that needs a quiescent state. */ for_each_rcu_flavor(rsp) { rdp = raw_cpu_ptr(rsp->rda); if (!(resched_mask & rsp->flavor_mask)) continue; smp_mb(); /* rcu_sched_qs_mask before cond_resched_completed. */ if (READ_ONCE(rdp->mynode->completed) != READ_ONCE(rdp->cond_resched_completed)) continue; /* * Pretend to be momentarily idle for the quiescent state. * This allows the grace-period kthread to record the * quiescent state, with no need for this CPU to do anything * further. */ rdtp = this_cpu_ptr(&rcu_dynticks); smp_mb__before_atomic(); /* Earlier stuff before QS. */ atomic_add(2, &rdtp->dynticks); /* QS. */ smp_mb__after_atomic(); /* Later stuff after QS. */ break; } local_irq_restore(flags); } /* * Note a context switch. This is a quiescent state for RCU-sched, * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */ void rcu_note_context_switch(void) { trace_rcu_utilization(TPS("Start context switch")); rcu_sched_qs(); rcu_preempt_note_context_switch(); if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) rcu_momentary_dyntick_idle(); trace_rcu_utilization(TPS("End context switch")); } EXPORT_SYMBOL_GPL(rcu_note_context_switch); /* * Register a quiescent state for all RCU flavors. If there is an * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight * dyntick-idle quiescent state visible to other CPUs (but only for those * RCU flavors in desperate need of a quiescent state, which will normally * be none of them). Either way, do a lightweight quiescent state for * all RCU flavors. */ void rcu_all_qs(void) { if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) rcu_momentary_dyntick_idle(); this_cpu_inc(rcu_qs_ctr); } EXPORT_SYMBOL_GPL(rcu_all_qs); static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */ static long qhimark = 10000; /* If this many pending, ignore blimit. */ static long qlowmark = 100; /* Once only this many pending, use blimit. */ module_param(blimit, long, 0444); module_param(qhimark, long, 0444); module_param(qlowmark, long, 0444); static ulong jiffies_till_first_fqs = ULONG_MAX; static ulong jiffies_till_next_fqs = ULONG_MAX; module_param(jiffies_till_first_fqs, ulong, 0644); module_param(jiffies_till_next_fqs, ulong, 0644); /* * How long the grace period must be before we start recruiting * quiescent-state help from rcu_note_context_switch(). */ static ulong jiffies_till_sched_qs = HZ / 20; module_param(jiffies_till_sched_qs, ulong, 0644); static bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp); static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp, bool *isidle, unsigned long *maxj), bool *isidle, unsigned long *maxj); static void force_quiescent_state(struct rcu_state *rsp); static int rcu_pending(void); /* * Return the number of RCU batches started thus far for debug & stats. */ unsigned long rcu_batches_started(void) { return rcu_state_p->gpnum; } EXPORT_SYMBOL_GPL(rcu_batches_started); /* * Return the number of RCU-sched batches started thus far for debug & stats. */ unsigned long rcu_batches_started_sched(void) { return rcu_sched_state.gpnum; } EXPORT_SYMBOL_GPL(rcu_batches_started_sched); /* * Return the number of RCU BH batches started thus far for debug & stats. */ unsigned long rcu_batches_started_bh(void) { return rcu_bh_state.gpnum; } EXPORT_SYMBOL_GPL(rcu_batches_started_bh); /* * Return the number of RCU batches completed thus far for debug & stats. */ unsigned long rcu_batches_completed(void) { return rcu_state_p->completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed); /* * Return the number of RCU-sched batches completed thus far for debug & stats. */ unsigned long rcu_batches_completed_sched(void) { return rcu_sched_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); /* * Return the number of RCU BH batches completed thus far for debug & stats. */ unsigned long rcu_batches_completed_bh(void) { return rcu_bh_state.completed; } EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); /* * Force a quiescent state. */ void rcu_force_quiescent_state(void) { force_quiescent_state(rcu_state_p); } EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); /* * Force a quiescent state for RCU BH. */ void rcu_bh_force_quiescent_state(void) { force_quiescent_state(&rcu_bh_state); } EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); /* * Force a quiescent state for RCU-sched. */ void rcu_sched_force_quiescent_state(void) { force_quiescent_state(&rcu_sched_state); } EXPORT_SYMBOL_GPL(rcu_sched_force_quiescent_state); /* * Show the state of the grace-period kthreads. */ void show_rcu_gp_kthreads(void) { struct rcu_state *rsp; for_each_rcu_flavor(rsp) { pr_info("%s: wait state: %d ->state: %#lx\n", rsp->name, rsp->gp_state, rsp->gp_kthread->state); /* sched_show_task(rsp->gp_kthread); */ } } EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); /* * Record the number of times rcutorture tests have been initiated and * terminated. This information allows the debugfs tracing stats to be * correlated to the rcutorture messages, even when the rcutorture module * is being repeatedly loaded and unloaded. In other words, we cannot * store this state in rcutorture itself. */