// SPDX-License-Identifier: GPL-2.0+/* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.ibm.com> * * Based on the original work by Paul McKenney <paulmck@linux.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #define pr_fmt(fmt) "rcu: " fmt #include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate_wait.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/sched/debug.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/kmemleak.h>#include <linux/moduleparam.h>#include <linux/panic.h>#include <linux/panic_notifier.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <uapi/linux/sched/types.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/random.h>#include <linux/trace_events.h>#include <linux/suspend.h>#include <linux/ftrace.h>#include <linux/tick.h>#include <linux/sysrq.h>#include <linux/kprobes.h>#include <linux/gfp.h>#include <linux/oom.h>#include <linux/smpboot.h>#include <linux/jiffies.h>#include <linux/slab.h>#include <linux/sched/isolation.h>#include <linux/sched/clock.h>#include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/kasan.h>#include <linux/context_tracking.h>#include "../time/tick-internal.h" #include "tree.h"#include "rcu.h" #ifdef MODULE_PARAM_PREFIX#undef MODULE_PARAM_PREFIX#endif#define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */static void rcu_sr_normal_gp_cleanup_work(struct work_struct *); static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .gpwrap = true,}; int rcu_get_gpwrap_count(int cpu){ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); return READ_ONCE(rdp->gpwrap_count);}EXPORT_SYMBOL_GPL(rcu_get_gpwrap_count); static struct rcu_state rcu_state = { .level = { &rcu_state.node[0] }, .gp_state = RCU_GP_IDLE, .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex), .barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock), .name = RCU_NAME, .abbr = RCU_ABBR, .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex), .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex), .ofl_lock = __ARCH_SPIN_LOCK_UNLOCKED, .srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work, rcu_sr_normal_gp_cleanup_work), .srs_cleanups_pending = ATOMIC_INIT(0),#ifdef CONFIG_RCU_NOCB_CPU .nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex),#endif}; /* Dump rcu_node combining tree at boot to verify correct setup. */static bool dump_tree;module_param(dump_tree, bool, 0444);/* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT);#ifndef CONFIG_PREEMPT_RTmodule_param(use_softirq, bool, 0444);#endif/* Control rcu_node-tree auto-balancing at boot time. */static bool rcu_fanout_exact;module_param(rcu_fanout_exact, bool, 0444);/* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */static int rcu_fanout_leaf = RCU_FANOUT_LEAF;module_param(rcu_fanout_leaf, int, 0444);int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;/* Number of rcu_nodes at specified level. */int num_rcu_lvl[] = NUM_RCU_LVL_INIT;int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable is initialized to the value * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, * RCU can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_rcu() to a simple barrier(). When this variable * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required * to detect real grace periods. This variable is also used to suppress * boot-time false positives from lockdep-RCU error checking. Finally, it * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU * is fully initialized, including all of its kthreads having been spawned. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, unsigned long gps, unsigned long flags);static void invoke_rcu_core(void);static void rcu_report_exp_rdp(struct rcu_data *rdp);static void rcu_report_qs_rdp(struct rcu_data *rdp);static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);static bool rcu_rdp_cpu_online(struct rcu_data *rdp);static bool rcu_init_invoked(void);static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); /* * rcuc/rcub/rcuop kthread realtime priority. The "rcuop" * real-time priority(enabling/disabling) is controlled by * the extra CONFIG_RCU_NOCB_CPU_CB_BOOST configuration. */static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;module_param(kthread_prio, int, 0444); /* Delay in jiffies for grace-period initialization delays, debug only. */ static int gp_preinit_delay;module_param(gp_preinit_delay, int, 0444);static int gp_init_delay;module_param(gp_init_delay, int, 0444);static int gp_cleanup_delay;module_param(gp_cleanup_delay, int, 0444);static int nohz_full_patience_delay;module_param(nohz_full_patience_delay, int, 0444);static int nohz_full_patience_delay_jiffies; // Add delay to rcu_read_unlock() for strict grace periods.static int rcu_unlock_delay;#ifdef CONFIG_RCU_STRICT_GRACE_PERIODmodule_param(rcu_unlock_delay, int, 0444);#endif /* Retrieve RCU kthreads priority for rcutorture */int rcu_get_gp_kthreads_prio(void){ return kthread_prio;}EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio); /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */#define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays for debugging. */ /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(void){ return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));} /* * Return the number of callbacks queued on the specified CPU. * Handles both the nocbs and normal cases. */static long rcu_get_n_cbs_cpu(int cpu){ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (rcu_segcblist_is_enabled(&rdp->cblist)) return rcu_segcblist_n_cbs(&rdp->cblist); return 0;} /** * rcu_softirq_qs - Provide a set of RCU quiescent states in softirq processing * * Mark a quiescent state for RCU, Tasks RCU, and Tasks Trace RCU. * This is a special-purpose function to be used in the softirq * infrastructure and perhaps the occasional long-running softirq * handler. * * Note that from RCU's viewpoint, a call to rcu_softirq_qs() is * equivalent to momentarily completely enabling preemption. For * example, given this code:: * * local_bh_disable(); * do_something(); * rcu_softirq_qs(); // A * do_something_else(); * local_bh_enable(); // B * * A call to synchronize_rcu() that began concurrently with the * call to do_something() would be guaranteed to wait only until * execution reached statement A. Without that rcu_softirq_qs(), * that same synchronize_rcu() would instead be guaranteed to wait * until execution reached statement B. */void rcu_softirq_qs(void){ RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal rcu_softirq_qs() in RCU read-side critical section"); rcu_qs(); rcu_preempt_deferred_qs(current); rcu_tasks_qs(current, false);} /* * Reset the current CPU's RCU_WATCHING counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. * This will either leave the counter unchanged, or increment it * to the next non-quiescent value. * * The non-atomic test/increment sequence works because the upper bits * of the ->state variable are manipulated only by the corresponding CPU, * or when the corresponding CPU is offline. */static void rcu_watching_online(void){ if (ct_rcu_watching() & CT_RCU_WATCHING) return; ct_state_inc(CT_RCU_WATCHING);} /* * Return true if the snapshot returned from ct_rcu_watching() * indicates that RCU is in an extended quiescent state. */static bool rcu_watching_snap_in_eqs(int snap){ return !(snap & CT_RCU_WATCHING);} /** * rcu_watching_snap_stopped_since() - Has RCU stopped watching a given CPU * since the specified @snap? * * @rdp: The rcu_data corresponding to the CPU for which to check EQS. * @snap: rcu_watching snapshot taken when the CPU wasn't in an EQS. * * Returns true if the CPU corresponding to @rdp has spent some time in an * extended quiescent state since @snap. Note that this doesn't check if it * /still/ is in an EQS, just that it went through one since @snap. * * This is meant to be used in a loop waiting for a CPU to go through an EQS. */static bool rcu_watching_snap_stopped_since(struct rcu_data *rdp, int snap){ /* * The first failing snapshot is already ordered against the accesses * performed by the remote CPU after it exits idle. * * The second snapshot therefore only needs to order against accesses * performed by the remote CPU prior to entering idle and therefore can * rely solely on acquire semantics. */ if (WARN_ON_ONCE(rcu_watching_snap_in_eqs(snap))) return true; return snap != ct_rcu_watching_cpu_acquire(rdp->cpu);} /* * Return true if the referenced integer is zero while the specified * CPU remains within a single extended quiescent state. */bool rcu_watching_zero_in_eqs(int cpu, int *vp){ int snap; // If not quiescent, force back to earlier extended quiescent state. snap = ct_rcu_watching_cpu(cpu) & ~CT_RCU_WATCHING; smp_rmb(); // Order CT state and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; smp_rmb(); // Order *vp read and CT state re-read. // If still in the same extended quiescent state, we are good! return snap == ct_rcu_watching_cpu(cpu);} /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle period. * * The caller must have disabled interrupts and must not be idle. */notrace void rcu_momentary_eqs(void){ struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); rcu_preempt_deferred_qs(current);}EXPORT_SYMBOL_GPL(rcu_momentary_eqs); /** * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle * * If the current CPU is idle and running at a first-level (not nested) * interrupt, or directly, from idle, return true. * * The caller must have at least disabled IRQs. */static int rcu_is_cpu_rrupt_from_idle(void){ long nmi_nesting = ct_nmi_nesting(); /* * Usually called from the tick; but also used from smp_function_call() * for expedited grace periods. This latter can result in running from * the idle task, instead of an actual IPI. */ lockdep_assert_irqs_disabled(); /* Check for counter underflows */ RCU_LOCKDEP_WARN(ct_nesting() < 0, "RCU nesting counter underflow!"); /* Non-idle interrupt or nested idle interrupt */ if (nmi_nesting > 1) return false; /* * Non nested idle interrupt (interrupting section where RCU * wasn't watching). */ if (nmi_nesting == 1) return true; /* Not in an interrupt */ if (!nmi_nesting) { RCU_LOCKDEP_WARN(!in_task() || !is_idle_task(current), "RCU nmi_nesting counter not in idle task!"); return !rcu_is_watching_curr_cpu(); } RCU_LOCKDEP_WARN(1, "RCU nmi_nesting counter underflow/zero!"); return false;} #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) // Maximum callbacks per rcu_do_batch ...#define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.static long blimit = DEFAULT_RCU_BLIMIT;#define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.static long qhimark = DEFAULT_RCU_QHIMARK;#define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.static long qlowmark = DEFAULT_RCU_QLOMARK;#define DEFAULT_RCU_QOVLD_MULT 2#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.static long qovld_calc = -1; // No pre-initialization lock acquisitions! module_param(blimit, long, 0444);module_param(qhimark, long, 0444);module_param(qlowmark, long, 0444);module_param(qovld, long, 0444); static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;static ulong jiffies_till_next_fqs = ULONG_MAX;static bool rcu_kick_kthreads;static int rcu_divisor = 7;module_param(rcu_divisor, int, 0644); /* Force an exit from rcu_do_batch() after 3 milliseconds. */static long rcu_resched_ns = 3 * NSEC_PER_MSEC;module_param(rcu_resched_ns, long, 0644); /* * How long the grace period must be before we start recruiting * quiescent-state help from rcu_note_context_switch(). */static ulong jiffies_till_sched_qs = ULONG_MAX;module_param(jiffies_till_sched_qs, ulong, 0444);static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */ /* * Make sure that we give the grace-period kthread time to detect any * idle CPUs before taking active measures to force quiescent states. * However, don't go below 100 milliseconds, adjusted upwards for really * large systems. */static void adjust_jiffies_till_sched_qs(void){ unsigned long j; /* If jiffies_till_sched_qs was specified, respect the request. */ if (jiffies_till_sched_qs != ULONG_MAX) { WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs); return; } /* Otherwise, set to third fqs scan, but bound below on large system. */ j = READ_ONCE(jiffies_till_first_fqs) + 2 * READ_ONCE(jiffies_till_next_fqs); if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV) j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV; pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j); WRITE_ONCE(jiffies_to_sched_qs, j);} static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp){ ulong j; int ret = kstrtoul(val, 0, &j); if (!ret) { WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j); adjust_jiffies_till_sched_qs(); } return ret;} static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp){ ulong j; int ret = kstrtoul(val, 0, &j); if (!ret) { WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); adjust_jiffies_till_sched_qs(); } return ret;}Lines 1–500 of 5004. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.ibm.com> * * Based on the original work by Paul McKenney <paulmck@linux.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #define pr_fmt(fmt) "rcu: " fmt #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate_wait.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/nmi.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/export.h> #include <linux/completion.h> #include <linux/kmemleak.h> #include <linux/moduleparam.h> #include <linux/panic.h> #include <linux/panic_notifier.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include <linux/wait.h> #include <linux/kthread.h> #include <uapi/linux/sched/types.h> #include <linux/prefetch.h> #include <linux/delay.h> #include <linux/random.h> #include <linux/trace_events.h> #include <linux/suspend.h> #include <linux/ftrace.h> #include <linux/tick.h> #include <linux/sysrq.h> #include <linux/kprobes.h> #include <linux/gfp.h> #include <linux/oom.h> #include <linux/smpboot.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched/isolation.h> #include <linux/sched/clock.h> #include <linux/vmalloc.h> #include <linux/mm.h> #include <linux/kasan.h> #include <linux/context_tracking.h> #include "../time/tick-internal.h" #include "tree.h" #include "rcu.h" #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif #define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */ static void rcu_sr_normal_gp_cleanup_work(struct work_struct *); static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .gpwrap = true, }; int rcu_get_gpwrap_count(int cpu) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); return READ_ONCE(rdp->gpwrap_count); } EXPORT_SYMBOL_GPL(rcu_get_gpwrap_count); static struct rcu_state rcu_state = { .level = { &rcu_state.node[0] }, .gp_state = RCU_GP_IDLE, .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex), .barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock), .name = RCU_NAME, .abbr = RCU_ABBR, .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex), .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex), .ofl_lock = __ARCH_SPIN_LOCK_UNLOCKED, .srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work, rcu_sr_normal_gp_cleanup_work), .srs_cleanups_pending = ATOMIC_INIT(0), #ifdef CONFIG_RCU_NOCB_CPU .nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex), #endif }; /* Dump rcu_node combining tree at boot to verify correct setup. */ static bool dump_tree; module_param(dump_tree, bool, 0444); /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */ static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT); #ifndef CONFIG_PREEMPT_RT module_param(use_softirq, bool, 0444); #endif /* Control rcu_node-tree auto-balancing at boot time. */ static bool rcu_fanout_exact; module_param(rcu_fanout_exact, bool, 0444); /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */ static int rcu_fanout_leaf = RCU_FANOUT_LEAF; module_param(rcu_fanout_leaf, int, 0444); int rcu_num_lvls __read_mostly = RCU_NUM_LVLS; /* Number of rcu_nodes at specified level. */ int num_rcu_lvl[] = NUM_RCU_LVL_INIT; int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable is initialized to the value * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, * RCU can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_rcu() to a simple barrier(). When this variable * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required * to detect real grace periods. This variable is also used to suppress * boot-time false positives from lockdep-RCU error checking. Finally, it * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU * is fully initialized, including all of its kthreads having been spawned. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */ static int rcu_scheduler_fully_active __read_mostly; static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, unsigned long gps, unsigned long flags); static void invoke_rcu_core(void); static void rcu_report_exp_rdp(struct rcu_data *rdp); static void rcu_report_qs_rdp(struct rcu_data *rdp); static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp); static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); static bool rcu_rdp_cpu_online(struct rcu_data *rdp); static bool rcu_init_invoked(void); static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); /* * rcuc/rcub/rcuop kthread realtime priority. The "rcuop" * real-time priority(enabling/disabling) is controlled by * the extra CONFIG_RCU_NOCB_CPU_CB_BOOST configuration. */ static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; module_param(kthread_prio, int, 0444); /* Delay in jiffies for grace-period initialization delays, debug only. */ static int gp_preinit_delay; module_param(gp_preinit_delay, int, 0444); static int gp_init_delay; module_param(gp_init_delay, int, 0444); static int gp_cleanup_delay; module_param(gp_cleanup_delay, int, 0444); static int nohz_full_patience_delay; module_param(nohz_full_patience_delay, int, 0444); static int nohz_full_patience_delay_jiffies; // Add delay to rcu_read_unlock() for strict grace periods. static int rcu_unlock_delay; #ifdef CONFIG_RCU_STRICT_GRACE_PERIOD module_param(rcu_unlock_delay, int, 0444); #endif /* Retrieve RCU kthreads priority for rcutorture */ int rcu_get_gp_kthreads_prio(void) { return kthread_prio; } EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio); /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */ #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays for debugging. */ /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(void) { return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); } /* * Return the number of callbacks queued on the specified CPU. * Handles both the nocbs and normal cases. */ static long rcu_get_n_cbs_cpu(int cpu) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (rcu_segcblist_is_enabled(&rdp->cblist)) return rcu_segcblist_n_cbs(&rdp->cblist); return 0; } /** * rcu_softirq_qs - Provide a set of RCU quiescent states in softirq processing * * Mark a quiescent state for RCU, Tasks RCU, and Tasks Trace RCU. * This is a special-purpose function to be used in the softirq * infrastructure and perhaps the occasional long-running softirq * handler. * * Note that from RCU's viewpoint, a call to rcu_softirq_qs() is * equivalent to momentarily completely enabling preemption. For * example, given this code:: * * local_bh_disable(); * do_something(); * rcu_softirq_qs(); // A * do_something_else(); * local_bh_enable(); // B * * A call to synchronize_rcu() that began concurrently with the * call to do_something() would be guaranteed to wait only until * execution reached statement A. Without that rcu_softirq_qs(), * that same synchronize_rcu() would instead be guaranteed to wait * until execution reached statement B. */ void rcu_softirq_qs(void) { RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal rcu_softirq_qs() in RCU read-side critical section"); rcu_qs(); rcu_preempt_deferred_qs(current); rcu_tasks_qs(current, false); } /* * Reset the current CPU's RCU_WATCHING counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. * This will either leave the counter unchanged, or increment it * to the next non-quiescent value. * * The non-atomic test/increment sequence works because the upper bits * of the ->state variable are manipulated only by the corresponding CPU, * or when the corresponding CPU is offline. */ static void rcu_watching_online(void) { if (ct_rcu_watching() & CT_RCU_WATCHING) return; ct_state_inc(CT_RCU_WATCHING); } /* * Return true if the snapshot returned from ct_rcu_watching() * indicates that RCU is in an extended quiescent state. */ static bool rcu_watching_snap_in_eqs(int snap) { return !(snap & CT_RCU_WATCHING); } /** * rcu_watching_snap_stopped_since() - Has RCU stopped watching a given CPU * since the specified @snap? * * @rdp: The rcu_data corresponding to the CPU for which to check EQS. * @snap: rcu_watching snapshot taken when the CPU wasn't in an EQS. * * Returns true if the CPU corresponding to @rdp has spent some time in an * extended quiescent state since @snap. Note that this doesn't check if it * /still/ is in an EQS, just that it went through one since @snap. * * This is meant to be used in a loop waiting for a CPU to go through an EQS. */ static bool rcu_watching_snap_stopped_since(struct rcu_data *rdp, int snap) { /* * The first failing snapshot is already ordered against the accesses * performed by the remote CPU after it exits idle. * * The second snapshot therefore only needs to order against accesses * performed by the remote CPU prior to entering idle and therefore can * rely solely on acquire semantics. */ if (WARN_ON_ONCE(rcu_watching_snap_in_eqs(snap))) return true; return snap != ct_rcu_watching_cpu_acquire(rdp->cpu); } /* * Return true if the referenced integer is zero while the specified * CPU remains within a single extended quiescent state. */ bool rcu_watching_zero_in_eqs(int cpu, int *vp) { int snap; // If not quiescent, force back to earlier extended quiescent state. snap = ct_rcu_watching_cpu(cpu) & ~CT_RCU_WATCHING; smp_rmb(); // Order CT state and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; smp_rmb(); // Order *vp read and CT state re-read. // If still in the same extended quiescent state, we are good! return snap == ct_rcu_watching_cpu(cpu); } /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle period. * * The caller must have disabled interrupts and must not be idle. */ notrace void rcu_momentary_eqs(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); rcu_preempt_deferred_qs(current); } EXPORT_SYMBOL_GPL(rcu_momentary_eqs); /** * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle * * If the current CPU is idle and running at a first-level (not nested) * interrupt, or directly, from idle, return true. * * The caller must have at least disabled IRQs. */ static int rcu_is_cpu_rrupt_from_idle(void) { long nmi_nesting = ct_nmi_nesting(); /* * Usually called from the tick; but also used from smp_function_call() * for expedited grace periods. This latter can result in running from * the idle task, instead of an actual IPI. */ lockdep_assert_irqs_disabled(); /* Check for counter underflows */ RCU_LOCKDEP_WARN(ct_nesting() < 0, "RCU nesting counter underflow!"); /* Non-idle interrupt or nested idle interrupt */ if (nmi_nesting > 1) return false; /* * Non nested idle interrupt (interrupting section where RCU * wasn't watching). */ if (nmi_nesting == 1) return true; /* Not in an interrupt */ if (!nmi_nesting) { RCU_LOCKDEP_WARN(!in_task() || !is_idle_task(current), "RCU nmi_nesting counter not in idle task!"); return !rcu_is_watching_curr_cpu(); } RCU_LOCKDEP_WARN(1, "RCU nmi_nesting counter underflow/zero!"); return false; } #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) // Maximum callbacks per rcu_do_batch ... #define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood. static long blimit = DEFAULT_RCU_BLIMIT; #define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit. static long qhimark = DEFAULT_RCU_QHIMARK; #define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit. static long qlowmark = DEFAULT_RCU_QLOMARK; #define DEFAULT_RCU_QOVLD_MULT 2 #define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK) static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS. static long qovld_calc = -1; // No pre-initialization lock acquisitions! module_param(blimit, long, 0444); module_param(qhimark, long, 0444); module_param(qlowmark, long, 0444); module_param(qovld, long, 0444); static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX; static ulong jiffies_till_next_fqs = ULONG_MAX; static bool rcu_kick_kthreads; static int rcu_divisor = 7; module_param(rcu_divisor, int, 0644); /* Force an exit from rcu_do_batch() after 3 milliseconds. */ static long rcu_resched_ns = 3 * NSEC_PER_MSEC; module_param(rcu_resched_ns, long, 0644); /* * How long the grace period must be before we start recruiting * quiescent-state help from rcu_note_context_switch(). */ static ulong jiffies_till_sched_qs = ULONG_MAX; module_param(jiffies_till_sched_qs, ulong, 0444); static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */ module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */ /* * Make sure that we give the grace-period kthread time to detect any * idle CPUs before taking active measures to force quiescent states. * However, don't go below 100 milliseconds, adjusted upwards for really * large systems. */ static void adjust_jiffies_till_sched_qs(void) { unsigned long j; /* If jiffies_till_sched_qs was specified, respect the request. */ if (jiffies_till_sched_qs != ULONG_MAX) { WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs); return; } /* Otherwise, set to third fqs scan, but bound below on large system. */ j = READ_ONCE(jiffies_till_first_fqs) + 2 * READ_ONCE(jiffies_till_next_fqs); if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV) j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV; pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j); WRITE_ONCE(jiffies_to_sched_qs, j); } static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp) { ulong j; int ret = kstrtoul(val, 0, &j); if (!ret) { WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j); adjust_jiffies_till_sched_qs(); } return ret; } static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp) { ulong j; int ret = kstrtoul(val, 0, &j); if (!ret) { WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); adjust_jiffies_till_sched_qs(); } return ret; }