// SPDX-License-Identifier: GPL-2.0+/* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.ibm.com> * * Based on the original work by Paul McKenney <paulmck@linux.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #define pr_fmt(fmt) "rcu: " fmt #include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate_wait.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/sched/debug.h>#include <linux/nmi.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/export.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/cpu.h>#include <linux/mutex.h>#include <linux/time.h>#include <linux/kernel_stat.h>#include <linux/wait.h>#include <linux/kthread.h>#include <uapi/linux/sched/types.h>#include <linux/prefetch.h>#include <linux/delay.h>#include <linux/random.h>#include <linux/trace_events.h>#include <linux/suspend.h>#include <linux/ftrace.h>#include <linux/tick.h>#include <linux/sysrq.h>#include <linux/kprobes.h>#include <linux/gfp.h>#include <linux/oom.h>#include <linux/smpboot.h>#include <linux/jiffies.h>#include <linux/slab.h>#include <linux/sched/isolation.h>#include <linux/sched/clock.h>#include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/kasan.h>#include "../time/tick-internal.h" #include "tree.h"#include "rcu.h" #ifdef MODULE_PARAM_PREFIX#undef MODULE_PARAM_PREFIX#endif#define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */ /* * Steal a bit from the bottom of ->dynticks for idle entry/exit * control. Initially this is for TLB flushing. */#define RCU_DYNTICK_CTRL_MASK 0x1#define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1) static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),#ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY,#endif};static struct rcu_state rcu_state = { .level = { &rcu_state.node[0] }, .gp_state = RCU_GP_IDLE, .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex), .name = RCU_NAME, .abbr = RCU_ABBR, .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex), .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex), .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),}; /* Dump rcu_node combining tree at boot to verify correct setup. */static bool dump_tree;module_param(dump_tree, bool, 0444);/* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT);#ifndef CONFIG_PREEMPT_RTmodule_param(use_softirq, bool, 0444);#endif/* Control rcu_node-tree auto-balancing at boot time. */static bool rcu_fanout_exact;module_param(rcu_fanout_exact, bool, 0444);/* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */static int rcu_fanout_leaf = RCU_FANOUT_LEAF;module_param(rcu_fanout_leaf, int, 0444);int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;/* Number of rcu_nodes at specified level. */int num_rcu_lvl[] = NUM_RCU_LVL_INIT;int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable is initialized to the value * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, * RCU can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_rcu() to a simple barrier(). When this variable * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required * to detect real grace periods. This variable is also used to suppress * boot-time false positives from lockdep-RCU error checking. Finally, it * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU * is fully initialized, including all of its kthreads having been spawned. */int rcu_scheduler_active __read_mostly;EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */static int rcu_scheduler_fully_active __read_mostly; static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, unsigned long gps, unsigned long flags);static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);static void invoke_rcu_core(void);static void rcu_report_exp_rdp(struct rcu_data *rdp);static void sync_sched_exp_online_cleanup(int cpu);static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); /* rcuc/rcub kthread realtime priority */static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;module_param(kthread_prio, int, 0444); /* Delay in jiffies for grace-period initialization delays, debug only. */ static int gp_preinit_delay;module_param(gp_preinit_delay, int, 0444);static int gp_init_delay;module_param(gp_init_delay, int, 0444);static int gp_cleanup_delay;module_param(gp_cleanup_delay, int, 0444); // Add delay to rcu_read_unlock() for strict grace periods.static int rcu_unlock_delay;#ifdef CONFIG_RCU_STRICT_GRACE_PERIODmodule_param(rcu_unlock_delay, int, 0444);#endif /* * This rcu parameter is runtime-read-only. It reflects * a minimum allowed number of objects which can be cached * per-CPU. Object size is equal to one page. This value * can be changed at boot time. */static int rcu_min_cached_objs = 5;module_param(rcu_min_cached_objs, int, 0444); /* Retrieve RCU kthreads priority for rcutorture */int rcu_get_gp_kthreads_prio(void){ return kthread_prio;}EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio); /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */#define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */ /* * Compute the mask of online CPUs for the specified rcu_node structure. * This will not be stable unless the rcu_node structure's ->lock is * held, but the bit corresponding to the current CPU will be stable * in most contexts. */static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp){ return READ_ONCE(rnp->qsmaskinitnext);} /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */static int rcu_gp_in_progress(void){ return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));} /* * Return the number of callbacks queued on the specified CPU. * Handles both the nocbs and normal cases. */static long rcu_get_n_cbs_cpu(int cpu){ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (rcu_segcblist_is_enabled(&rdp->cblist)) return rcu_segcblist_n_cbs(&rdp->cblist); return 0;} void rcu_softirq_qs(void){ rcu_qs(); rcu_preempt_deferred_qs(current);} /* * Record entry into an extended quiescent state. This is only to be * called when not already in an extended quiescent state, that is, * RCU is watching prior to the call to this function and is no longer * watching upon return. */static noinstr void rcu_dynticks_eqs_enter(void){ struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; /* * CPUs seeing atomic_add_return() must see prior RCU read-side * critical sections, and we also must force ordering with the * next idle sojourn. */ rcu_dynticks_task_trace_enter(); // Before ->dynticks update! seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); // RCU is no longer watching. Better be in extended quiescent state! WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICK_CTRL_CTR)); /* Better not have special action (TLB flush) pending! */ WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICK_CTRL_MASK));} /* * Record exit from an extended quiescent state. This is only to be * called from an extended quiescent state, that is, RCU is not watching * prior to the call to this function and is watching upon return. */static noinstr void rcu_dynticks_eqs_exit(void){ struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; /* * CPUs seeing atomic_add_return() must see prior idle sojourns, * and we also must force ordering with the next RCU read-side * critical section. */ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); // RCU is now watching. Better not be in an extended quiescent state! rcu_dynticks_task_trace_exit(); // After ->dynticks update! WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & RCU_DYNTICK_CTRL_CTR)); if (seq & RCU_DYNTICK_CTRL_MASK) { arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); smp_mb__after_atomic(); /* _exit after clearing mask. */ }} /* * Reset the current CPU's ->dynticks counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. * This will either leave the counter unchanged, or increment it * to the next non-quiescent value. * * The non-atomic test/increment sequence works because the upper bits * of the ->dynticks counter are manipulated only by the corresponding CPU, * or when the corresponding CPU is offline. */static void rcu_dynticks_eqs_online(void){ struct rcu_data *rdp = this_cpu_ptr(&rcu_data); if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR) return; atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);} /* * Is the current CPU in an extended quiescent state? * * No ordering, as we are sampling CPU-local information. */static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void){ struct rcu_data *rdp = this_cpu_ptr(&rcu_data); return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);} /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. */static int rcu_dynticks_snap(struct rcu_data *rdp){ int snap = atomic_add_return(0, &rdp->dynticks); return snap & ~RCU_DYNTICK_CTRL_MASK;} /* * Return true if the snapshot returned from rcu_dynticks_snap() * indicates that RCU is in an extended quiescent state. */static bool rcu_dynticks_in_eqs(int snap){ return !(snap & RCU_DYNTICK_CTRL_CTR);} /* Return true if the specified CPU is currently idle from an RCU viewpoint. */bool rcu_is_idle_cpu(int cpu){ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); return rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp));} /* * Return true if the CPU corresponding to the specified rcu_data * structure has spent some time in an extended quiescent state since * rcu_dynticks_snap() returned the specified snapshot. */static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap){ return snap != rcu_dynticks_snap(rdp);} /* * Return true if the referenced integer is zero while the specified * CPU remains within a single extended quiescent state. */bool rcu_dynticks_zero_in_eqs(int cpu, int *vp){ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int snap; // If not quiescent, force back to earlier extended quiescent state. snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK | RCU_DYNTICK_CTRL_CTR); smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; smp_rmb(); // Order *vp read and ->dynticks re-read. // If still in the same extended quiescent state, we are good! return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK);} /* * Set the special (bottom) bit of the specified CPU so that it * will take special action (such as flushing its TLB) on the * next exit from an extended quiescent state. Returns true if * the bit was successfully set, or false if the CPU was not in * an extended quiescent state. */bool rcu_eqs_special_set(int cpu){ int old; int new; int new_old; struct rcu_data *rdp = &per_cpu(rcu_data, cpu); new_old = atomic_read(&rdp->dynticks); do { old = new_old; if (old & RCU_DYNTICK_CTRL_CTR) return false; new = old | RCU_DYNTICK_CTRL_MASK; new_old = atomic_cmpxchg(&rdp->dynticks, old, new); } while (new_old != old); return true;} /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle period. * * The caller must have disabled interrupts and must not be idle. */notrace void rcu_momentary_dyntick_idle(void){ int special; raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &this_cpu_ptr(&rcu_data)->dynticks); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR)); rcu_preempt_deferred_qs(current);}EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle); /** * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle * * If the current CPU is idle and running at a first-level (not nested) * interrupt, or directly, from idle, return true. * * The caller must have at least disabled IRQs. */static int rcu_is_cpu_rrupt_from_idle(void){ long nesting; /* * Usually called from the tick; but also used from smp_function_call() * for expedited grace periods. This latter can result in running from * the idle task, instead of an actual IPI. */ lockdep_assert_irqs_disabled(); /* Check for counter underflows */ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0, "RCU dynticks_nesting counter underflow!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); /* Are we at first interrupt nesting level? */ nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting); if (nesting > 1) return false; /* * If we're not in an interrupt, we must be in the idle task! */ WARN_ON_ONCE(!nesting && !is_idle_task(current)); /* Does CPU appear to be idle from an RCU standpoint? */ return __this_cpu_read(rcu_data.dynticks_nesting) == 0;} #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) // Maximum callbacks per rcu_do_batch ...#define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood.static long blimit = DEFAULT_RCU_BLIMIT;#define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit.static long qhimark = DEFAULT_RCU_QHIMARK;#define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit.static long qlowmark = DEFAULT_RCU_QLOMARK;#define DEFAULT_RCU_QOVLD_MULT 2#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS.static long qovld_calc = -1; // No pre-initialization lock acquisitions! module_param(blimit, long, 0444);module_param(qhimark, long, 0444);module_param(qlowmark, long, 0444);module_param(qovld, long, 0444); static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX;static ulong jiffies_till_next_fqs = ULONG_MAX;static bool rcu_kick_kthreads;static int rcu_divisor = 7;module_param(rcu_divisor, int, 0644);Lines 1–500 of 4750. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * * Copyright IBM Corporation, 2008 * * Authors: Dipankar Sarma <dipankar@in.ibm.com> * Manfred Spraul <manfred@colorfullife.com> * Paul E. McKenney <paulmck@linux.ibm.com> * * Based on the original work by Paul McKenney <paulmck@linux.ibm.com> * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. * * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU */ #define pr_fmt(fmt) "rcu: " fmt #include <linux/types.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> #include <linux/rcupdate_wait.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/nmi.h> #include <linux/atomic.h> #include <linux/bitops.h> #include <linux/export.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/time.h> #include <linux/kernel_stat.h> #include <linux/wait.h> #include <linux/kthread.h> #include <uapi/linux/sched/types.h> #include <linux/prefetch.h> #include <linux/delay.h> #include <linux/random.h> #include <linux/trace_events.h> #include <linux/suspend.h> #include <linux/ftrace.h> #include <linux/tick.h> #include <linux/sysrq.h> #include <linux/kprobes.h> #include <linux/gfp.h> #include <linux/oom.h> #include <linux/smpboot.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched/isolation.h> #include <linux/sched/clock.h> #include <linux/vmalloc.h> #include <linux/mm.h> #include <linux/kasan.h> #include "../time/tick-internal.h" #include "tree.h" #include "rcu.h" #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif #define MODULE_PARAM_PREFIX "rcutree." /* Data structures. */ /* * Steal a bit from the bottom of ->dynticks for idle entry/exit * control. Initially this is for TLB flushing. */ #define RCU_DYNTICK_CTRL_MASK 0x1 #define RCU_DYNTICK_CTRL_CTR (RCU_DYNTICK_CTRL_MASK + 1) static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR), #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY, #endif }; static struct rcu_state rcu_state = { .level = { &rcu_state.node[0] }, .gp_state = RCU_GP_IDLE, .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT, .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex), .name = RCU_NAME, .abbr = RCU_ABBR, .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex), .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex), .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock), }; /* Dump rcu_node combining tree at boot to verify correct setup. */ static bool dump_tree; module_param(dump_tree, bool, 0444); /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */ static bool use_softirq = !IS_ENABLED(CONFIG_PREEMPT_RT); #ifndef CONFIG_PREEMPT_RT module_param(use_softirq, bool, 0444); #endif /* Control rcu_node-tree auto-balancing at boot time. */ static bool rcu_fanout_exact; module_param(rcu_fanout_exact, bool, 0444); /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */ static int rcu_fanout_leaf = RCU_FANOUT_LEAF; module_param(rcu_fanout_leaf, int, 0444); int rcu_num_lvls __read_mostly = RCU_NUM_LVLS; /* Number of rcu_nodes at specified level. */ int num_rcu_lvl[] = NUM_RCU_LVL_INIT; int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */ /* * The rcu_scheduler_active variable is initialized to the value * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the * first task is spawned. So when this variable is RCU_SCHEDULER_INACTIVE, * RCU can assume that there is but one task, allowing RCU to (for example) * optimize synchronize_rcu() to a simple barrier(). When this variable * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required * to detect real grace periods. This variable is also used to suppress * boot-time false positives from lockdep-RCU error checking. Finally, it * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU * is fully initialized, including all of its kthreads having been spawned. */ int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); /* * The rcu_scheduler_fully_active variable transitions from zero to one * during the early_initcall() processing, which is after the scheduler * is capable of creating new tasks. So RCU processing (for example, * creating tasks for RCU priority boosting) must be delayed until after * rcu_scheduler_fully_active transitions from zero to one. We also * currently delay invocation of any RCU callbacks until after this point. * * It might later prove better for people registering RCU callbacks during * early boot to take responsibility for these callbacks, but one step at * a time. */ static int rcu_scheduler_fully_active __read_mostly; static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, unsigned long gps, unsigned long flags); static void rcu_init_new_rnp(struct rcu_node *rnp_leaf); static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf); static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void rcu_report_exp_rdp(struct rcu_data *rdp); static void sync_sched_exp_online_cleanup(int cpu); static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp); static bool rcu_rdp_is_offloaded(struct rcu_data *rdp); /* rcuc/rcub kthread realtime priority */ static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0; module_param(kthread_prio, int, 0444); /* Delay in jiffies for grace-period initialization delays, debug only. */ static int gp_preinit_delay; module_param(gp_preinit_delay, int, 0444); static int gp_init_delay; module_param(gp_init_delay, int, 0444); static int gp_cleanup_delay; module_param(gp_cleanup_delay, int, 0444); // Add delay to rcu_read_unlock() for strict grace periods. static int rcu_unlock_delay; #ifdef CONFIG_RCU_STRICT_GRACE_PERIOD module_param(rcu_unlock_delay, int, 0444); #endif /* * This rcu parameter is runtime-read-only. It reflects * a minimum allowed number of objects which can be cached * per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ static int rcu_min_cached_objs = 5; module_param(rcu_min_cached_objs, int, 0444); /* Retrieve RCU kthreads priority for rcutorture */ int rcu_get_gp_kthreads_prio(void) { return kthread_prio; } EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio); /* * Number of grace periods between delays, normalized by the duration of * the delay. The longer the delay, the more the grace periods between * each delay. The reason for this normalization is that it means that, * for non-zero delays, the overall slowdown of grace periods is constant * regardless of the duration of the delay. This arrangement balances * the need for long delays to increase some race probabilities with the * need for fast grace periods to increase other race probabilities. */ #define PER_RCU_NODE_PERIOD 3 /* Number of grace periods between delays. */ /* * Compute the mask of online CPUs for the specified rcu_node structure. * This will not be stable unless the rcu_node structure's ->lock is * held, but the bit corresponding to the current CPU will be stable * in most contexts. */ static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp) { return READ_ONCE(rnp->qsmaskinitnext); } /* * Return true if an RCU grace period is in progress. The READ_ONCE()s * permit this function to be invoked without holding the root rcu_node * structure's ->lock, but of course results can be subject to change. */ static int rcu_gp_in_progress(void) { return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq)); } /* * Return the number of callbacks queued on the specified CPU. * Handles both the nocbs and normal cases. */ static long rcu_get_n_cbs_cpu(int cpu) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (rcu_segcblist_is_enabled(&rdp->cblist)) return rcu_segcblist_n_cbs(&rdp->cblist); return 0; } void rcu_softirq_qs(void) { rcu_qs(); rcu_preempt_deferred_qs(current); } /* * Record entry into an extended quiescent state. This is only to be * called when not already in an extended quiescent state, that is, * RCU is watching prior to the call to this function and is no longer * watching upon return. */ static noinstr void rcu_dynticks_eqs_enter(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; /* * CPUs seeing atomic_add_return() must see prior RCU read-side * critical sections, and we also must force ordering with the * next idle sojourn. */ rcu_dynticks_task_trace_enter(); // Before ->dynticks update! seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); // RCU is no longer watching. Better be in extended quiescent state! WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICK_CTRL_CTR)); /* Better not have special action (TLB flush) pending! */ WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && (seq & RCU_DYNTICK_CTRL_MASK)); } /* * Record exit from an extended quiescent state. This is only to be * called from an extended quiescent state, that is, RCU is not watching * prior to the call to this function and is watching upon return. */ static noinstr void rcu_dynticks_eqs_exit(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; /* * CPUs seeing atomic_add_return() must see prior idle sojourns, * and we also must force ordering with the next RCU read-side * critical section. */ seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); // RCU is now watching. Better not be in an extended quiescent state! rcu_dynticks_task_trace_exit(); // After ->dynticks update! WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(seq & RCU_DYNTICK_CTRL_CTR)); if (seq & RCU_DYNTICK_CTRL_MASK) { arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); smp_mb__after_atomic(); /* _exit after clearing mask. */ } } /* * Reset the current CPU's ->dynticks counter to indicate that the * newly onlined CPU is no longer in an extended quiescent state. * This will either leave the counter unchanged, or increment it * to the next non-quiescent value. * * The non-atomic test/increment sequence works because the upper bits * of the ->dynticks counter are manipulated only by the corresponding CPU, * or when the corresponding CPU is offline. */ static void rcu_dynticks_eqs_online(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR) return; atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); } /* * Is the current CPU in an extended quiescent state? * * No ordering, as we are sampling CPU-local information. */ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR); } /* * Snapshot the ->dynticks counter with full ordering so as to allow * stable comparison of this counter with past and future snapshots. */ static int rcu_dynticks_snap(struct rcu_data *rdp) { int snap = atomic_add_return(0, &rdp->dynticks); return snap & ~RCU_DYNTICK_CTRL_MASK; } /* * Return true if the snapshot returned from rcu_dynticks_snap() * indicates that RCU is in an extended quiescent state. */ static bool rcu_dynticks_in_eqs(int snap) { return !(snap & RCU_DYNTICK_CTRL_CTR); } /* Return true if the specified CPU is currently idle from an RCU viewpoint. */ bool rcu_is_idle_cpu(int cpu) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); return rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)); } /* * Return true if the CPU corresponding to the specified rcu_data * structure has spent some time in an extended quiescent state since * rcu_dynticks_snap() returned the specified snapshot. */ static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap) { return snap != rcu_dynticks_snap(rdp); } /* * Return true if the referenced integer is zero while the specified * CPU remains within a single extended quiescent state. */ bool rcu_dynticks_zero_in_eqs(int cpu, int *vp) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); int snap; // If not quiescent, force back to earlier extended quiescent state. snap = atomic_read(&rdp->dynticks) & ~(RCU_DYNTICK_CTRL_MASK | RCU_DYNTICK_CTRL_CTR); smp_rmb(); // Order ->dynticks and *vp reads. if (READ_ONCE(*vp)) return false; // Non-zero, so report failure; smp_rmb(); // Order *vp read and ->dynticks re-read. // If still in the same extended quiescent state, we are good! return snap == (atomic_read(&rdp->dynticks) & ~RCU_DYNTICK_CTRL_MASK); } /* * Set the special (bottom) bit of the specified CPU so that it * will take special action (such as flushing its TLB) on the * next exit from an extended quiescent state. Returns true if * the bit was successfully set, or false if the CPU was not in * an extended quiescent state. */ bool rcu_eqs_special_set(int cpu) { int old; int new; int new_old; struct rcu_data *rdp = &per_cpu(rcu_data, cpu); new_old = atomic_read(&rdp->dynticks); do { old = new_old; if (old & RCU_DYNTICK_CTRL_CTR) return false; new = old | RCU_DYNTICK_CTRL_MASK; new_old = atomic_cmpxchg(&rdp->dynticks, old, new); } while (new_old != old); return true; } /* * Let the RCU core know that this CPU has gone through the scheduler, * which is a quiescent state. This is called when the need for a * quiescent state is urgent, so we burn an atomic operation and full * memory barriers to let the RCU core know about it, regardless of what * this CPU might (or might not) do in the near future. * * We inform the RCU core by emulating a zero-duration dyntick-idle period. * * The caller must have disabled interrupts and must not be idle. */ notrace void rcu_momentary_dyntick_idle(void) { int special; raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR, &this_cpu_ptr(&rcu_data)->dynticks); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR)); rcu_preempt_deferred_qs(current); } EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle); /** * rcu_is_cpu_rrupt_from_idle - see if 'interrupted' from idle * * If the current CPU is idle and running at a first-level (not nested) * interrupt, or directly, from idle, return true. * * The caller must have at least disabled IRQs. */ static int rcu_is_cpu_rrupt_from_idle(void) { long nesting; /* * Usually called from the tick; but also used from smp_function_call() * for expedited grace periods. This latter can result in running from * the idle task, instead of an actual IPI. */ lockdep_assert_irqs_disabled(); /* Check for counter underflows */ RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0, "RCU dynticks_nesting counter underflow!"); RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0, "RCU dynticks_nmi_nesting counter underflow/zero!"); /* Are we at first interrupt nesting level? */ nesting = __this_cpu_read(rcu_data.dynticks_nmi_nesting); if (nesting > 1) return false; /* * If we're not in an interrupt, we must be in the idle task! */ WARN_ON_ONCE(!nesting && !is_idle_task(current)); /* Does CPU appear to be idle from an RCU standpoint? */ return __this_cpu_read(rcu_data.dynticks_nesting) == 0; } #define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10) // Maximum callbacks per rcu_do_batch ... #define DEFAULT_MAX_RCU_BLIMIT 10000 // ... even during callback flood. static long blimit = DEFAULT_RCU_BLIMIT; #define DEFAULT_RCU_QHIMARK 10000 // If this many pending, ignore blimit. static long qhimark = DEFAULT_RCU_QHIMARK; #define DEFAULT_RCU_QLOMARK 100 // Once only this many pending, use blimit. static long qlowmark = DEFAULT_RCU_QLOMARK; #define DEFAULT_RCU_QOVLD_MULT 2 #define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK) static long qovld = DEFAULT_RCU_QOVLD; // If this many pending, hammer QS. static long qovld_calc = -1; // No pre-initialization lock acquisitions! module_param(blimit, long, 0444); module_param(qhimark, long, 0444); module_param(qlowmark, long, 0444); module_param(qovld, long, 0444); static ulong jiffies_till_first_fqs = IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 0 : ULONG_MAX; static ulong jiffies_till_next_fqs = ULONG_MAX; static bool rcu_kick_kthreads; static int rcu_divisor = 7; module_param(rcu_divisor, int, 0644);