/* SPDX-License-Identifier: GPL-2.0+ *//* * Task-based RCU implementations. * * Copyright (C) 2020 Paul E. McKenney */ #ifdef CONFIG_TASKS_RCU_GENERIC#include "rcu_segcblist.h" //////////////////////////////////////////////////////////////////////////// Generic data structures. struct rcu_tasks;typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);typedef void (*pregp_func_t)(struct list_head *hop);typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);typedef void (*postscan_func_t)(struct list_head *hop);typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp);typedef void (*postgp_func_t)(struct rcu_tasks *rtp); /** * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism. * @cblist: Callback list. * @lock: Lock protecting per-CPU callback list. * @rtp_jiffies: Jiffies counter value for statistics. * @lazy_timer: Timer to unlazify callbacks. * @urgent_gp: Number of additional non-lazy grace periods. * @rtp_n_lock_retries: Rough lock-contention statistic. * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. * @barrier_q_head: RCU callback for barrier operation. * @rtp_blkd_tasks: List of tasks blocked as readers. * @rtp_exit_list: List of tasks in the latter portion of do_exit(). * @cpu: CPU number corresponding to this entry. * @index: Index of this CPU in rtpcp_array of the rcu_tasks structure. * @rtpp: Pointer to the rcu_tasks structure. */struct rcu_tasks_percpu { struct rcu_segcblist cblist; raw_spinlock_t __private lock; unsigned long rtp_jiffies; unsigned long rtp_n_lock_retries; struct timer_list lazy_timer; unsigned int urgent_gp; struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; struct list_head rtp_blkd_tasks; struct list_head rtp_exit_list; int cpu; int index; struct rcu_tasks *rtpp;}; /** * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism. * @cbs_wait: RCU wait allowing a new callback to get kthread's attention. * @cbs_gbl_lock: Lock protecting callback list. * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping. * @init_fract: Initial backoff sleep interval. * @gp_jiffies: Time of last @gp_state transition. * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). * @pertask_func: This flavor's per-task scan function (optional). * @postscan_func: This flavor's post-task scan function (optional). * @holdouts_func: This flavor's holdout-list scan function (optional). * @postgp_func: This flavor's post-grace-period function (optional). * @call_func: This flavor's call_rcu()-equivalent function. * @wait_state: Task state for synchronous grace-period waits (default TASK_UNINTERRUPTIBLE). * @rtpcpu: This flavor's rcu_tasks_percpu structure. * @rtpcp_array: Array of pointers to rcu_tasks_percpu structure of CPUs in cpu_possible_mask. * @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks. * @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing. * @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing. * @percpu_dequeue_gpseq: RCU grace-period number to propagate enqueue limit to dequeuers. * @barrier_q_mutex: Serialize barrier operations. * @barrier_q_count: Number of queues being waited on. * @barrier_q_completion: Barrier wait/wakeup mechanism. * @barrier_q_seq: Sequence number for barrier operations. * @barrier_q_start: Most recent barrier start in jiffies. * @name: This flavor's textual name. * @kname: This flavor's kthread name. */struct rcu_tasks { struct rcuwait cbs_wait; raw_spinlock_t cbs_gbl_lock; struct mutex tasks_gp_mutex; int gp_state; int gp_sleep; int init_fract; unsigned long gp_jiffies; unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; pregp_func_t pregp_func; pertask_func_t pertask_func; postscan_func_t postscan_func; holdouts_func_t holdouts_func; postgp_func_t postgp_func; call_rcu_func_t call_func; unsigned int wait_state; struct rcu_tasks_percpu __percpu *rtpcpu; struct rcu_tasks_percpu **rtpcp_array; int percpu_enqueue_shift; int percpu_enqueue_lim; int percpu_dequeue_lim; unsigned long percpu_dequeue_gpseq; struct mutex barrier_q_mutex; atomic_t barrier_q_count; struct completion barrier_q_completion; unsigned long barrier_q_seq; unsigned long barrier_q_start; char *name; char *kname;}; static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp); #define DEFINE_RCU_TASKS(rt_name, gp, call, n) \static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = { \ .lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock), \ .rtp_irq_work = IRQ_WORK_INIT_HARD(call_rcu_tasks_iw_wakeup), \}; \static struct rcu_tasks rt_name = \{ \ .cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait), \ .cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock), \ .tasks_gp_mutex = __MUTEX_INITIALIZER(rt_name.tasks_gp_mutex), \ .gp_func = gp, \ .call_func = call, \ .wait_state = TASK_UNINTERRUPTIBLE, \ .rtpcpu = &rt_name ## __percpu, \ .lazy_jiffies = DIV_ROUND_UP(HZ, 4), \ .name = n, \ .percpu_enqueue_shift = order_base_2(CONFIG_NR_CPUS), \ .percpu_enqueue_lim = 1, \ .percpu_dequeue_lim = 1, \ .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \ .barrier_q_seq = (0UL - 50UL) << RCU_SEQ_CTR_SHIFT, \ .kname = #rt_name, \} #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */static void tasks_rcu_exit_stall(struct timer_list *unused);static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall);#endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */#define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30)#define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10)static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;module_param(rcu_task_stall_timeout, int, 0644);#define RCU_TASK_STALL_INFO (HZ * 10)static int rcu_task_stall_info __read_mostly = RCU_TASK_STALL_INFO;module_param(rcu_task_stall_info, int, 0644);static int rcu_task_stall_info_mult __read_mostly = 3;module_param(rcu_task_stall_info_mult, int, 0444); static int rcu_task_enqueue_lim __read_mostly = -1;module_param(rcu_task_enqueue_lim, int, 0444); static bool rcu_task_cb_adjust;static int rcu_task_contend_lim __read_mostly = 100;module_param(rcu_task_contend_lim, int, 0444);static int rcu_task_collapse_lim __read_mostly = 10;module_param(rcu_task_collapse_lim, int, 0444);static int rcu_task_lazy_lim __read_mostly = 32;module_param(rcu_task_lazy_lim, int, 0444); static int rcu_task_cpu_ids; /* RCU tasks grace-period state for debugging. */#define RTGS_INIT 0#define RTGS_WAIT_WAIT_CBS 1#define RTGS_WAIT_GP 2#define RTGS_PRE_WAIT_GP 3#define RTGS_SCAN_TASKLIST 4#define RTGS_POST_SCAN_TASKLIST 5#define RTGS_WAIT_SCAN_HOLDOUTS 6#define RTGS_SCAN_HOLDOUTS 7#define RTGS_POST_GP 8#define RTGS_WAIT_READERS 9#define RTGS_INVOKE_CBS 10#define RTGS_WAIT_CBS 11#ifndef CONFIG_TINY_RCUstatic const char * const rcu_tasks_gp_state_names[] = { "RTGS_INIT", "RTGS_WAIT_WAIT_CBS", "RTGS_WAIT_GP", "RTGS_PRE_WAIT_GP", "RTGS_SCAN_TASKLIST", "RTGS_POST_SCAN_TASKLIST", "RTGS_WAIT_SCAN_HOLDOUTS", "RTGS_SCAN_HOLDOUTS", "RTGS_POST_GP", "RTGS_WAIT_READERS", "RTGS_INVOKE_CBS", "RTGS_WAIT_CBS",};#endif /* #ifndef CONFIG_TINY_RCU */ //////////////////////////////////////////////////////////////////////////// Generic code. static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp); /* Record grace-period phase and time. */static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate){ rtp->gp_state = newstate; rtp->gp_jiffies = jiffies;} #ifndef CONFIG_TINY_RCU/* Return state name. */static const char *tasks_gp_state_getname(struct rcu_tasks *rtp){ int i = data_race(rtp->gp_state); // Let KCSAN detect update races int j = READ_ONCE(i); // Prevent the compiler from reading twice if (j >= ARRAY_SIZE(rcu_tasks_gp_state_names)) return "???"; return rcu_tasks_gp_state_names[j];}#endif /* #ifndef CONFIG_TINY_RCU */ // Initialize per-CPU callback lists for the specified flavor of// Tasks RCU. Do not enqueue callbacks before this function is invoked.static void cblist_init_generic(struct rcu_tasks *rtp){ int cpu; int lim; int shift; int maxcpu; int index = 0; if (rcu_task_enqueue_lim < 0) { rcu_task_enqueue_lim = 1; rcu_task_cb_adjust = true; } else if (rcu_task_enqueue_lim == 0) { rcu_task_enqueue_lim = 1; } lim = rcu_task_enqueue_lim; rtp->rtpcp_array = kzalloc_objs(struct rcu_tasks_percpu *, num_possible_cpus()); BUG_ON(!rtp->rtpcp_array); for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); WARN_ON_ONCE(!rtpcp); if (cpu) raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock)); if (rcu_segcblist_empty(&rtpcp->cblist)) rcu_segcblist_init(&rtpcp->cblist); INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; rtpcp->index = index; rtp->rtpcp_array[index] = rtpcp; index++; if (!rtpcp->rtp_blkd_tasks.next) INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); if (!rtpcp->rtp_exit_list.next) INIT_LIST_HEAD(&rtpcp->rtp_exit_list); rtpcp->barrier_q_head.next = &rtpcp->barrier_q_head; maxcpu = cpu; } rcu_task_cpu_ids = maxcpu + 1; if (lim > rcu_task_cpu_ids) lim = rcu_task_cpu_ids; shift = ilog2(rcu_task_cpu_ids / lim); if (((rcu_task_cpu_ids - 1) >> shift) >= lim) shift++; rtp->percpu_enqueue_shift = shift; rtp->percpu_dequeue_lim = lim; rtp->percpu_enqueue_lim = lim; pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d rcu_task_cpu_ids=%d.\n", rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust, rcu_task_cpu_ids);} // Compute wakeup time for lazy callback timer.static unsigned long rcu_tasks_lazy_time(struct rcu_tasks *rtp){ return jiffies + rtp->lazy_jiffies;} // Timer handler that unlazifies lazy callbacks.static void call_rcu_tasks_generic_timer(struct timer_list *tlp){ unsigned long flags; bool needwake = false; struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp = timer_container_of(rtpcp, tlp, lazy_timer); rtp = rtpcp->rtpp; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) { if (!rtpcp->urgent_gp) rtpcp->urgent_gp = 1; needwake = true; mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); } raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (needwake) rcuwait_wake_up(&rtp->cbs_wait);} // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic().static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp){ struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work); rtp = rtpcp->rtpp; rcuwait_wake_up(&rtp->cbs_wait);} // Enqueue a callback for the specified flavor of Tasks RCU.static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, struct rcu_tasks *rtp){ int chosen_cpu; unsigned long flags; bool havekthread = smp_load_acquire(&rtp->kthread_ptr); int ideal_cpu; unsigned long j; bool needadjust = false; bool needwake; struct rcu_tasks_percpu *rtpcp; rhp->next = NULL; rhp->func = func; local_irq_save(flags); rcu_read_lock(); ideal_cpu = smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift); chosen_cpu = cpumask_next(ideal_cpu - 1, cpu_possible_mask); WARN_ON_ONCE(chosen_cpu >= rcu_task_cpu_ids); rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu); if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled. raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. j = jiffies; if (rtpcp->rtp_jiffies != j) { rtpcp->rtp_jiffies = j; rtpcp->rtp_n_lock_retries = 0; } if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim && READ_ONCE(rtp->percpu_enqueue_lim) != rcu_task_cpu_ids) needadjust = true; // Defer adjustment to avoid deadlock. } // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); else needwake = rcu_segcblist_empty(&rtpcp->cblist); } if (needwake) rtpcp->urgent_gp = 3; rcu_segcblist_enqueue(&rtpcp->cblist, rhp); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (unlikely(needadjust)) { raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rtp->percpu_enqueue_lim != rcu_task_cpu_ids) { WRITE_ONCE(rtp->percpu_enqueue_shift, 0); WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids); smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids); pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); // We can't create the kthread with interrupts disabled because a // scheduler spinlock might be held, so kthread creation is deferred // until core_initcall() time. Similarly, wakeups are deferred using // irq_work in order to avoid potential scheduler-lock-deadlock // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work);} // RCU callback function for rcu_barrier_tasks_generic().static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp){ struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp; rhp->next = rhp; // Mark the callback as having been invoked. rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head); rtp = rtpcp->rtpp; if (atomic_dec_and_test(&rtp->barrier_q_count)) complete(&rtp->barrier_q_completion);} // Wait for all in-flight callbacks for the specified RCU Tasks flavor.// Operates in a manner similar to rcu_barrier().static void __maybe_unused rcu_barrier_tasks_generic(struct rcu_tasks *rtp){ int cpu; unsigned long flags; struct rcu_tasks_percpu *rtpcp; unsigned long s = rcu_seq_snap(&rtp->barrier_q_seq); mutex_lock(&rtp->barrier_q_mutex); if (rcu_seq_done(&rtp->barrier_q_seq, s)) { smp_mb(); mutex_unlock(&rtp->barrier_q_mutex); return; } rtp->barrier_q_start = jiffies; rcu_seq_start(&rtp->barrier_q_seq); init_completion(&rtp->barrier_q_completion); atomic_set(&rtp->barrier_q_count, 2); for_each_possible_cpu(cpu) { if (cpu >= smp_load_acquire(&rtp->percpu_dequeue_lim)) break; rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head)) atomic_inc(&rtp->barrier_q_count); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } if (atomic_sub_and_test(2, &rtp->barrier_q_count)) complete(&rtp->barrier_q_completion); wait_for_completion(&rtp->barrier_q_completion); rcu_seq_end(&rtp->barrier_q_seq); mutex_unlock(&rtp->barrier_q_mutex);} // Advance callbacks and indicate whether either a grace period or// callback invocation is needed.static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp){ int cpu; int dequeue_limit; unsigned long flags; bool gpdone = poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq); long n; long ncbs = 0; long ncbsnz = 0; int needgpcb = 0; dequeue_limit = smp_load_acquire(&rtp->percpu_dequeue_lim); for (cpu = 0; cpu < dequeue_limit; cpu++) { if (!cpu_possible(cpu)) continue; struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); /* Advance and accelerate any new callbacks. */ if (!rcu_segcblist_n_cbs(&rtpcp->cblist)) continue; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); // Should we shrink down to a single callback queue? n = rcu_segcblist_n_cbs(&rtpcp->cblist); if (n) { ncbs += n; if (cpu > 0) ncbsnz += n; } srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { if (rtp->lazy_jiffies) rtpcp->urgent_gp--; needgpcb |= 0x3; } else if (rcu_segcblist_empty(&rtpcp->cblist)) { rtpcp->urgent_gp = 0; } if (rcu_segcblist_ready_cbs(&rtpcp->cblist)) needgpcb |= 0x1; raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } // Shrink down to a single callback queue if appropriate.Lines 1–500 of 1625. Load more
/* SPDX-License-Identifier: GPL-2.0+ */ /* * Task-based RCU implementations. * * Copyright (C) 2020 Paul E. McKenney */ #ifdef CONFIG_TASKS_RCU_GENERIC #include "rcu_segcblist.h" //////////////////////////////////////////////////////////////////////// // // Generic data structures. struct rcu_tasks; typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp); typedef void (*pregp_func_t)(struct list_head *hop); typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop); typedef void (*postscan_func_t)(struct list_head *hop); typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp); typedef void (*postgp_func_t)(struct rcu_tasks *rtp); /** * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism. * @cblist: Callback list. * @lock: Lock protecting per-CPU callback list. * @rtp_jiffies: Jiffies counter value for statistics. * @lazy_timer: Timer to unlazify callbacks. * @urgent_gp: Number of additional non-lazy grace periods. * @rtp_n_lock_retries: Rough lock-contention statistic. * @rtp_work: Work queue for invoking callbacks. * @rtp_irq_work: IRQ work queue for deferred wakeups. * @barrier_q_head: RCU callback for barrier operation. * @rtp_blkd_tasks: List of tasks blocked as readers. * @rtp_exit_list: List of tasks in the latter portion of do_exit(). * @cpu: CPU number corresponding to this entry. * @index: Index of this CPU in rtpcp_array of the rcu_tasks structure. * @rtpp: Pointer to the rcu_tasks structure. */ struct rcu_tasks_percpu { struct rcu_segcblist cblist; raw_spinlock_t __private lock; unsigned long rtp_jiffies; unsigned long rtp_n_lock_retries; struct timer_list lazy_timer; unsigned int urgent_gp; struct work_struct rtp_work; struct irq_work rtp_irq_work; struct rcu_head barrier_q_head; struct list_head rtp_blkd_tasks; struct list_head rtp_exit_list; int cpu; int index; struct rcu_tasks *rtpp; }; /** * struct rcu_tasks - Definition for a Tasks-RCU-like mechanism. * @cbs_wait: RCU wait allowing a new callback to get kthread's attention. * @cbs_gbl_lock: Lock protecting callback list. * @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone. * @gp_func: This flavor's grace-period-wait function. * @gp_state: Grace period's most recent state transition (debugging). * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping. * @init_fract: Initial backoff sleep interval. * @gp_jiffies: Time of last @gp_state transition. * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). * @pertask_func: This flavor's per-task scan function (optional). * @postscan_func: This flavor's post-task scan function (optional). * @holdouts_func: This flavor's holdout-list scan function (optional). * @postgp_func: This flavor's post-grace-period function (optional). * @call_func: This flavor's call_rcu()-equivalent function. * @wait_state: Task state for synchronous grace-period waits (default TASK_UNINTERRUPTIBLE). * @rtpcpu: This flavor's rcu_tasks_percpu structure. * @rtpcp_array: Array of pointers to rcu_tasks_percpu structure of CPUs in cpu_possible_mask. * @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks. * @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing. * @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing. * @percpu_dequeue_gpseq: RCU grace-period number to propagate enqueue limit to dequeuers. * @barrier_q_mutex: Serialize barrier operations. * @barrier_q_count: Number of queues being waited on. * @barrier_q_completion: Barrier wait/wakeup mechanism. * @barrier_q_seq: Sequence number for barrier operations. * @barrier_q_start: Most recent barrier start in jiffies. * @name: This flavor's textual name. * @kname: This flavor's kthread name. */ struct rcu_tasks { struct rcuwait cbs_wait; raw_spinlock_t cbs_gbl_lock; struct mutex tasks_gp_mutex; int gp_state; int gp_sleep; int init_fract; unsigned long gp_jiffies; unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; pregp_func_t pregp_func; pertask_func_t pertask_func; postscan_func_t postscan_func; holdouts_func_t holdouts_func; postgp_func_t postgp_func; call_rcu_func_t call_func; unsigned int wait_state; struct rcu_tasks_percpu __percpu *rtpcpu; struct rcu_tasks_percpu **rtpcp_array; int percpu_enqueue_shift; int percpu_enqueue_lim; int percpu_dequeue_lim; unsigned long percpu_dequeue_gpseq; struct mutex barrier_q_mutex; atomic_t barrier_q_count; struct completion barrier_q_completion; unsigned long barrier_q_seq; unsigned long barrier_q_start; char *name; char *kname; }; static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp); #define DEFINE_RCU_TASKS(rt_name, gp, call, n) \ static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = { \ .lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock), \ .rtp_irq_work = IRQ_WORK_INIT_HARD(call_rcu_tasks_iw_wakeup), \ }; \ static struct rcu_tasks rt_name = \ { \ .cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait), \ .cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock), \ .tasks_gp_mutex = __MUTEX_INITIALIZER(rt_name.tasks_gp_mutex), \ .gp_func = gp, \ .call_func = call, \ .wait_state = TASK_UNINTERRUPTIBLE, \ .rtpcpu = &rt_name ## __percpu, \ .lazy_jiffies = DIV_ROUND_UP(HZ, 4), \ .name = n, \ .percpu_enqueue_shift = order_base_2(CONFIG_NR_CPUS), \ .percpu_enqueue_lim = 1, \ .percpu_dequeue_lim = 1, \ .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \ .barrier_q_seq = (0UL - 50UL) << RCU_SEQ_CTR_SHIFT, \ .kname = #rt_name, \ } #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */ static void tasks_rcu_exit_stall(struct timer_list *unused); static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall); #endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ #define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30) #define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10) static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT; module_param(rcu_task_stall_timeout, int, 0644); #define RCU_TASK_STALL_INFO (HZ * 10) static int rcu_task_stall_info __read_mostly = RCU_TASK_STALL_INFO; module_param(rcu_task_stall_info, int, 0644); static int rcu_task_stall_info_mult __read_mostly = 3; module_param(rcu_task_stall_info_mult, int, 0444); static int rcu_task_enqueue_lim __read_mostly = -1; module_param(rcu_task_enqueue_lim, int, 0444); static bool rcu_task_cb_adjust; static int rcu_task_contend_lim __read_mostly = 100; module_param(rcu_task_contend_lim, int, 0444); static int rcu_task_collapse_lim __read_mostly = 10; module_param(rcu_task_collapse_lim, int, 0444); static int rcu_task_lazy_lim __read_mostly = 32; module_param(rcu_task_lazy_lim, int, 0444); static int rcu_task_cpu_ids; /* RCU tasks grace-period state for debugging. */ #define RTGS_INIT 0 #define RTGS_WAIT_WAIT_CBS 1 #define RTGS_WAIT_GP 2 #define RTGS_PRE_WAIT_GP 3 #define RTGS_SCAN_TASKLIST 4 #define RTGS_POST_SCAN_TASKLIST 5 #define RTGS_WAIT_SCAN_HOLDOUTS 6 #define RTGS_SCAN_HOLDOUTS 7 #define RTGS_POST_GP 8 #define RTGS_WAIT_READERS 9 #define RTGS_INVOKE_CBS 10 #define RTGS_WAIT_CBS 11 #ifndef CONFIG_TINY_RCU static const char * const rcu_tasks_gp_state_names[] = { "RTGS_INIT", "RTGS_WAIT_WAIT_CBS", "RTGS_WAIT_GP", "RTGS_PRE_WAIT_GP", "RTGS_SCAN_TASKLIST", "RTGS_POST_SCAN_TASKLIST", "RTGS_WAIT_SCAN_HOLDOUTS", "RTGS_SCAN_HOLDOUTS", "RTGS_POST_GP", "RTGS_WAIT_READERS", "RTGS_INVOKE_CBS", "RTGS_WAIT_CBS", }; #endif /* #ifndef CONFIG_TINY_RCU */ //////////////////////////////////////////////////////////////////////// // // Generic code. static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp); /* Record grace-period phase and time. */ static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate) { rtp->gp_state = newstate; rtp->gp_jiffies = jiffies; } #ifndef CONFIG_TINY_RCU /* Return state name. */ static const char *tasks_gp_state_getname(struct rcu_tasks *rtp) { int i = data_race(rtp->gp_state); // Let KCSAN detect update races int j = READ_ONCE(i); // Prevent the compiler from reading twice if (j >= ARRAY_SIZE(rcu_tasks_gp_state_names)) return "???"; return rcu_tasks_gp_state_names[j]; } #endif /* #ifndef CONFIG_TINY_RCU */ // Initialize per-CPU callback lists for the specified flavor of // Tasks RCU. Do not enqueue callbacks before this function is invoked. static void cblist_init_generic(struct rcu_tasks *rtp) { int cpu; int lim; int shift; int maxcpu; int index = 0; if (rcu_task_enqueue_lim < 0) { rcu_task_enqueue_lim = 1; rcu_task_cb_adjust = true; } else if (rcu_task_enqueue_lim == 0) { rcu_task_enqueue_lim = 1; } lim = rcu_task_enqueue_lim; rtp->rtpcp_array = kzalloc_objs(struct rcu_tasks_percpu *, num_possible_cpus()); BUG_ON(!rtp->rtpcp_array); for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); WARN_ON_ONCE(!rtpcp); if (cpu) raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock)); if (rcu_segcblist_empty(&rtpcp->cblist)) rcu_segcblist_init(&rtpcp->cblist); INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); rtpcp->cpu = cpu; rtpcp->rtpp = rtp; rtpcp->index = index; rtp->rtpcp_array[index] = rtpcp; index++; if (!rtpcp->rtp_blkd_tasks.next) INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks); if (!rtpcp->rtp_exit_list.next) INIT_LIST_HEAD(&rtpcp->rtp_exit_list); rtpcp->barrier_q_head.next = &rtpcp->barrier_q_head; maxcpu = cpu; } rcu_task_cpu_ids = maxcpu + 1; if (lim > rcu_task_cpu_ids) lim = rcu_task_cpu_ids; shift = ilog2(rcu_task_cpu_ids / lim); if (((rcu_task_cpu_ids - 1) >> shift) >= lim) shift++; rtp->percpu_enqueue_shift = shift; rtp->percpu_dequeue_lim = lim; rtp->percpu_enqueue_lim = lim; pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d rcu_task_cpu_ids=%d.\n", rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim), rcu_task_cb_adjust, rcu_task_cpu_ids); } // Compute wakeup time for lazy callback timer. static unsigned long rcu_tasks_lazy_time(struct rcu_tasks *rtp) { return jiffies + rtp->lazy_jiffies; } // Timer handler that unlazifies lazy callbacks. static void call_rcu_tasks_generic_timer(struct timer_list *tlp) { unsigned long flags; bool needwake = false; struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp = timer_container_of(rtpcp, tlp, lazy_timer); rtp = rtpcp->rtpp; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) { if (!rtpcp->urgent_gp) rtpcp->urgent_gp = 1; needwake = true; mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); } raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (needwake) rcuwait_wake_up(&rtp->cbs_wait); } // IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic(). static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp) { struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work); rtp = rtpcp->rtpp; rcuwait_wake_up(&rtp->cbs_wait); } // Enqueue a callback for the specified flavor of Tasks RCU. static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, struct rcu_tasks *rtp) { int chosen_cpu; unsigned long flags; bool havekthread = smp_load_acquire(&rtp->kthread_ptr); int ideal_cpu; unsigned long j; bool needadjust = false; bool needwake; struct rcu_tasks_percpu *rtpcp; rhp->next = NULL; rhp->func = func; local_irq_save(flags); rcu_read_lock(); ideal_cpu = smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift); chosen_cpu = cpumask_next(ideal_cpu - 1, cpu_possible_mask); WARN_ON_ONCE(chosen_cpu >= rcu_task_cpu_ids); rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu); if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled. raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. j = jiffies; if (rtpcp->rtp_jiffies != j) { rtpcp->rtp_jiffies = j; rtpcp->rtp_n_lock_retries = 0; } if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim && READ_ONCE(rtp->percpu_enqueue_lim) != rcu_task_cpu_ids) needadjust = true; // Defer adjustment to avoid deadlock. } // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp)); else needwake = rcu_segcblist_empty(&rtpcp->cblist); } if (needwake) rtpcp->urgent_gp = 3; rcu_segcblist_enqueue(&rtpcp->cblist, rhp); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); if (unlikely(needadjust)) { raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rtp->percpu_enqueue_lim != rcu_task_cpu_ids) { WRITE_ONCE(rtp->percpu_enqueue_shift, 0); WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids); smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids); pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); // We can't create the kthread with interrupts disabled because a // scheduler spinlock might be held, so kthread creation is deferred // until core_initcall() time. Similarly, wakeups are deferred using // irq_work in order to avoid potential scheduler-lock-deadlock // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work); } // RCU callback function for rcu_barrier_tasks_generic(). static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp) { struct rcu_tasks *rtp; struct rcu_tasks_percpu *rtpcp; rhp->next = rhp; // Mark the callback as having been invoked. rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head); rtp = rtpcp->rtpp; if (atomic_dec_and_test(&rtp->barrier_q_count)) complete(&rtp->barrier_q_completion); } // Wait for all in-flight callbacks for the specified RCU Tasks flavor. // Operates in a manner similar to rcu_barrier(). static void __maybe_unused rcu_barrier_tasks_generic(struct rcu_tasks *rtp) { int cpu; unsigned long flags; struct rcu_tasks_percpu *rtpcp; unsigned long s = rcu_seq_snap(&rtp->barrier_q_seq); mutex_lock(&rtp->barrier_q_mutex); if (rcu_seq_done(&rtp->barrier_q_seq, s)) { smp_mb(); mutex_unlock(&rtp->barrier_q_mutex); return; } rtp->barrier_q_start = jiffies; rcu_seq_start(&rtp->barrier_q_seq); init_completion(&rtp->barrier_q_completion); atomic_set(&rtp->barrier_q_count, 2); for_each_possible_cpu(cpu) { if (cpu >= smp_load_acquire(&rtp->percpu_dequeue_lim)) break; rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head)) atomic_inc(&rtp->barrier_q_count); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } if (atomic_sub_and_test(2, &rtp->barrier_q_count)) complete(&rtp->barrier_q_completion); wait_for_completion(&rtp->barrier_q_completion); rcu_seq_end(&rtp->barrier_q_seq); mutex_unlock(&rtp->barrier_q_mutex); } // Advance callbacks and indicate whether either a grace period or // callback invocation is needed. static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) { int cpu; int dequeue_limit; unsigned long flags; bool gpdone = poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq); long n; long ncbs = 0; long ncbsnz = 0; int needgpcb = 0; dequeue_limit = smp_load_acquire(&rtp->percpu_dequeue_lim); for (cpu = 0; cpu < dequeue_limit; cpu++) { if (!cpu_possible(cpu)) continue; struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); /* Advance and accelerate any new callbacks. */ if (!rcu_segcblist_n_cbs(&rtpcp->cblist)) continue; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); // Should we shrink down to a single callback queue? n = rcu_segcblist_n_cbs(&rtpcp->cblist); if (n) { ncbs += n; if (cpu > 0) ncbsnz += n; } srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { if (rtp->lazy_jiffies) rtpcp->urgent_gp--; needgpcb |= 0x3; } else if (rcu_segcblist_empty(&rtpcp->cblist)) { rtpcp->urgent_gp = 0; } if (rcu_segcblist_ready_cbs(&rtpcp->cblist)) needgpcb |= 0x1; raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } // Shrink down to a single callback queue if appropriate.