// SPDX-License-Identifier: GPL-2.0+/* * RCU CPU stall warnings for normal RCU grace periods * * Copyright IBM Corporation, 2019 * * Author: Paul E. McKenney <paulmck@linux.ibm.com> */ #include <linux/console.h>#include <linux/kvm_para.h>#include <linux/rcu_notifier.h>#include <linux/smp.h> ////////////////////////////////////////////////////////////////////////////////// Controlling CPU stall warnings, including delay calculation. /* panic() on RCU Stall sysctl. */static int sysctl_panic_on_rcu_stall __read_mostly;static int sysctl_max_rcu_stall_to_panic __read_mostly; static const struct ctl_table rcu_stall_sysctl_table[] = { { .procname = "panic_on_rcu_stall", .data = &sysctl_panic_on_rcu_stall, .maxlen = sizeof(sysctl_panic_on_rcu_stall), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "max_rcu_stall_to_panic", .data = &sysctl_max_rcu_stall_to_panic, .maxlen = sizeof(sysctl_max_rcu_stall_to_panic), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, },}; static int __init init_rcu_stall_sysctl(void){ register_sysctl_init("kernel", rcu_stall_sysctl_table); return 0;} subsys_initcall(init_rcu_stall_sysctl); #ifdef CONFIG_SYSFS static unsigned int rcu_stall_count; static ssize_t rcu_stall_count_show(struct kobject *kobj, struct kobj_attribute *attr, char *page){ return sysfs_emit(page, "%u\n", rcu_stall_count);} static struct kobj_attribute rcu_stall_count_attr = __ATTR_RO(rcu_stall_count); static __init int kernel_rcu_stall_sysfs_init(void){ sysfs_add_file_to_group(kernel_kobj, &rcu_stall_count_attr.attr, NULL); return 0;} late_initcall(kernel_rcu_stall_sysfs_init); #endif // CONFIG_SYSFS #ifdef CONFIG_PROVE_RCU#define RCU_STALL_DELAY_DELTA (5 * HZ)#else#define RCU_STALL_DELAY_DELTA 0#endif#define RCU_STALL_MIGHT_DIV 8#define RCU_STALL_MIGHT_MIN (2 * HZ) int rcu_exp_jiffies_till_stall_check(void){ int cpu_stall_timeout = READ_ONCE(rcu_exp_cpu_stall_timeout); int exp_stall_delay_delta = 0; int till_stall_check; // Zero says to use rcu_cpu_stall_timeout, but in milliseconds. if (!cpu_stall_timeout) cpu_stall_timeout = jiffies_to_msecs(rcu_jiffies_till_stall_check()); // Limit check must be consistent with the Kconfig limits for // CONFIG_RCU_EXP_CPU_STALL_TIMEOUT, so check the allowed range. // The minimum clamped value is "2UL", because at least one full // tick has to be guaranteed. till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ); if (cpu_stall_timeout && jiffies_to_msecs(till_stall_check) != cpu_stall_timeout) WRITE_ONCE(rcu_exp_cpu_stall_timeout, jiffies_to_msecs(till_stall_check)); #ifdef CONFIG_PROVE_RCU /* Add extra ~25% out of till_stall_check. */ exp_stall_delay_delta = ((till_stall_check * 25) / 100) + 1;#endif return till_stall_check + exp_stall_delay_delta;}EXPORT_SYMBOL_GPL(rcu_exp_jiffies_till_stall_check); /* Limit-check stall timeouts specified at boottime and runtime. */int rcu_jiffies_till_stall_check(void){ int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout); /* * Limit check must be consistent with the Kconfig limits * for CONFIG_RCU_CPU_STALL_TIMEOUT. */ if (till_stall_check < 3) { WRITE_ONCE(rcu_cpu_stall_timeout, 3); till_stall_check = 3; } else if (till_stall_check > 300) { WRITE_ONCE(rcu_cpu_stall_timeout, 300); till_stall_check = 300; } return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;}EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check); /* Don't do RCU CPU stall warnings during long sysrq printouts. */void rcu_sysrq_start(void){ if (!rcu_cpu_stall_suppress) rcu_cpu_stall_suppress = 2;} void rcu_sysrq_end(void){ if (rcu_cpu_stall_suppress == 2) rcu_cpu_stall_suppress = 0;} /* Don't print RCU CPU stall warnings during a kernel panic. */static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr){ rcu_cpu_stall_suppress = 1; return NOTIFY_DONE;} static struct notifier_block rcu_panic_block = { .notifier_call = rcu_panic,}; static int __init check_cpu_stall_init(void){ atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block); return 0;}early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */static void panic_on_rcu_stall(const struct cpumask *stalled_mask){ static int cpu_stall; /* * Attempt to kick out the BPF scheduler if it's installed and defer * the panic to give the system a chance to recover. */ if (scx_rcu_cpu_stall(stalled_mask)) return; if (++cpu_stall < sysctl_max_rcu_stall_to_panic) return; if (sysctl_panic_on_rcu_stall) panic("RCU Stall\n");} /** * rcu_cpu_stall_reset - restart stall-warning timeout for current grace period * * To perform the reset request from the caller, disable stall detection until * 3 fqs loops have passed. This is required to ensure a fresh jiffies is * loaded. It should be safe to do from the fqs loop as enough timer * interrupts and context switches should have passed. * * The caller must disable hard irqs. */void rcu_cpu_stall_reset(void){ WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 3); WRITE_ONCE(rcu_state.jiffies_stall, ULONG_MAX);} ////////////////////////////////////////////////////////////////////////////////// Interaction with RCU grace periods /* Start of new grace period, so record stall time (and forcing times). */static void record_gp_stall_check_time(void){ unsigned long j = jiffies; unsigned long j1; WRITE_ONCE(rcu_state.gp_start, j); j1 = rcu_jiffies_till_stall_check(); smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq. WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 0); WRITE_ONCE(rcu_state.jiffies_stall, j + j1); rcu_state.jiffies_resched = j + j1 / 2; rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);} /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */static void zero_cpu_stall_ticks(struct rcu_data *rdp){ rdp->ticks_this_gp = 0; rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); WRITE_ONCE(rdp->last_fqs_resched, jiffies);} /* * If too much time has passed in the current grace period, and if * so configured, go kick the relevant kthreads. */static void rcu_stall_kick_kthreads(void){ unsigned long j; if (!READ_ONCE(rcu_kick_kthreads)) return; j = READ_ONCE(rcu_state.jiffies_kick_kthreads); if (time_after(jiffies, j) && rcu_state.gp_kthread && (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) { WARN_ONCE(1, "Kicking %s grace-period kthread\n", rcu_state.name); rcu_ftrace_dump(DUMP_ALL); wake_up_process(rcu_state.gp_kthread); WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ); }} /* * Handler for the irq_work request posted about halfway into the RCU CPU * stall timeout, and used to detect excessive irq disabling. Set state * appropriately, but just complain if there is unexpected state on entry. */static void rcu_iw_handler(struct irq_work *iwp){ struct rcu_data *rdp; struct rcu_node *rnp; rdp = container_of(iwp, struct rcu_data, rcu_iw); rnp = rdp->mynode; raw_spin_lock_rcu_node(rnp); if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) { rdp->rcu_iw_gp_seq = rnp->gp_seq; rdp->rcu_iw_pending = false; } raw_spin_unlock_rcu_node(rnp);} ////////////////////////////////////////////////////////////////////////////////// Printing RCU CPU stall warnings #ifdef CONFIG_PREEMPT_RCU /* * Dump detailed information for all tasks blocking the current RCU * grace period on the specified rcu_node structure. */static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp){ unsigned long flags; struct task_struct *t; raw_spin_lock_irqsave_rcu_node(rnp, flags); if (!rcu_preempt_blocked_readers_cgp(rnp)) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return; } t = list_entry(rnp->gp_tasks->prev, struct task_struct, rcu_node_entry); list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { /* * We could be printing a lot while holding a spinlock. * Avoid triggering hard lockup. */ touch_nmi_watchdog(); sched_show_task(t); } raw_spin_unlock_irqrestore_rcu_node(rnp, flags);} // Communicate task state back to the RCU CPU stall warning request.struct rcu_stall_chk_rdr { int nesting; union rcu_special rs; bool on_blkd_list;}; /* * Report out the state of a not-running task that is stalling the * current RCU grace period. */static int check_slow_task(struct task_struct *t, void *arg){ struct rcu_stall_chk_rdr *rscrp = arg; if (task_curr(t)) return -EBUSY; // It is running, so decline to inspect it. rscrp->nesting = t->rcu_read_lock_nesting; rscrp->rs = t->rcu_read_unlock_special; rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry); return 0;} /* * Scan the current list of tasks blocked within RCU read-side critical * sections, printing out the tid of each of the first few of them. */static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock){ int i = 0; int ndetected = 0; struct rcu_stall_chk_rdr rscr; struct task_struct *t; struct task_struct *ts[8]; lockdep_assert_irqs_disabled(); if (!rcu_preempt_blocked_readers_cgp(rnp)) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return 0; } pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", rnp->level, rnp->grplo, rnp->grphi); t = list_entry(rnp->gp_tasks->prev, struct task_struct, rcu_node_entry); list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { get_task_struct(t); ts[i++] = t; if (i >= ARRAY_SIZE(ts)) break; } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); while (i) { t = ts[--i]; if (task_call_func(t, check_slow_task, &rscr)) pr_cont(" P%d", t->pid); else pr_cont(" P%d/%d:%c%c%c%c", t->pid, rscr.nesting, ".b"[rscr.rs.b.blocked], ".q"[rscr.rs.b.need_qs], ".e"[rscr.rs.b.exp_hint], ".l"[rscr.on_blkd_list]); lockdep_assert_irqs_disabled(); put_task_struct(t); ndetected++; } pr_cont("\n"); return ndetected;} #else /* #ifdef CONFIG_PREEMPT_RCU */ /* * Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections. */static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp){} /* * Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections. */static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock){ raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return 0;}#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* * Dump stacks of all tasks running on stalled CPUs. First try using * NMIs, but fall back to manual remote stack tracing on architectures * that don't support NMI-based stack dumps. The NMI-triggered stack * traces are more accurate because they are printed by the target CPU. */static void rcu_dump_cpu_stacks(unsigned long gp_seq){ int cpu; unsigned long flags; struct rcu_node *rnp; rcu_for_each_leaf_node(rnp) { printk_deferred_enter(); for_each_leaf_node_possible_cpu(rnp, cpu) { if (gp_seq != data_race(rcu_state.gp_seq)) { printk_deferred_exit(); pr_err("INFO: Stall ended during stack backtracing.\n"); return; } if (!(data_race(rnp->qsmask) & leaf_node_cpu_bit(rnp, cpu))) continue; raw_spin_lock_irqsave_rcu_node(rnp, flags); if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { if (cpu_is_offline(cpu)) pr_err("Offline CPU %d blocking current GP.\n", cpu); else dump_cpu_task(cpu); } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } printk_deferred_exit(); }} static const char * const gp_state_names[] = { [RCU_GP_IDLE] = "RCU_GP_IDLE", [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS", [RCU_GP_DONE_GPS] = "RCU_GP_DONE_GPS", [RCU_GP_ONOFF] = "RCU_GP_ONOFF", [RCU_GP_INIT] = "RCU_GP_INIT", [RCU_GP_WAIT_FQS] = "RCU_GP_WAIT_FQS", [RCU_GP_DOING_FQS] = "RCU_GP_DOING_FQS", [RCU_GP_CLEANUP] = "RCU_GP_CLEANUP", [RCU_GP_CLEANED] = "RCU_GP_CLEANED",}; /* * Convert a ->gp_state value to a character string. */static const char *gp_state_getname(short gs){ if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names)) return "???"; return gp_state_names[gs];} /* Is the RCU grace-period kthread being starved of CPU time? */static bool rcu_is_gp_kthread_starving(unsigned long *jp){ unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity); if (jp) *jp = j; return j > 2 * HZ;} static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp){ int cpu; struct task_struct *rcuc; unsigned long j; rcuc = rdp->rcu_cpu_kthread_task; if (!rcuc) return false; cpu = task_cpu(rcuc); if (cpu_is_offline(cpu) || idle_cpu(cpu)) return false; j = jiffies - READ_ONCE(rdp->rcuc_activity); if (jp) *jp = j; return j > 2 * HZ;} static void print_cpu_stat_info(int cpu){ struct rcu_snap_record rsr, *rsrp; struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (!rcu_cpu_stall_cputime) return; rsrp = &rdp->snap_record; if (rsrp->gp_seq != rdp->gp_seq) return; rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); pr_err("\t hardirqs softirqs csw/system\n"); pr_err("\t number: %8lld %10d %12lld\n", kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu) - rsrp->nr_hardirqs, kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs, nr_context_switches_cpu(cpu) - rsrp->nr_csw); pr_err("\tcputime: %8lld %10lld %12lld ==> %d(ms)\n", div_u64(rsr.cputime_irq - rsrp->cputime_irq, NSEC_PER_MSEC),Lines 1–500 of 1199. Load more
// SPDX-License-Identifier: GPL-2.0+ /* * RCU CPU stall warnings for normal RCU grace periods * * Copyright IBM Corporation, 2019 * * Author: Paul E. McKenney <paulmck@linux.ibm.com> */ #include <linux/console.h> #include <linux/kvm_para.h> #include <linux/rcu_notifier.h> #include <linux/smp.h> ////////////////////////////////////////////////////////////////////////////// // // Controlling CPU stall warnings, including delay calculation. /* panic() on RCU Stall sysctl. */ static int sysctl_panic_on_rcu_stall __read_mostly; static int sysctl_max_rcu_stall_to_panic __read_mostly; static const struct ctl_table rcu_stall_sysctl_table[] = { { .procname = "panic_on_rcu_stall", .data = &sysctl_panic_on_rcu_stall, .maxlen = sizeof(sysctl_panic_on_rcu_stall), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, { .procname = "max_rcu_stall_to_panic", .data = &sysctl_max_rcu_stall_to_panic, .maxlen = sizeof(sysctl_max_rcu_stall_to_panic), .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_INT_MAX, }, }; static int __init init_rcu_stall_sysctl(void) { register_sysctl_init("kernel", rcu_stall_sysctl_table); return 0; } subsys_initcall(init_rcu_stall_sysctl); #ifdef CONFIG_SYSFS static unsigned int rcu_stall_count; static ssize_t rcu_stall_count_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { return sysfs_emit(page, "%u\n", rcu_stall_count); } static struct kobj_attribute rcu_stall_count_attr = __ATTR_RO(rcu_stall_count); static __init int kernel_rcu_stall_sysfs_init(void) { sysfs_add_file_to_group(kernel_kobj, &rcu_stall_count_attr.attr, NULL); return 0; } late_initcall(kernel_rcu_stall_sysfs_init); #endif // CONFIG_SYSFS #ifdef CONFIG_PROVE_RCU #define RCU_STALL_DELAY_DELTA (5 * HZ) #else #define RCU_STALL_DELAY_DELTA 0 #endif #define RCU_STALL_MIGHT_DIV 8 #define RCU_STALL_MIGHT_MIN (2 * HZ) int rcu_exp_jiffies_till_stall_check(void) { int cpu_stall_timeout = READ_ONCE(rcu_exp_cpu_stall_timeout); int exp_stall_delay_delta = 0; int till_stall_check; // Zero says to use rcu_cpu_stall_timeout, but in milliseconds. if (!cpu_stall_timeout) cpu_stall_timeout = jiffies_to_msecs(rcu_jiffies_till_stall_check()); // Limit check must be consistent with the Kconfig limits for // CONFIG_RCU_EXP_CPU_STALL_TIMEOUT, so check the allowed range. // The minimum clamped value is "2UL", because at least one full // tick has to be guaranteed. till_stall_check = clamp(msecs_to_jiffies(cpu_stall_timeout), 2UL, 300UL * HZ); if (cpu_stall_timeout && jiffies_to_msecs(till_stall_check) != cpu_stall_timeout) WRITE_ONCE(rcu_exp_cpu_stall_timeout, jiffies_to_msecs(till_stall_check)); #ifdef CONFIG_PROVE_RCU /* Add extra ~25% out of till_stall_check. */ exp_stall_delay_delta = ((till_stall_check * 25) / 100) + 1; #endif return till_stall_check + exp_stall_delay_delta; } EXPORT_SYMBOL_GPL(rcu_exp_jiffies_till_stall_check); /* Limit-check stall timeouts specified at boottime and runtime. */ int rcu_jiffies_till_stall_check(void) { int till_stall_check = READ_ONCE(rcu_cpu_stall_timeout); /* * Limit check must be consistent with the Kconfig limits * for CONFIG_RCU_CPU_STALL_TIMEOUT. */ if (till_stall_check < 3) { WRITE_ONCE(rcu_cpu_stall_timeout, 3); till_stall_check = 3; } else if (till_stall_check > 300) { WRITE_ONCE(rcu_cpu_stall_timeout, 300); till_stall_check = 300; } return till_stall_check * HZ + RCU_STALL_DELAY_DELTA; } EXPORT_SYMBOL_GPL(rcu_jiffies_till_stall_check); /* Don't do RCU CPU stall warnings during long sysrq printouts. */ void rcu_sysrq_start(void) { if (!rcu_cpu_stall_suppress) rcu_cpu_stall_suppress = 2; } void rcu_sysrq_end(void) { if (rcu_cpu_stall_suppress == 2) rcu_cpu_stall_suppress = 0; } /* Don't print RCU CPU stall warnings during a kernel panic. */ static int rcu_panic(struct notifier_block *this, unsigned long ev, void *ptr) { rcu_cpu_stall_suppress = 1; return NOTIFY_DONE; } static struct notifier_block rcu_panic_block = { .notifier_call = rcu_panic, }; static int __init check_cpu_stall_init(void) { atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block); return 0; } early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ static void panic_on_rcu_stall(const struct cpumask *stalled_mask) { static int cpu_stall; /* * Attempt to kick out the BPF scheduler if it's installed and defer * the panic to give the system a chance to recover. */ if (scx_rcu_cpu_stall(stalled_mask)) return; if (++cpu_stall < sysctl_max_rcu_stall_to_panic) return; if (sysctl_panic_on_rcu_stall) panic("RCU Stall\n"); } /** * rcu_cpu_stall_reset - restart stall-warning timeout for current grace period * * To perform the reset request from the caller, disable stall detection until * 3 fqs loops have passed. This is required to ensure a fresh jiffies is * loaded. It should be safe to do from the fqs loop as enough timer * interrupts and context switches should have passed. * * The caller must disable hard irqs. */ void rcu_cpu_stall_reset(void) { WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 3); WRITE_ONCE(rcu_state.jiffies_stall, ULONG_MAX); } ////////////////////////////////////////////////////////////////////////////// // // Interaction with RCU grace periods /* Start of new grace period, so record stall time (and forcing times). */ static void record_gp_stall_check_time(void) { unsigned long j = jiffies; unsigned long j1; WRITE_ONCE(rcu_state.gp_start, j); j1 = rcu_jiffies_till_stall_check(); smp_mb(); // ->gp_start before ->jiffies_stall and caller's ->gp_seq. WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 0); WRITE_ONCE(rcu_state.jiffies_stall, j + j1); rcu_state.jiffies_resched = j + j1 / 2; rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs); } /* Zero ->ticks_this_gp and snapshot the number of RCU softirq handlers. */ static void zero_cpu_stall_ticks(struct rcu_data *rdp) { rdp->ticks_this_gp = 0; rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id()); WRITE_ONCE(rdp->last_fqs_resched, jiffies); } /* * If too much time has passed in the current grace period, and if * so configured, go kick the relevant kthreads. */ static void rcu_stall_kick_kthreads(void) { unsigned long j; if (!READ_ONCE(rcu_kick_kthreads)) return; j = READ_ONCE(rcu_state.jiffies_kick_kthreads); if (time_after(jiffies, j) && rcu_state.gp_kthread && (rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) { WARN_ONCE(1, "Kicking %s grace-period kthread\n", rcu_state.name); rcu_ftrace_dump(DUMP_ALL); wake_up_process(rcu_state.gp_kthread); WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ); } } /* * Handler for the irq_work request posted about halfway into the RCU CPU * stall timeout, and used to detect excessive irq disabling. Set state * appropriately, but just complain if there is unexpected state on entry. */ static void rcu_iw_handler(struct irq_work *iwp) { struct rcu_data *rdp; struct rcu_node *rnp; rdp = container_of(iwp, struct rcu_data, rcu_iw); rnp = rdp->mynode; raw_spin_lock_rcu_node(rnp); if (!WARN_ON_ONCE(!rdp->rcu_iw_pending)) { rdp->rcu_iw_gp_seq = rnp->gp_seq; rdp->rcu_iw_pending = false; } raw_spin_unlock_rcu_node(rnp); } ////////////////////////////////////////////////////////////////////////////// // // Printing RCU CPU stall warnings #ifdef CONFIG_PREEMPT_RCU /* * Dump detailed information for all tasks blocking the current RCU * grace period on the specified rcu_node structure. */ static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp) { unsigned long flags; struct task_struct *t; raw_spin_lock_irqsave_rcu_node(rnp, flags); if (!rcu_preempt_blocked_readers_cgp(rnp)) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return; } t = list_entry(rnp->gp_tasks->prev, struct task_struct, rcu_node_entry); list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { /* * We could be printing a lot while holding a spinlock. * Avoid triggering hard lockup. */ touch_nmi_watchdog(); sched_show_task(t); } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } // Communicate task state back to the RCU CPU stall warning request. struct rcu_stall_chk_rdr { int nesting; union rcu_special rs; bool on_blkd_list; }; /* * Report out the state of a not-running task that is stalling the * current RCU grace period. */ static int check_slow_task(struct task_struct *t, void *arg) { struct rcu_stall_chk_rdr *rscrp = arg; if (task_curr(t)) return -EBUSY; // It is running, so decline to inspect it. rscrp->nesting = t->rcu_read_lock_nesting; rscrp->rs = t->rcu_read_unlock_special; rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry); return 0; } /* * Scan the current list of tasks blocked within RCU read-side critical * sections, printing out the tid of each of the first few of them. */ static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock) { int i = 0; int ndetected = 0; struct rcu_stall_chk_rdr rscr; struct task_struct *t; struct task_struct *ts[8]; lockdep_assert_irqs_disabled(); if (!rcu_preempt_blocked_readers_cgp(rnp)) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return 0; } pr_err("\tTasks blocked on level-%d rcu_node (CPUs %d-%d):", rnp->level, rnp->grplo, rnp->grphi); t = list_entry(rnp->gp_tasks->prev, struct task_struct, rcu_node_entry); list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) { get_task_struct(t); ts[i++] = t; if (i >= ARRAY_SIZE(ts)) break; } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); while (i) { t = ts[--i]; if (task_call_func(t, check_slow_task, &rscr)) pr_cont(" P%d", t->pid); else pr_cont(" P%d/%d:%c%c%c%c", t->pid, rscr.nesting, ".b"[rscr.rs.b.blocked], ".q"[rscr.rs.b.need_qs], ".e"[rscr.rs.b.exp_hint], ".l"[rscr.on_blkd_list]); lockdep_assert_irqs_disabled(); put_task_struct(t); ndetected++; } pr_cont("\n"); return ndetected; } #else /* #ifdef CONFIG_PREEMPT_RCU */ /* * Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections. */ static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp) { } /* * Because preemptible RCU does not exist, we never have to check for * tasks blocked within RCU read-side critical sections. */ static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) __releases(rnp->lock) { raw_spin_unlock_irqrestore_rcu_node(rnp, flags); return 0; } #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ /* * Dump stacks of all tasks running on stalled CPUs. First try using * NMIs, but fall back to manual remote stack tracing on architectures * that don't support NMI-based stack dumps. The NMI-triggered stack * traces are more accurate because they are printed by the target CPU. */ static void rcu_dump_cpu_stacks(unsigned long gp_seq) { int cpu; unsigned long flags; struct rcu_node *rnp; rcu_for_each_leaf_node(rnp) { printk_deferred_enter(); for_each_leaf_node_possible_cpu(rnp, cpu) { if (gp_seq != data_race(rcu_state.gp_seq)) { printk_deferred_exit(); pr_err("INFO: Stall ended during stack backtracing.\n"); return; } if (!(data_race(rnp->qsmask) & leaf_node_cpu_bit(rnp, cpu))) continue; raw_spin_lock_irqsave_rcu_node(rnp, flags); if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { if (cpu_is_offline(cpu)) pr_err("Offline CPU %d blocking current GP.\n", cpu); else dump_cpu_task(cpu); } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } printk_deferred_exit(); } } static const char * const gp_state_names[] = { [RCU_GP_IDLE] = "RCU_GP_IDLE", [RCU_GP_WAIT_GPS] = "RCU_GP_WAIT_GPS", [RCU_GP_DONE_GPS] = "RCU_GP_DONE_GPS", [RCU_GP_ONOFF] = "RCU_GP_ONOFF", [RCU_GP_INIT] = "RCU_GP_INIT", [RCU_GP_WAIT_FQS] = "RCU_GP_WAIT_FQS", [RCU_GP_DOING_FQS] = "RCU_GP_DOING_FQS", [RCU_GP_CLEANUP] = "RCU_GP_CLEANUP", [RCU_GP_CLEANED] = "RCU_GP_CLEANED", }; /* * Convert a ->gp_state value to a character string. */ static const char *gp_state_getname(short gs) { if (gs < 0 || gs >= ARRAY_SIZE(gp_state_names)) return "???"; return gp_state_names[gs]; } /* Is the RCU grace-period kthread being starved of CPU time? */ static bool rcu_is_gp_kthread_starving(unsigned long *jp) { unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity); if (jp) *jp = j; return j > 2 * HZ; } static bool rcu_is_rcuc_kthread_starving(struct rcu_data *rdp, unsigned long *jp) { int cpu; struct task_struct *rcuc; unsigned long j; rcuc = rdp->rcu_cpu_kthread_task; if (!rcuc) return false; cpu = task_cpu(rcuc); if (cpu_is_offline(cpu) || idle_cpu(cpu)) return false; j = jiffies - READ_ONCE(rdp->rcuc_activity); if (jp) *jp = j; return j > 2 * HZ; } static void print_cpu_stat_info(int cpu) { struct rcu_snap_record rsr, *rsrp; struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); if (!rcu_cpu_stall_cputime) return; rsrp = &rdp->snap_record; if (rsrp->gp_seq != rdp->gp_seq) return; rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu); rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu); rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu); pr_err("\t hardirqs softirqs csw/system\n"); pr_err("\t number: %8lld %10d %12lld\n", kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu) - rsrp->nr_hardirqs, kstat_cpu_softirqs_sum(cpu) - rsrp->nr_softirqs, nr_context_switches_cpu(cpu) - rsrp->nr_csw); pr_err("\tcputime: %8lld %10lld %12lld ==> %d(ms)\n", div_u64(rsr.cputime_irq - rsrp->cputime_irq, NSEC_PER_MSEC),