// SPDX-License-Identifier: GPL-2.0-only/* * Generic helpers for smp ipi calls * * (C) Jens Axboe <jens.axboe@oracle.com> 2008 */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/irq_work.h>#include <linux/rcupdate.h>#include <linux/rculist.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/percpu.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/gfp.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/sched.h>#include <linux/sched/idle.h>#include <linux/hypervisor.h>#include <linux/sched/clock.h>#include <linux/nmi.h>#include <linux/sched/debug.h>#include <linux/jump_label.h> #include "smpboot.h"#include "sched/smp.h" #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK) #ifdef CONFIG_CSD_LOCK_WAIT_DEBUGunion cfd_seq_cnt { u64 val; struct { u64 src:16; u64 dst:16;#define CFD_SEQ_NOCPU 0xffff u64 type:4;#define CFD_SEQ_QUEUE 0#define CFD_SEQ_IPI 1#define CFD_SEQ_NOIPI 2#define CFD_SEQ_PING 3#define CFD_SEQ_PINGED 4#define CFD_SEQ_HANDLE 5#define CFD_SEQ_DEQUEUE 6#define CFD_SEQ_IDLE 7#define CFD_SEQ_GOTIPI 8#define CFD_SEQ_HDLEND 9 u64 cnt:28; } u;}; static char *seq_type[] = { [CFD_SEQ_QUEUE] = "queue", [CFD_SEQ_IPI] = "ipi", [CFD_SEQ_NOIPI] = "noipi", [CFD_SEQ_PING] = "ping", [CFD_SEQ_PINGED] = "pinged", [CFD_SEQ_HANDLE] = "handle", [CFD_SEQ_DEQUEUE] = "dequeue (src CPU 0 == empty)", [CFD_SEQ_IDLE] = "idle", [CFD_SEQ_GOTIPI] = "gotipi", [CFD_SEQ_HDLEND] = "hdlend (src CPU 0 == early)",}; struct cfd_seq_local { u64 ping; u64 pinged; u64 handle; u64 dequeue; u64 idle; u64 gotipi; u64 hdlend;};#endif struct cfd_percpu { call_single_data_t csd;#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG u64 seq_queue; u64 seq_ipi; u64 seq_noipi;#endif}; struct call_function_data { struct cfd_percpu __percpu *pcpu; cpumask_var_t cpumask; cpumask_var_t cpumask_ipi;}; static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); static void flush_smp_call_function_queue(bool warn_cpu_offline); int smpcfd_prepare_cpu(unsigned int cpu){ struct call_function_data *cfd = &per_cpu(cfd_data, cpu); if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, cpu_to_node(cpu))) return -ENOMEM; if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL, cpu_to_node(cpu))) { free_cpumask_var(cfd->cpumask); return -ENOMEM; } cfd->pcpu = alloc_percpu(struct cfd_percpu); if (!cfd->pcpu) { free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); return -ENOMEM; } return 0;} int smpcfd_dead_cpu(unsigned int cpu){ struct call_function_data *cfd = &per_cpu(cfd_data, cpu); free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); free_percpu(cfd->pcpu); return 0;} int smpcfd_dying_cpu(unsigned int cpu){ /* * The IPIs for the smp-call-function callbacks queued by other * CPUs might arrive late, either due to hardware latencies or * because this CPU disabled interrupts (inside stop-machine) * before the IPIs were sent. So flush out any pending callbacks * explicitly (without waiting for the IPIs to arrive), to * ensure that the outgoing CPU doesn't go offline with work * still pending. */ flush_smp_call_function_queue(false); irq_work_run(); return 0;} void __init call_function_init(void){ int i; for_each_possible_cpu(i) init_llist_head(&per_cpu(call_single_queue, i)); smpcfd_prepare_cpu(smp_processor_id());} #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG static DEFINE_STATIC_KEY_FALSE(csdlock_debug_enabled);static DEFINE_STATIC_KEY_FALSE(csdlock_debug_extended); static int __init csdlock_debug(char *str){ unsigned int val = 0; if (str && !strcmp(str, "ext")) { val = 1; static_branch_enable(&csdlock_debug_extended); } else get_option(&str, &val); if (val) static_branch_enable(&csdlock_debug_enabled); return 0;}early_param("csdlock_debug", csdlock_debug); static DEFINE_PER_CPU(call_single_data_t *, cur_csd);static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func);static DEFINE_PER_CPU(void *, cur_csd_info);static DEFINE_PER_CPU(struct cfd_seq_local, cfd_seq_local); #define CSD_LOCK_TIMEOUT (5ULL * NSEC_PER_SEC)static atomic_t csd_bug_count = ATOMIC_INIT(0);static u64 cfd_seq; #define CFD_SEQ(s, d, t, c) \ (union cfd_seq_cnt){ .u.src = s, .u.dst = d, .u.type = t, .u.cnt = c } static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type){ union cfd_seq_cnt new, old; new = CFD_SEQ(src, dst, type, 0); do { old.val = READ_ONCE(cfd_seq); new.u.cnt = old.u.cnt + 1; } while (cmpxchg(&cfd_seq, old.val, new.val) != old.val); return old.val;} #define cfd_seq_store(var, src, dst, type) \ do { \ if (static_branch_unlikely(&csdlock_debug_extended)) \ var = cfd_seq_inc(src, dst, type); \ } while (0) /* Record current CSD work for current CPU, NULL to erase. */static void __csd_lock_record(struct __call_single_data *csd){ if (!csd) { smp_mb(); /* NULL cur_csd after unlock. */ __this_cpu_write(cur_csd, NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); smp_wmb(); /* func and info before csd. */ __this_cpu_write(cur_csd, csd); smp_mb(); /* Update cur_csd before function call. */ /* Or before unlock, as the case may be. */} static __always_inline void csd_lock_record(struct __call_single_data *csd){ if (static_branch_unlikely(&csdlock_debug_enabled)) __csd_lock_record(csd);} static int csd_lock_wait_getcpu(struct __call_single_data *csd){ unsigned int csd_type; csd_type = CSD_TYPE(csd); if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC) return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ return -1;} static void cfd_seq_data_add(u64 val, unsigned int src, unsigned int dst, unsigned int type, union cfd_seq_cnt *data, unsigned int *n_data, unsigned int now){ union cfd_seq_cnt new[2]; unsigned int i, j, k; new[0].val = val; new[1] = CFD_SEQ(src, dst, type, new[0].u.cnt + 1); for (i = 0; i < 2; i++) { if (new[i].u.cnt <= now) new[i].u.cnt |= 0x80000000U; for (j = 0; j < *n_data; j++) { if (new[i].u.cnt == data[j].u.cnt) { /* Direct read value trumps generated one. */ if (i == 0) data[j].val = new[i].val; break; } if (new[i].u.cnt < data[j].u.cnt) { for (k = *n_data; k > j; k--) data[k].val = data[k - 1].val; data[j].val = new[i].val; (*n_data)++; break; } } if (j == *n_data) { data[j].val = new[i].val; (*n_data)++; } }} static const char *csd_lock_get_type(unsigned int type){ return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type];} static void csd_lock_print_extended(struct __call_single_data *csd, int cpu){ struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu); unsigned int srccpu = csd->node.src; struct call_function_data *cfd = per_cpu_ptr(&cfd_data, srccpu); struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu); unsigned int now; union cfd_seq_cnt data[2 * ARRAY_SIZE(seq_type)]; unsigned int n_data = 0, i; data[0].val = READ_ONCE(cfd_seq); now = data[0].u.cnt; cfd_seq_data_add(pcpu->seq_queue, srccpu, cpu, CFD_SEQ_QUEUE, data, &n_data, now); cfd_seq_data_add(pcpu->seq_ipi, srccpu, cpu, CFD_SEQ_IPI, data, &n_data, now); cfd_seq_data_add(pcpu->seq_noipi, srccpu, cpu, CFD_SEQ_NOIPI, data, &n_data, now); cfd_seq_data_add(per_cpu(cfd_seq_local.ping, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PING, data, &n_data, now); cfd_seq_data_add(per_cpu(cfd_seq_local.pinged, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PINGED, data, &n_data, now); cfd_seq_data_add(seq->idle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_IDLE, data, &n_data, now); cfd_seq_data_add(seq->gotipi, CFD_SEQ_NOCPU, cpu, CFD_SEQ_GOTIPI, data, &n_data, now); cfd_seq_data_add(seq->handle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HANDLE, data, &n_data, now); cfd_seq_data_add(seq->dequeue, CFD_SEQ_NOCPU, cpu, CFD_SEQ_DEQUEUE, data, &n_data, now); cfd_seq_data_add(seq->hdlend, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HDLEND, data, &n_data, now); for (i = 0; i < n_data; i++) { pr_alert("\tcsd: cnt(%07x): %04x->%04x %s\n", data[i].u.cnt & ~0x80000000U, data[i].u.src, data[i].u.dst, csd_lock_get_type(data[i].u.type)); } pr_alert("\tcsd: cnt now: %07x\n", now);} /* * Complain if too much time spent waiting. Note that only * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id){ int cpu = -1; int cpux; bool firsttime; u64 ts2, ts_delta; call_single_data_t *cpu_cur_csd; unsigned int flags = READ_ONCE(csd->node.u_flags); if (!(flags & CSD_FLAG_LOCK)) { if (!unlikely(*bug_id)) return true; cpu = csd_lock_wait_getcpu(csd); pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n", *bug_id, raw_smp_processor_id(), cpu); return true; } ts2 = sched_clock(); ts_delta = ts2 - *ts1; if (likely(ts_delta <= CSD_LOCK_TIMEOUT)) return false; firsttime = !*bug_id; if (firsttime) *bug_id = atomic_inc_return(&csd_bug_count); cpu = csd_lock_wait_getcpu(csd); if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu)) cpux = 0; else cpux = cpu; cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %llu ns for CPU#%02d %pS(%ps).\n", firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), ts2 - ts0, cpu, csd->func, csd->info); if (cpu_cur_csd && csd != cpu_cur_csd) { pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n", *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)), READ_ONCE(per_cpu(cur_csd_info, cpux))); } else { pr_alert("\tcsd: CSD lock (#%d) %s.\n", *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); } if (cpu >= 0) { if (static_branch_unlikely(&csdlock_debug_extended)) csd_lock_print_extended(csd, cpu); if (!trigger_single_cpu_backtrace(cpu)) dump_cpu_task(cpu); if (!cpu_cur_csd) { pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); } } dump_stack(); *ts1 = ts2; return false;} /* * csd_lock/csd_unlock used to serialize access to per-cpu csd resources * * For non-synchronous ipi calls the csd can still be in use by the * previous function call. For multi-cpu calls its even more interesting * as we'll have to ensure no other cpu is observing our csd. */static void __csd_lock_wait(struct __call_single_data *csd){ int bug_id = 0; u64 ts0, ts1; ts1 = ts0 = sched_clock(); for (;;) { if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id)) break; cpu_relax(); } smp_acquire__after_ctrl_dep();} static __always_inline void csd_lock_wait(struct __call_single_data *csd){ if (static_branch_unlikely(&csdlock_debug_enabled)) { __csd_lock_wait(csd); return; } smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));} static void __smp_call_single_queue_debug(int cpu, struct llist_node *node){ unsigned int this_cpu = smp_processor_id(); struct cfd_seq_local *seq = this_cpu_ptr(&cfd_seq_local); struct call_function_data *cfd = this_cpu_ptr(&cfd_data); struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu); cfd_seq_store(pcpu->seq_queue, this_cpu, cpu, CFD_SEQ_QUEUE); if (llist_add(node, &per_cpu(call_single_queue, cpu))) { cfd_seq_store(pcpu->seq_ipi, this_cpu, cpu, CFD_SEQ_IPI); cfd_seq_store(seq->ping, this_cpu, cpu, CFD_SEQ_PING); send_call_function_single_ipi(cpu); cfd_seq_store(seq->pinged, this_cpu, cpu, CFD_SEQ_PINGED); } else { cfd_seq_store(pcpu->seq_noipi, this_cpu, cpu, CFD_SEQ_NOIPI); }}#else#define cfd_seq_store(var, src, dst, type) static void csd_lock_record(struct __call_single_data *csd){} static __always_inline void csd_lock_wait(struct __call_single_data *csd){ smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));}#endif static __always_inline void csd_lock(struct __call_single_data *csd){ csd_lock_wait(csd); csd->node.u_flags |= CSD_FLAG_LOCK; /* * prevent CPU from reordering the above assignment * to ->flags with any subsequent assignments to other * fields of the specified call_single_data_t structure: */ smp_wmb();} static __always_inline void csd_unlock(struct __call_single_data *csd){ WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); /* * ensure we're all done before releasing data: */ smp_store_release(&csd->node.u_flags, 0);} static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); void __smp_call_single_queue(int cpu, struct llist_node *node){#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG if (static_branch_unlikely(&csdlock_debug_extended)) { unsigned int type; type = CSD_TYPE(container_of(node, call_single_data_t, node.llist)); if (type == CSD_TYPE_SYNC || type == CSD_TYPE_ASYNC) { __smp_call_single_queue_debug(cpu, node); return; } }#endif /* * The list addition should be visible before sending the IPI * handler locks the list to pull the entry off it because of * normal cache coherency rules implied by spinlocks. * * If IPIs can go out of order to the cache coherency protocol * in an architecture, sufficient synchronisation should be added * to arch code to make it appear to obey cache coherency WRT * locking and barrier primitives. Generic code isn't really * equipped to do the right thing... */ if (llist_add(node, &per_cpu(call_single_queue, cpu))) send_call_function_single_ipi(cpu);} /* * Insert a previously allocated call_single_data_t elementLines 1–500 of 1237. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Generic helpers for smp ipi calls * * (C) Jens Axboe <jens.axboe@oracle.com> 2008 */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/irq_work.h> #include <linux/rcupdate.h> #include <linux/rculist.h> #include <linux/kernel.h> #include <linux/export.h> #include <linux/percpu.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/gfp.h> #include <linux/smp.h> #include <linux/cpu.h> #include <linux/sched.h> #include <linux/sched/idle.h> #include <linux/hypervisor.h> #include <linux/sched/clock.h> #include <linux/nmi.h> #include <linux/sched/debug.h> #include <linux/jump_label.h> #include "smpboot.h" #include "sched/smp.h" #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK) #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG union cfd_seq_cnt { u64 val; struct { u64 src:16; u64 dst:16; #define CFD_SEQ_NOCPU 0xffff u64 type:4; #define CFD_SEQ_QUEUE 0 #define CFD_SEQ_IPI 1 #define CFD_SEQ_NOIPI 2 #define CFD_SEQ_PING 3 #define CFD_SEQ_PINGED 4 #define CFD_SEQ_HANDLE 5 #define CFD_SEQ_DEQUEUE 6 #define CFD_SEQ_IDLE 7 #define CFD_SEQ_GOTIPI 8 #define CFD_SEQ_HDLEND 9 u64 cnt:28; } u; }; static char *seq_type[] = { [CFD_SEQ_QUEUE] = "queue", [CFD_SEQ_IPI] = "ipi", [CFD_SEQ_NOIPI] = "noipi", [CFD_SEQ_PING] = "ping", [CFD_SEQ_PINGED] = "pinged", [CFD_SEQ_HANDLE] = "handle", [CFD_SEQ_DEQUEUE] = "dequeue (src CPU 0 == empty)", [CFD_SEQ_IDLE] = "idle", [CFD_SEQ_GOTIPI] = "gotipi", [CFD_SEQ_HDLEND] = "hdlend (src CPU 0 == early)", }; struct cfd_seq_local { u64 ping; u64 pinged; u64 handle; u64 dequeue; u64 idle; u64 gotipi; u64 hdlend; }; #endif struct cfd_percpu { call_single_data_t csd; #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG u64 seq_queue; u64 seq_ipi; u64 seq_noipi; #endif }; struct call_function_data { struct cfd_percpu __percpu *pcpu; cpumask_var_t cpumask; cpumask_var_t cpumask_ipi; }; static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); static void flush_smp_call_function_queue(bool warn_cpu_offline); int smpcfd_prepare_cpu(unsigned int cpu) { struct call_function_data *cfd = &per_cpu(cfd_data, cpu); if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, cpu_to_node(cpu))) return -ENOMEM; if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL, cpu_to_node(cpu))) { free_cpumask_var(cfd->cpumask); return -ENOMEM; } cfd->pcpu = alloc_percpu(struct cfd_percpu); if (!cfd->pcpu) { free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); return -ENOMEM; } return 0; } int smpcfd_dead_cpu(unsigned int cpu) { struct call_function_data *cfd = &per_cpu(cfd_data, cpu); free_cpumask_var(cfd->cpumask); free_cpumask_var(cfd->cpumask_ipi); free_percpu(cfd->pcpu); return 0; } int smpcfd_dying_cpu(unsigned int cpu) { /* * The IPIs for the smp-call-function callbacks queued by other * CPUs might arrive late, either due to hardware latencies or * because this CPU disabled interrupts (inside stop-machine) * before the IPIs were sent. So flush out any pending callbacks * explicitly (without waiting for the IPIs to arrive), to * ensure that the outgoing CPU doesn't go offline with work * still pending. */ flush_smp_call_function_queue(false); irq_work_run(); return 0; } void __init call_function_init(void) { int i; for_each_possible_cpu(i) init_llist_head(&per_cpu(call_single_queue, i)); smpcfd_prepare_cpu(smp_processor_id()); } #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG static DEFINE_STATIC_KEY_FALSE(csdlock_debug_enabled); static DEFINE_STATIC_KEY_FALSE(csdlock_debug_extended); static int __init csdlock_debug(char *str) { unsigned int val = 0; if (str && !strcmp(str, "ext")) { val = 1; static_branch_enable(&csdlock_debug_extended); } else get_option(&str, &val); if (val) static_branch_enable(&csdlock_debug_enabled); return 0; } early_param("csdlock_debug", csdlock_debug); static DEFINE_PER_CPU(call_single_data_t *, cur_csd); static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func); static DEFINE_PER_CPU(void *, cur_csd_info); static DEFINE_PER_CPU(struct cfd_seq_local, cfd_seq_local); #define CSD_LOCK_TIMEOUT (5ULL * NSEC_PER_SEC) static atomic_t csd_bug_count = ATOMIC_INIT(0); static u64 cfd_seq; #define CFD_SEQ(s, d, t, c) \ (union cfd_seq_cnt){ .u.src = s, .u.dst = d, .u.type = t, .u.cnt = c } static u64 cfd_seq_inc(unsigned int src, unsigned int dst, unsigned int type) { union cfd_seq_cnt new, old; new = CFD_SEQ(src, dst, type, 0); do { old.val = READ_ONCE(cfd_seq); new.u.cnt = old.u.cnt + 1; } while (cmpxchg(&cfd_seq, old.val, new.val) != old.val); return old.val; } #define cfd_seq_store(var, src, dst, type) \ do { \ if (static_branch_unlikely(&csdlock_debug_extended)) \ var = cfd_seq_inc(src, dst, type); \ } while (0) /* Record current CSD work for current CPU, NULL to erase. */ static void __csd_lock_record(struct __call_single_data *csd) { if (!csd) { smp_mb(); /* NULL cur_csd after unlock. */ __this_cpu_write(cur_csd, NULL); return; } __this_cpu_write(cur_csd_func, csd->func); __this_cpu_write(cur_csd_info, csd->info); smp_wmb(); /* func and info before csd. */ __this_cpu_write(cur_csd, csd); smp_mb(); /* Update cur_csd before function call. */ /* Or before unlock, as the case may be. */ } static __always_inline void csd_lock_record(struct __call_single_data *csd) { if (static_branch_unlikely(&csdlock_debug_enabled)) __csd_lock_record(csd); } static int csd_lock_wait_getcpu(struct __call_single_data *csd) { unsigned int csd_type; csd_type = CSD_TYPE(csd); if (csd_type == CSD_TYPE_ASYNC || csd_type == CSD_TYPE_SYNC) return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ return -1; } static void cfd_seq_data_add(u64 val, unsigned int src, unsigned int dst, unsigned int type, union cfd_seq_cnt *data, unsigned int *n_data, unsigned int now) { union cfd_seq_cnt new[2]; unsigned int i, j, k; new[0].val = val; new[1] = CFD_SEQ(src, dst, type, new[0].u.cnt + 1); for (i = 0; i < 2; i++) { if (new[i].u.cnt <= now) new[i].u.cnt |= 0x80000000U; for (j = 0; j < *n_data; j++) { if (new[i].u.cnt == data[j].u.cnt) { /* Direct read value trumps generated one. */ if (i == 0) data[j].val = new[i].val; break; } if (new[i].u.cnt < data[j].u.cnt) { for (k = *n_data; k > j; k--) data[k].val = data[k - 1].val; data[j].val = new[i].val; (*n_data)++; break; } } if (j == *n_data) { data[j].val = new[i].val; (*n_data)++; } } } static const char *csd_lock_get_type(unsigned int type) { return (type >= ARRAY_SIZE(seq_type)) ? "?" : seq_type[type]; } static void csd_lock_print_extended(struct __call_single_data *csd, int cpu) { struct cfd_seq_local *seq = &per_cpu(cfd_seq_local, cpu); unsigned int srccpu = csd->node.src; struct call_function_data *cfd = per_cpu_ptr(&cfd_data, srccpu); struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu); unsigned int now; union cfd_seq_cnt data[2 * ARRAY_SIZE(seq_type)]; unsigned int n_data = 0, i; data[0].val = READ_ONCE(cfd_seq); now = data[0].u.cnt; cfd_seq_data_add(pcpu->seq_queue, srccpu, cpu, CFD_SEQ_QUEUE, data, &n_data, now); cfd_seq_data_add(pcpu->seq_ipi, srccpu, cpu, CFD_SEQ_IPI, data, &n_data, now); cfd_seq_data_add(pcpu->seq_noipi, srccpu, cpu, CFD_SEQ_NOIPI, data, &n_data, now); cfd_seq_data_add(per_cpu(cfd_seq_local.ping, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PING, data, &n_data, now); cfd_seq_data_add(per_cpu(cfd_seq_local.pinged, srccpu), srccpu, CFD_SEQ_NOCPU, CFD_SEQ_PINGED, data, &n_data, now); cfd_seq_data_add(seq->idle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_IDLE, data, &n_data, now); cfd_seq_data_add(seq->gotipi, CFD_SEQ_NOCPU, cpu, CFD_SEQ_GOTIPI, data, &n_data, now); cfd_seq_data_add(seq->handle, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HANDLE, data, &n_data, now); cfd_seq_data_add(seq->dequeue, CFD_SEQ_NOCPU, cpu, CFD_SEQ_DEQUEUE, data, &n_data, now); cfd_seq_data_add(seq->hdlend, CFD_SEQ_NOCPU, cpu, CFD_SEQ_HDLEND, data, &n_data, now); for (i = 0; i < n_data; i++) { pr_alert("\tcsd: cnt(%07x): %04x->%04x %s\n", data[i].u.cnt & ~0x80000000U, data[i].u.src, data[i].u.dst, csd_lock_get_type(data[i].u.type)); } pr_alert("\tcsd: cnt now: %07x\n", now); } /* * Complain if too much time spent waiting. Note that only * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * so waiting on other types gets much less information. */ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id) { int cpu = -1; int cpux; bool firsttime; u64 ts2, ts_delta; call_single_data_t *cpu_cur_csd; unsigned int flags = READ_ONCE(csd->node.u_flags); if (!(flags & CSD_FLAG_LOCK)) { if (!unlikely(*bug_id)) return true; cpu = csd_lock_wait_getcpu(csd); pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n", *bug_id, raw_smp_processor_id(), cpu); return true; } ts2 = sched_clock(); ts_delta = ts2 - *ts1; if (likely(ts_delta <= CSD_LOCK_TIMEOUT)) return false; firsttime = !*bug_id; if (firsttime) *bug_id = atomic_inc_return(&csd_bug_count); cpu = csd_lock_wait_getcpu(csd); if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu)) cpux = 0; else cpux = cpu; cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */ pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %llu ns for CPU#%02d %pS(%ps).\n", firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), ts2 - ts0, cpu, csd->func, csd->info); if (cpu_cur_csd && csd != cpu_cur_csd) { pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n", *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)), READ_ONCE(per_cpu(cur_csd_info, cpux))); } else { pr_alert("\tcsd: CSD lock (#%d) %s.\n", *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request"); } if (cpu >= 0) { if (static_branch_unlikely(&csdlock_debug_extended)) csd_lock_print_extended(csd, cpu); if (!trigger_single_cpu_backtrace(cpu)) dump_cpu_task(cpu); if (!cpu_cur_csd) { pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); } } dump_stack(); *ts1 = ts2; return false; } /* * csd_lock/csd_unlock used to serialize access to per-cpu csd resources * * For non-synchronous ipi calls the csd can still be in use by the * previous function call. For multi-cpu calls its even more interesting * as we'll have to ensure no other cpu is observing our csd. */ static void __csd_lock_wait(struct __call_single_data *csd) { int bug_id = 0; u64 ts0, ts1; ts1 = ts0 = sched_clock(); for (;;) { if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id)) break; cpu_relax(); } smp_acquire__after_ctrl_dep(); } static __always_inline void csd_lock_wait(struct __call_single_data *csd) { if (static_branch_unlikely(&csdlock_debug_enabled)) { __csd_lock_wait(csd); return; } smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); } static void __smp_call_single_queue_debug(int cpu, struct llist_node *node) { unsigned int this_cpu = smp_processor_id(); struct cfd_seq_local *seq = this_cpu_ptr(&cfd_seq_local); struct call_function_data *cfd = this_cpu_ptr(&cfd_data); struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu); cfd_seq_store(pcpu->seq_queue, this_cpu, cpu, CFD_SEQ_QUEUE); if (llist_add(node, &per_cpu(call_single_queue, cpu))) { cfd_seq_store(pcpu->seq_ipi, this_cpu, cpu, CFD_SEQ_IPI); cfd_seq_store(seq->ping, this_cpu, cpu, CFD_SEQ_PING); send_call_function_single_ipi(cpu); cfd_seq_store(seq->pinged, this_cpu, cpu, CFD_SEQ_PINGED); } else { cfd_seq_store(pcpu->seq_noipi, this_cpu, cpu, CFD_SEQ_NOIPI); } } #else #define cfd_seq_store(var, src, dst, type) static void csd_lock_record(struct __call_single_data *csd) { } static __always_inline void csd_lock_wait(struct __call_single_data *csd) { smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); } #endif static __always_inline void csd_lock(struct __call_single_data *csd) { csd_lock_wait(csd); csd->node.u_flags |= CSD_FLAG_LOCK; /* * prevent CPU from reordering the above assignment * to ->flags with any subsequent assignments to other * fields of the specified call_single_data_t structure: */ smp_wmb(); } static __always_inline void csd_unlock(struct __call_single_data *csd) { WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); /* * ensure we're all done before releasing data: */ smp_store_release(&csd->node.u_flags, 0); } static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); void __smp_call_single_queue(int cpu, struct llist_node *node) { #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG if (static_branch_unlikely(&csdlock_debug_extended)) { unsigned int type; type = CSD_TYPE(container_of(node, call_single_data_t, node.llist)); if (type == CSD_TYPE_SYNC || type == CSD_TYPE_ASYNC) { __smp_call_single_queue_debug(cpu, node); return; } } #endif /* * The list addition should be visible before sending the IPI * handler locks the list to pull the entry off it because of * normal cache coherency rules implied by spinlocks. * * If IPIs can go out of order to the cache coherency protocol * in an architecture, sufficient synchronisation should be added * to arch code to make it appear to obey cache coherency WRT * locking and barrier primitives. Generic code isn't really * equipped to do the right thing... */ if (llist_add(node, &per_cpu(call_single_queue, cpu))) send_call_function_single_ipi(cpu); } /* * Insert a previously allocated call_single_data_t element