// SPDX-License-Identifier: GPL-2.0-or-later/* * SMP support for ppc. * * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great * deal of code from the sparc and intel versions. * * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> * * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com */ #undef DEBUG #include <linux/kernel.h>#include <linux/export.h>#include <linux/sched/mm.h>#include <linux/sched/task_stack.h>#include <linux/sched/topology.h>#include <linux/smp.h>#include <linux/interrupt.h>#include <linux/delay.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/cache.h>#include <linux/err.h>#include <linux/device.h>#include <linux/cpu.h>#include <linux/notifier.h>#include <linux/topology.h>#include <linux/profile.h>#include <linux/processor.h>#include <linux/random.h>#include <linux/stackprotector.h>#include <linux/pgtable.h>#include <linux/clockchips.h>#include <linux/kexec.h> #include <asm/ptrace.h>#include <linux/atomic.h>#include <asm/irq.h>#include <asm/hw_irq.h>#include <asm/kvm_ppc.h>#include <asm/dbell.h>#include <asm/page.h>#include <asm/smp.h>#include <asm/time.h>#include <asm/machdep.h>#include <asm/mmu_context.h>#include <asm/cputhreads.h>#include <asm/cputable.h>#include <asm/mpic.h>#include <asm/vdso_datapage.h>#ifdef CONFIG_PPC64#include <asm/paca.h>#endif#include <asm/vdso.h>#include <asm/debug.h>#include <asm/cpu_has_feature.h>#include <asm/ftrace.h>#include <asm/kup.h>#include <asm/fadump.h>#include <asm/systemcfg.h> #include <trace/events/ipi.h> #ifdef DEBUG#include <asm/udbg.h>#define DBG(fmt...) udbg_printf(fmt)#else#define DBG(fmt...)#endif #ifdef CONFIG_HOTPLUG_CPU/* State of each CPU during hotplug phases */static DEFINE_PER_CPU(int, cpu_state) = { 0 };#endif struct task_struct *secondary_current;bool has_big_cores __ro_after_init;bool coregroup_enabled __ro_after_init;bool thread_group_shares_l2 __ro_after_init;bool thread_group_shares_l3 __ro_after_init; DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map); EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);EXPORT_PER_CPU_SYMBOL(cpu_core_map);EXPORT_SYMBOL_GPL(has_big_cores); #define MAX_THREAD_LIST_SIZE	8#define THREAD_GROUP_SHARE_L1   1#define THREAD_GROUP_SHARE_L2_L3 2struct thread_groups {	unsigned int property;	unsigned int nr_groups;	unsigned int threads_per_group;	unsigned int thread_list[MAX_THREAD_LIST_SIZE];}; /* Maximum number of properties that groups of threads within a core can share */#define MAX_THREAD_GROUP_PROPERTIES 2 struct thread_groups_list {	unsigned int nr_properties;	struct thread_groups property_tgs[MAX_THREAD_GROUP_PROPERTIES];}; static struct thread_groups_list tgl[NR_CPUS] __initdata;/* * On big-cores system, thread_group_l1_cache_map for each CPU corresponds to * the set its siblings that share the L1-cache. */DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map); /* * On some big-cores system, thread_group_l2_cache_map for each CPU * corresponds to the set its siblings within the core that share the * L2-cache. */DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map); /* * On P10, thread_group_l3_cache_map for each CPU is equal to the * thread_group_l2_cache_map */DEFINE_PER_CPU(cpumask_var_t, thread_group_l3_cache_map); /* SMP operations for this machine */struct smp_ops_t *smp_ops; /* Can't be static due to PowerMac hackery */volatile unsigned int cpu_callin_map[NR_CPUS]; int smt_enabled_at_boot = 1; /* * Returns 1 if the specified cpu should be brought up during boot. * Used to inhibit booting threads if they've been disabled or * limited on the command line */int smp_generic_cpu_bootable(unsigned int nr){	/* Special case - we inhibit secondary thread startup	 * during boot if the user requests it.	 */	if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {		if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)			return 0;		if (smt_enabled_at_boot		    && cpu_thread_in_core(nr) >= smt_enabled_at_boot)			return 0;	} 	return 1;}  #ifdef CONFIG_PPC64int smp_generic_kick_cpu(int nr){	if (nr < 0 || nr >= nr_cpu_ids)		return -EINVAL; 	/*	 * The processor is currently spinning, waiting for the	 * cpu_start field to become non-zero After we set cpu_start,	 * the processor will continue on to secondary_start	 */	if (!paca_ptrs[nr]->cpu_start) {		paca_ptrs[nr]->cpu_start = 1;		smp_mb();		return 0;	} #ifdef CONFIG_HOTPLUG_CPU	/*	 * Ok it's not there, so it might be soft-unplugged, let's	 * try to bring it back	 */	generic_set_cpu_up(nr);	smp_wmb();	smp_send_reschedule(nr);#endif /* CONFIG_HOTPLUG_CPU */ 	return 0;}#endif /* CONFIG_PPC64 */ static irqreturn_t call_function_action(int irq, void *data){	generic_smp_call_function_interrupt();	return IRQ_HANDLED;} static irqreturn_t reschedule_action(int irq, void *data){	scheduler_ipi();	return IRQ_HANDLED;} #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCASTstatic irqreturn_t tick_broadcast_ipi_action(int irq, void *data){	timer_broadcast_interrupt();	return IRQ_HANDLED;}#endif #ifdef CONFIG_NMI_IPIstatic irqreturn_t nmi_ipi_action(int irq, void *data){	smp_handle_nmi_ipi(get_irq_regs());	return IRQ_HANDLED;}#endif static irq_handler_t smp_ipi_action[] = {	[PPC_MSG_CALL_FUNCTION] =  call_function_action,	[PPC_MSG_RESCHEDULE] = reschedule_action,#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST	[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,#endif#ifdef CONFIG_NMI_IPI	[PPC_MSG_NMI_IPI] = nmi_ipi_action,#endif}; /* * The NMI IPI is a fallback and not truly non-maskable. It is simpler * than going through the call function infrastructure, and strongly * serialized, so it is more appropriate for debugging. */const char *smp_ipi_name[] = {	[PPC_MSG_CALL_FUNCTION] =  "ipi call function",	[PPC_MSG_RESCHEDULE] = "ipi reschedule",#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST	[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",#endif#ifdef CONFIG_NMI_IPI	[PPC_MSG_NMI_IPI] = "nmi ipi",#endif}; /* optional function to request ipi, for controllers with >= 4 ipis */int smp_request_message_ipi(int virq, int msg){	int err; 	if (msg < 0 || msg > PPC_MSG_NMI_IPI)		return -EINVAL;#ifndef CONFIG_NMI_IPI	if (msg == PPC_MSG_NMI_IPI)		return 1;#endif 	err = request_irq(virq, smp_ipi_action[msg],			  IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,			  smp_ipi_name[msg], NULL);	WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",		virq, smp_ipi_name[msg], err); 	return err;} #ifdef CONFIG_PPC_SMP_MUXED_IPIstruct cpu_messages {	long messages;			/* current messages */};static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message); void smp_muxed_ipi_set_message(int cpu, int msg){	struct cpu_messages *info = &per_cpu(ipi_message, cpu);	char *message = (char *)&info->messages; 	/*	 * Order previous accesses before accesses in the IPI handler.	 */	smp_mb();	WRITE_ONCE(message[msg], 1);} void smp_muxed_ipi_message_pass(int cpu, int msg){	if (!smp_ops->cause_ipi)		return; 	smp_muxed_ipi_set_message(cpu, msg); 	/*	 * cause_ipi functions are required to include a full barrier	 * before doing whatever causes the IPI.	 */	smp_ops->cause_ipi(cpu);} #ifdef __BIG_ENDIAN__#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))#else#define IPI_MESSAGE(A) (1uL << (8 * (A)))#endif irqreturn_t smp_ipi_demux(void){	mb();	/* order any irq clear */ 	return smp_ipi_demux_relaxed();} /* sync-free variant. Callers should ensure synchronization */irqreturn_t smp_ipi_demux_relaxed(void){	struct cpu_messages *info;	unsigned long all; 	info = this_cpu_ptr(&ipi_message);	do {		all = xchg(&info->messages, 0);#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)		/*		 * Must check for PPC_MSG_RM_HOST_ACTION messages		 * before PPC_MSG_CALL_FUNCTION messages because when		 * a VM is destroyed, we call kick_all_cpus_sync()		 * to ensure that any pending PPC_MSG_RM_HOST_ACTION		 * messages have completed before we free any VCPUs.		 */		if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))			kvmppc_xics_ipi_action();#endif		if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))			generic_smp_call_function_interrupt();		if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))			scheduler_ipi();#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST		if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))			timer_broadcast_interrupt();#endif#ifdef CONFIG_NMI_IPI		if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))			nmi_ipi_action(0, NULL);#endif	} while (READ_ONCE(info->messages)); 	return IRQ_HANDLED;}#endif /* CONFIG_PPC_SMP_MUXED_IPI */ static inline void do_message_pass(int cpu, int msg){	if (smp_ops->message_pass)		smp_ops->message_pass(cpu, msg);#ifdef CONFIG_PPC_SMP_MUXED_IPI	else		smp_muxed_ipi_message_pass(cpu, msg);#endif} void arch_smp_send_reschedule(int cpu){	if (likely(smp_ops))		do_message_pass(cpu, PPC_MSG_RESCHEDULE);}EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); void arch_send_call_function_single_ipi(int cpu){	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);} void arch_send_call_function_ipi_mask(const struct cpumask *mask){	unsigned int cpu; 	for_each_cpu(cpu, mask)		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);} #ifdef CONFIG_NMI_IPI /* * "NMI IPI" system. * * NMI IPIs may not be recoverable, so should not be used as ongoing part of * a running system. They can be used for crash, debug, halt/reboot, etc. * * The IPI call waits with interrupts disabled until all targets enter the * NMI handler, then returns. Subsequent IPIs can be issued before targets * have returned from their handlers, so there is no guarantee about * concurrency or re-entrancy. * * A new NMI can be issued before all targets exit the handler. * * The IPI call may time out without all targets entering the NMI handler. * In that case, there is some logic to recover (and ignore subsequent * NMI interrupts that may eventually be raised), but the platform interrupt * handler may not be able to distinguish this from other exception causes, * which may cause a crash. */ static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);static struct cpumask nmi_ipi_pending_mask;static bool nmi_ipi_busy = false;static void (*nmi_ipi_function)(struct pt_regs *) = NULL; noinstr static void nmi_ipi_lock_start(unsigned long *flags){	raw_local_irq_save(*flags);	hard_irq_disable();	while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {		raw_local_irq_restore(*flags);		spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);		raw_local_irq_save(*flags);		hard_irq_disable();	}} noinstr static void nmi_ipi_lock(void){	while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)		spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0);} noinstr static void nmi_ipi_unlock(void){	smp_mb();	WARN_ON(raw_atomic_read(&__nmi_ipi_lock) != 1);	raw_atomic_set(&__nmi_ipi_lock, 0);} noinstr static void nmi_ipi_unlock_end(unsigned long *flags){	nmi_ipi_unlock();	raw_local_irq_restore(*flags);} /* * Platform NMI handler calls this to ack */noinstr int smp_handle_nmi_ipi(struct pt_regs *regs){	void (*fn)(struct pt_regs *) = NULL;	unsigned long flags;	int me = raw_smp_processor_id();	int ret = 0; 	/*	 * Unexpected NMIs are possible here because the interrupt may not	 * be able to distinguish NMI IPIs from other types of NMIs, or	 * because the caller may have timed out.	 */	nmi_ipi_lock_start(&flags);	if (cpumask_test_cpu(me, &nmi_ipi_pending_mask)) {		cpumask_clear_cpu(me, &nmi_ipi_pending_mask);		fn = READ_ONCE(nmi_ipi_function);		WARN_ON_ONCE(!fn);		ret = 1;	}	nmi_ipi_unlock_end(&flags); 	if (fn)		fn(regs); 	return ret;} static void do_smp_send_nmi_ipi(int cpu, bool safe){	if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))		return; 	if (cpu >= 0) {		do_message_pass(cpu, PPC_MSG_NMI_IPI);	} else {		int c; 		for_each_online_cpu(c) {			if (c == raw_smp_processor_id())				continue;			do_message_pass(c, PPC_MSG_NMI_IPI);		}	}} /* * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS. * - fn is the target callback function. * - delay_us > 0 is the delay before giving up waiting for targets to *   begin executing the handler, == 0 specifies indefinite delay. */static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *),				u64 delay_us, bool safe){	unsigned long flags;

Lines 1–500 of 1828. Load more