// SPDX-License-Identifier: GPL-2.0-only/* *  linux/arch/arm/kernel/smp.c * *  Copyright (C) 2002 ARM Limited, All Rights Reserved. */#include <linux/module.h>#include <linux/delay.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/sched/mm.h>#include <linux/sched/hotplug.h>#include <linux/sched/task_stack.h>#include <linux/interrupt.h>#include <linux/cache.h>#include <linux/profile.h>#include <linux/errno.h>#include <linux/mm.h>#include <linux/err.h>#include <linux/cpu.h>#include <linux/seq_file.h>#include <linux/irq.h>#include <linux/nmi.h>#include <linux/percpu.h>#include <linux/clockchips.h>#include <linux/completion.h>#include <linux/cpufreq.h>#include <linux/irq_work.h>#include <linux/kernel_stat.h> #include <linux/atomic.h>#include <asm/bugs.h>#include <asm/smp.h>#include <asm/cacheflush.h>#include <asm/cpu.h>#include <asm/cputype.h>#include <asm/exception.h>#include <asm/idmap.h>#include <asm/topology.h>#include <asm/mmu_context.h>#include <asm/procinfo.h>#include <asm/processor.h>#include <asm/sections.h>#include <asm/tlbflush.h>#include <asm/ptrace.h>#include <asm/smp_plat.h>#include <asm/virt.h>#include <asm/mach/arch.h>#include <asm/mpu.h> #include <trace/events/ipi.h> /* * as from 2.5, kernels no longer have an init_tasks structure * so we need some other way of telling a new secondary core * where to place its SVC stack */struct secondary_data secondary_data; enum ipi_msg_type {	IPI_WAKEUP,	IPI_TIMER,	IPI_RESCHEDULE,	IPI_CALL_FUNC,	IPI_CPU_STOP,	IPI_IRQ_WORK,	IPI_COMPLETION,	NR_IPI,	/*	 * CPU_BACKTRACE is special and not included in NR_IPI	 * or tracable with trace_ipi_*	 */	IPI_CPU_BACKTRACE = NR_IPI,	/*	 * SGI8-15 can be reserved by secure firmware, and thus may	 * not be usable by the kernel. Please keep the above limited	 * to at most 8 entries.	 */	MAX_IPI}; static int ipi_irq_base __read_mostly;static int nr_ipi __read_mostly = NR_IPI;static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly; static void ipi_setup(int cpu); static DECLARE_COMPLETION(cpu_running); static struct smp_operations smp_ops __ro_after_init; void __init smp_set_ops(const struct smp_operations *ops){	if (ops)		smp_ops = *ops;}; static unsigned long get_arch_pgd(pgd_t *pgd){#ifdef CONFIG_ARM_LPAE	return __phys_to_pfn(virt_to_phys(pgd));#else	return virt_to_phys(pgd);#endif} #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)static int secondary_biglittle_prepare(unsigned int cpu){	if (!cpu_vtable[cpu])		cpu_vtable[cpu] = kzalloc_obj(*cpu_vtable[cpu]); 	return cpu_vtable[cpu] ? 0 : -ENOMEM;} static void secondary_biglittle_init(void){	init_proc_vtable(lookup_processor(read_cpuid_id())->proc);}#elsestatic int secondary_biglittle_prepare(unsigned int cpu){	return 0;} static void secondary_biglittle_init(void){}#endif int __cpu_up(unsigned int cpu, struct task_struct *idle){	int ret; 	if (!smp_ops.smp_boot_secondary)		return -ENOSYS; 	ret = secondary_biglittle_prepare(cpu);	if (ret)		return ret; 	/*	 * We need to tell the secondary core where to find	 * its stack and the page tables.	 */	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;#ifdef CONFIG_ARM_MPU	secondary_data.mpu_rgn_info = &mpu_rgn_info;#endif #ifdef CONFIG_MMU	secondary_data.pgdir = virt_to_phys(idmap_pgd);	secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir);#endif	secondary_data.task = idle;	sync_cache_w(&secondary_data); 	/*	 * Now bring the CPU into our world.	 */	ret = smp_ops.smp_boot_secondary(cpu, idle);	if (ret == 0) {		/*		 * CPU was successfully started, wait for it		 * to come online or time out.		 */		wait_for_completion_timeout(&cpu_running,						 msecs_to_jiffies(1000)); 		if (!cpu_online(cpu)) {			pr_crit("CPU%u: failed to come online\n", cpu);			ret = -EIO;		}	} else {		pr_err("CPU%u: failed to boot: %d\n", cpu, ret);	}  	memset(&secondary_data, 0, sizeof(secondary_data));	return ret;} /* platform specific SMP operations */void __init smp_init_cpus(void){	if (smp_ops.smp_init_cpus)		smp_ops.smp_init_cpus();} int platform_can_secondary_boot(void){	return !!smp_ops.smp_boot_secondary;} int platform_can_cpu_hotplug(void){#ifdef CONFIG_HOTPLUG_CPU	if (smp_ops.cpu_kill)		return 1;#endif 	return 0;} #ifdef CONFIG_HOTPLUG_CPUstatic int platform_cpu_kill(unsigned int cpu){	if (smp_ops.cpu_kill)		return smp_ops.cpu_kill(cpu);	return 1;} static int platform_cpu_disable(unsigned int cpu){	if (smp_ops.cpu_disable)		return smp_ops.cpu_disable(cpu); 	return 0;} int platform_can_hotplug_cpu(unsigned int cpu){	/* cpu_die must be specified to support hotplug */	if (!smp_ops.cpu_die)		return 0; 	if (smp_ops.cpu_can_disable)		return smp_ops.cpu_can_disable(cpu); 	/*	 * By default, allow disabling all CPUs except the first one,	 * since this is special on a lot of platforms, e.g. because	 * of clock tick interrupts.	 */	return cpu != 0;} static void ipi_teardown(int cpu){	int i; 	if (WARN_ON_ONCE(!ipi_irq_base))		return; 	for (i = 0; i < nr_ipi; i++)		disable_percpu_irq(ipi_irq_base + i);} /* * __cpu_disable runs on the processor to be shutdown. */int __cpu_disable(void){	unsigned int cpu = smp_processor_id();	int ret; 	ret = platform_cpu_disable(cpu);	if (ret)		return ret; #ifdef CONFIG_GENERIC_ARCH_TOPOLOGY	remove_cpu_topology(cpu);#endif 	/*	 * Take this CPU offline.  Once we clear this, we can't return,	 * and we must not schedule until we're ready to give up the cpu.	 */	set_cpu_online(cpu, false);	ipi_teardown(cpu); 	/*	 * OK - migrate IRQs away from this CPU	 */	irq_migrate_all_off_this_cpu(); 	/*	 * Flush user cache and TLB mappings, and then remove this CPU	 * from the vm mask set of all processes.	 *	 * Caches are flushed to the Level of Unification Inner Shareable	 * to write-back dirty lines to unified caches shared by all CPUs.	 */	flush_cache_louis();	local_flush_tlb_all(); 	return 0;} /* * called on the thread which is asking for a CPU to be shutdown after the * shutdown completed. */void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu){	pr_debug("CPU%u: shutdown\n", cpu); 	clear_tasks_mm_cpumask(cpu);	/*	 * platform_cpu_kill() is generally expected to do the powering off	 * and/or cutting of clocks to the dying CPU.  Optionally, this may	 * be done by the CPU which is dying in preference to supporting	 * this call, but that means there is _no_ synchronisation between	 * the requesting CPU and the dying CPU actually losing power.	 */	if (!platform_cpu_kill(cpu))		pr_err("CPU%u: unable to kill\n", cpu);} /* * Called from the idle thread for the CPU which has been shutdown. * * Note that we disable IRQs here, but do not re-enable them * before returning to the caller. This is also the behaviour * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */void __noreturn arch_cpu_idle_dead(void){	unsigned int cpu = smp_processor_id(); 	idle_task_exit(); 	local_irq_disable(); 	/*	 * Flush the data out of the L1 cache for this CPU.  This must be	 * before the completion to ensure that data is safely written out	 * before platform_cpu_kill() gets called - which may disable	 * *this* CPU and power down its cache.	 */	flush_cache_louis(); 	/*	 * Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose	 * of. Once this returns, power and/or clocks can be removed at	 * any point from this CPU and its cache by platform_cpu_kill().	 */	cpuhp_ap_report_dead(); 	/*	 * Ensure that the cache lines associated with that completion are	 * written out.  This covers the case where _this_ CPU is doing the	 * powering down, to ensure that the completion is visible to the	 * CPU waiting for this one.	 */	flush_cache_louis(); 	/*	 * The actual CPU shutdown procedure is at least platform (if not	 * CPU) specific.  This may remove power, or it may simply spin.	 *	 * Platforms are generally expected *NOT* to return from this call,	 * although there are some which do because they have no way to	 * power down the CPU.  These platforms are the _only_ reason we	 * have a return path which uses the fragment of assembly below.	 *	 * The return path should not be used for platforms which can	 * power off the CPU.	 */	if (smp_ops.cpu_die)		smp_ops.cpu_die(cpu); 	pr_warn("CPU%u: smp_ops.cpu_die() returned, trying to resuscitate\n",		cpu); 	/*	 * Do not return to the idle loop - jump back to the secondary	 * cpu initialisation.  There's some initialisation which needs	 * to be repeated to undo the effects of taking the CPU offline.	 */	__asm__("mov	sp, %0\n"	"	mov	fp, #0\n"	"	mov	r0, %1\n"	"	b	secondary_start_kernel"		:		: "r" (task_stack_page(current) + THREAD_SIZE - 8),		  "r" (current)		: "r0"); 	unreachable();}#endif /* CONFIG_HOTPLUG_CPU */ /* * Called by both boot and secondaries to move global data into * per-processor storage. */static void smp_store_cpu_info(unsigned int cpuid){	struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); 	cpu_info->loops_per_jiffy = loops_per_jiffy;	cpu_info->cpuid = read_cpuid_id(); 	store_cpu_topology(cpuid);	check_cpu_icache_size(cpuid);} static void set_current(struct task_struct *cur){	/* Set TPIDRURO */	asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");} /* * This is the secondary CPU boot entry.  We're using this CPUs * idle thread stack, but a set of temporary page tables. */asmlinkage void secondary_start_kernel(struct task_struct *task){	struct mm_struct *mm = &init_mm;	unsigned int cpu; 	set_current(task); 	secondary_biglittle_init(); 	/*	 * The identity mapping is uncached (strongly ordered), so	 * switch away from it before attempting any exclusive accesses.	 */	cpu_switch_mm(mm->pgd, mm);	local_flush_bp_all();	enter_lazy_tlb(mm, current);	local_flush_tlb_all(); 	/*	 * All kernel threads share the same mm context; grab a	 * reference and switch to it.	 */	cpu = smp_processor_id();	mmgrab(mm);	current->active_mm = mm;	cpumask_set_cpu(cpu, mm_cpumask(mm)); 	cpu_init(); #ifndef CONFIG_MMU	setup_vectors_base();#endif	pr_debug("CPU%u: Booted secondary processor\n", cpu); 	trace_hardirqs_off(); 	/*	 * Give the platform a chance to do its own initialisation.	 */	if (smp_ops.smp_secondary_init)		smp_ops.smp_secondary_init(cpu); 	notify_cpu_starting(cpu); 	ipi_setup(cpu); 	calibrate_delay(); 	smp_store_cpu_info(cpu); 	/*	 * OK, now it's safe to let the boot CPU continue.  Wait for	 * the CPU migration code to notice that the CPU is online	 * before we continue - which happens after __cpu_up returns.	 */	set_cpu_online(cpu, true); 	check_other_bugs(); 	complete(&cpu_running); 	local_irq_enable();	local_fiq_enable();	local_abt_enable(); 	/*	 * OK, it's off to the idle thread for us	 */	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);} void __init smp_cpus_done(unsigned int max_cpus){	int cpu;	unsigned long bogosum = 0; 	for_each_online_cpu(cpu)		bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy; 	printk(KERN_INFO "SMP: Total of %d processors activated "	       "(%lu.%02lu BogoMIPS).\n",	       num_online_cpus(),	       bogosum / (500000/HZ),	       (bogosum / (5000/HZ)) % 100); 	hyp_mode_check();} void __init smp_prepare_boot_cpu(void){	set_my_cpu_offset(per_cpu_offset(smp_processor_id()));

Lines 1–500 of 852. Load more