1364 lines · 33.3 KB · C
RawDownload
// SPDX-License-Identifier: GPL-2.0-only/* * SMP initialisation and IPI support * Based on arch/arm/kernel/smp.c * * Copyright (C) 2012 ARM Ltd. */ #include <linux/acpi.h>#include <linux/arm_sdei.h>#include <linux/delay.h>#include <linux/init.h>#include <linux/spinlock.h>#include <linux/sched/mm.h>#include <linux/sched/hotplug.h>#include <linux/sched/task_stack.h>#include <linux/interrupt.h>#include <linux/cache.h>#include <linux/profile.h>#include <linux/errno.h>#include <linux/mm.h>#include <linux/err.h>#include <linux/cpu.h>#include <linux/smp.h>#include <linux/seq_file.h>#include <linux/irq.h>#include <linux/irqchip/arm-gic-v3.h>#include <linux/percpu.h>#include <linux/clockchips.h>#include <linux/completion.h>#include <linux/of.h>#include <linux/irq_work.h>#include <linux/kernel_stat.h>#include <linux/kexec.h>#include <linux/kgdb.h>#include <linux/kprobes.h>#include <linux/kvm_host.h>#include <linux/nmi.h> #include <asm/alternative.h>#include <asm/atomic.h>#include <asm/cacheflush.h>#include <asm/cpu.h>#include <asm/cputype.h>#include <asm/cpu_ops.h>#include <asm/daifflags.h>#include <asm/kvm_mmu.h>#include <asm/mmu_context.h>#include <asm/nmi.h>#include <asm/numa.h>#include <asm/processor.h>#include <asm/smp_plat.h>#include <asm/sections.h>#include <asm/tlbflush.h>#include <asm/ptrace.h>#include <asm/virt.h> #include <trace/events/ipi.h> /* * as from 2.5, kernels no longer have an init_tasks structure * so we need some other way of telling a new secondary core * where to place its SVC stack */struct secondary_data secondary_data;/* Number of CPUs which aren't online, but looping in kernel text. */static int cpus_stuck_in_kernel; static int ipi_irq_base __ro_after_init;static int nr_ipi __ro_after_init = NR_IPI; struct ipi_descs {	struct irq_desc *descs[MAX_IPI];}; static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc); #define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi]) static bool percpu_ipi_descs __ro_after_init; static bool crash_stop; static void ipi_setup(int cpu); #ifdef CONFIG_HOTPLUG_CPUstatic void ipi_teardown(int cpu);static int op_cpu_kill(unsigned int cpu);#elsestatic inline int op_cpu_kill(unsigned int cpu){	return -ENOSYS;}#endif  /* * Boot a secondary CPU, and assign it the specified idle task. * This also gives us the initial stack to use for this CPU. */static int boot_secondary(unsigned int cpu, struct task_struct *idle){	const struct cpu_operations *ops = get_cpu_ops(cpu); 	if (ops->cpu_boot)		return ops->cpu_boot(cpu); 	return -EOPNOTSUPP;} static DECLARE_COMPLETION(cpu_running); int __cpu_up(unsigned int cpu, struct task_struct *idle){	int ret;	long status; 	/*	 * We need to tell the secondary core where to find its stack and the	 * page tables.	 */	secondary_data.task = idle;	update_cpu_boot_status(CPU_MMU_OFF); 	/* Now bring the CPU into our world */	ret = boot_secondary(cpu, idle);	if (ret) {		if (ret != -EPERM)			pr_err("CPU%u: failed to boot: %d\n", cpu, ret);		return ret;	} 	/*	 * CPU was successfully started, wait for it to come online or	 * time out.	 */	wait_for_completion_timeout(&cpu_running,				    msecs_to_jiffies(5000));	if (cpu_online(cpu))		return 0; 	pr_crit("CPU%u: failed to come online\n", cpu);	secondary_data.task = NULL;	status = READ_ONCE(secondary_data.status);	if (status == CPU_MMU_OFF)		status = READ_ONCE(__early_cpu_boot_status); 	switch (status & CPU_BOOT_STATUS_MASK) {	default:		pr_err("CPU%u: failed in unknown state : 0x%lx\n",		       cpu, status);		cpus_stuck_in_kernel++;		break;	case CPU_KILL_ME:		if (!op_cpu_kill(cpu)) {			pr_crit("CPU%u: died during early boot\n", cpu);			break;		}		pr_crit("CPU%u: may not have shut down cleanly\n", cpu);		fallthrough;	case CPU_STUCK_IN_KERNEL:		pr_crit("CPU%u: is stuck in kernel\n", cpu);		if (status & CPU_STUCK_REASON_52_BIT_VA)			pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);		if (status & CPU_STUCK_REASON_NO_GRAN) {			pr_crit("CPU%u: does not support %luK granule\n",				cpu, PAGE_SIZE / SZ_1K);		}		cpus_stuck_in_kernel++;		break;	case CPU_PANIC_KERNEL:		panic("CPU%u detected unsupported configuration\n", cpu);	} 	return -EIO;} static void init_gic_priority_masking(void){	u32 cpuflags; 	if (WARN_ON(!gic_enable_sre()))		return; 	cpuflags = read_sysreg(daif); 	WARN_ON(!(cpuflags & PSR_I_BIT));	WARN_ON(!(cpuflags & PSR_F_BIT)); 	gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);} /* * This is the secondary CPU boot entry.  We're using this CPUs * idle thread stack, but a set of temporary page tables. */asmlinkage notrace void secondary_start_kernel(void){	u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;	struct mm_struct *mm = &init_mm;	const struct cpu_operations *ops;	unsigned int cpu = smp_processor_id(); 	/*	 * All kernel threads share the same mm context; grab a	 * reference and switch to it.	 */	mmgrab(mm);	current->active_mm = mm; 	/*	 * TTBR0 is only used for the identity mapping at this stage. Make it	 * point to zero page to avoid speculatively fetching new entries.	 */	cpu_uninstall_idmap(); 	if (system_uses_irq_prio_masking())		init_gic_priority_masking(); 	rcutree_report_cpu_starting(cpu);	trace_hardirqs_off(); 	/*	 * If the system has established the capabilities, make sure	 * this CPU ticks all of those. If it doesn't, the CPU will	 * fail to come online.	 */	check_local_cpu_capabilities(); 	ops = get_cpu_ops(cpu);	if (ops->cpu_postboot)		ops->cpu_postboot(); 	/*	 * Log the CPU info before it is marked online and might get read.	 */	cpuinfo_store_cpu();	store_cpu_topology(cpu); 	/*	 * Enable GIC and timers.	 */	notify_cpu_starting(cpu); 	ipi_setup(cpu); 	numa_add_cpu(cpu); 	/*	 * OK, now it's safe to let the boot CPU continue.  Wait for	 * the CPU migration code to notice that the CPU is online	 * before we continue.	 */	pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",					 cpu, (unsigned long)mpidr,					 read_cpuid_id());	update_cpu_boot_status(CPU_BOOT_SUCCESS);	set_cpu_online(cpu, true);	complete(&cpu_running); 	/*	 * Secondary CPUs enter the kernel with all DAIF exceptions masked.	 *	 * As with setup_arch() we must unmask Debug and SError exceptions, and	 * as the root irqchip has already been detected and initialized we can	 * unmask IRQ and FIQ at the same time.	 */	local_daif_restore(DAIF_PROCCTX); 	/*	 * OK, it's off to the idle thread for us	 */	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);} #ifdef CONFIG_HOTPLUG_CPUstatic int op_cpu_disable(unsigned int cpu){	const struct cpu_operations *ops = get_cpu_ops(cpu); 	/*	 * If we don't have a cpu_die method, abort before we reach the point	 * of no return. CPU0 may not have an cpu_ops, so test for it.	 */	if (!ops || !ops->cpu_die)		return -EOPNOTSUPP; 	/*	 * We may need to abort a hot unplug for some other mechanism-specific	 * reason.	 */	if (ops->cpu_disable)		return ops->cpu_disable(cpu); 	return 0;} /* * __cpu_disable runs on the processor to be shutdown. */int __cpu_disable(void){	unsigned int cpu = smp_processor_id();	int ret; 	ret = op_cpu_disable(cpu);	if (ret)		return ret; 	remove_cpu_topology(cpu);	numa_remove_cpu(cpu); 	/*	 * Take this CPU offline.  Once we clear this, we can't return,	 * and we must not schedule until we're ready to give up the cpu.	 */	set_cpu_online(cpu, false);	ipi_teardown(cpu); 	/*	 * OK - migrate IRQs away from this CPU	 */	irq_migrate_all_off_this_cpu(); 	return 0;} static int op_cpu_kill(unsigned int cpu){	const struct cpu_operations *ops = get_cpu_ops(cpu); 	/*	 * If we have no means of synchronising with the dying CPU, then assume	 * that it is really dead. We can only wait for an arbitrary length of	 * time and hope that it's dead, so let's skip the wait and just hope.	 */	if (!ops->cpu_kill)		return 0; 	return ops->cpu_kill(cpu);} /* * Called on the thread which is asking for a CPU to be shutdown after the * shutdown completed. */void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu){	int err; 	pr_debug("CPU%u: shutdown\n", cpu); 	/*	 * Now that the dying CPU is beyond the point of no return w.r.t.	 * in-kernel synchronisation, try to get the firmware to help us to	 * verify that it has really left the kernel before we consider	 * clobbering anything it might still be using.	 */	err = op_cpu_kill(cpu);	if (err)		pr_warn("CPU%d may not have shut down cleanly: %d\n", cpu, err);} /* * Called from the idle thread for the CPU which has been shutdown. * */void __noreturn cpu_die(void){	unsigned int cpu = smp_processor_id();	const struct cpu_operations *ops = get_cpu_ops(cpu); 	idle_task_exit(); 	local_daif_mask(); 	/* Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose of */	cpuhp_ap_report_dead(); 	/*	 * Actually shutdown the CPU. This must never fail. The specific hotplug	 * mechanism must perform all required cache maintenance to ensure that	 * no dirty lines are lost in the process of shutting down the CPU.	 */	ops->cpu_die(cpu); 	BUG();}#endif static void __cpu_try_die(int cpu){#ifdef CONFIG_HOTPLUG_CPU	const struct cpu_operations *ops = get_cpu_ops(cpu); 	if (ops && ops->cpu_die)		ops->cpu_die(cpu);#endif} /* * Kill the calling secondary CPU, early in bringup before it is turned * online. */void __noreturn cpu_die_early(void){	int cpu = smp_processor_id(); 	pr_crit("CPU%d: will not boot\n", cpu); 	/* Mark this CPU absent */	set_cpu_present(cpu, 0);	rcutree_report_cpu_dead(); 	if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {		update_cpu_boot_status(CPU_KILL_ME);		__cpu_try_die(cpu);	} 	update_cpu_boot_status(CPU_STUCK_IN_KERNEL); 	cpu_park_loop();} static void __init hyp_mode_check(void){	if (is_hyp_mode_available())		pr_info("CPU: All CPU(s) started at EL2\n");	else if (is_hyp_mode_mismatched())		WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,			   "CPU: CPUs started in inconsistent modes");	else		pr_info("CPU: All CPU(s) started at EL1\n");	if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {		kvm_compute_layout();		kvm_apply_hyp_relocations();	}} void __init smp_cpus_done(unsigned int max_cpus){	pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());	hyp_mode_check();	setup_system_features();	setup_user_features();	mark_linear_text_alias_ro();} void __init smp_prepare_boot_cpu(void){	/*	 * The runtime per-cpu areas have been allocated by	 * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be	 * freed shortly, so we must move over to the runtime per-cpu area.	 */	set_my_cpu_offset(per_cpu_offset(smp_processor_id())); 	cpuinfo_store_boot_cpu();	setup_boot_cpu_features(); 	/* Conditionally switch to GIC PMR for interrupt masking */	if (system_uses_irq_prio_masking())		init_gic_priority_masking(); 	kasan_init_hw_tags();	/* Init percpu seeds for random tags after cpus are set up. */	kasan_init_sw_tags();} /* * Duplicate MPIDRs are a recipe for disaster. Scan all initialized * entries and check for duplicates. If any is found just ignore the * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid * matching valid MPIDR values. */static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid){	unsigned int i; 	for (i = 1; (i < cpu) && (i < NR_CPUS); i++)		if (cpu_logical_map(i) == hwid)			return true;	return false;} /* * Initialize cpu operations for a logical cpu and * set it in the possible mask on success */static int __init smp_cpu_setup(int cpu){	const struct cpu_operations *ops; 	if (init_cpu_ops(cpu))		return -ENODEV; 	ops = get_cpu_ops(cpu);	if (ops->cpu_init(cpu))		return -ENODEV; 

Lines 1–500 of 1364.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t