// SPDX-License-Identifier: GPL-2.0-only/* * Based on arch/arm/kernel/process.c * * Original Copyright (C) 1995 Linus Torvalds * Copyright (C) 1996-2000 Russell King - Converted to ARM. * Copyright (C) 2012 ARM Ltd. */#include <linux/compat.h>#include <linux/efi.h>#include <linux/elf.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/sched/debug.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/kernel.h>#include <linux/mman.h>#include <linux/mm.h>#include <linux/nospec.h>#include <linux/stddef.h>#include <linux/sysctl.h>#include <linux/unistd.h>#include <linux/user.h>#include <linux/delay.h>#include <linux/reboot.h>#include <linux/interrupt.h>#include <linux/init.h>#include <linux/cpu.h>#include <linux/elfcore.h>#include <linux/pm.h>#include <linux/tick.h>#include <linux/utsname.h>#include <linux/uaccess.h>#include <linux/random.h>#include <linux/hw_breakpoint.h>#include <linux/personality.h>#include <linux/notifier.h>#include <trace/events/power.h>#include <linux/percpu.h>#include <linux/thread_info.h>#include <linux/prctl.h>#include <linux/stacktrace.h> #include <asm/alternative.h>#include <asm/compat.h>#include <asm/cpufeature.h>#include <asm/cacheflush.h>#include <asm/exec.h>#include <asm/fpsimd.h>#include <asm/mmu_context.h>#include <asm/mte.h>#include <asm/processor.h>#include <asm/pointer_auth.h>#include <asm/stacktrace.h>#include <asm/switch_to.h>#include <asm/system_misc.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)#include <linux/stackprotector.h>unsigned long __stack_chk_guard __ro_after_init;EXPORT_SYMBOL(__stack_chk_guard);#endif /* * Function pointers to optional machine specific functions */void (*pm_power_off)(void);EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPUvoid arch_cpu_idle_dead(void){ cpu_die();}#endif /* * Called by kexec, immediately prior to machine_kexec(). * * This must completely disable all secondary CPUs; simply causing those CPUs * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. */void machine_shutdown(void){ smp_shutdown_nonboot_cpus(reboot_cpu);} /* * Halting simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. */void machine_halt(void){ local_irq_disable(); smp_send_stop(); while (1);} /* * Power-off simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. When the system power is turned off, it will take all CPUs * with it. */void machine_power_off(void){ local_irq_disable(); smp_send_stop(); do_kernel_power_off();} /* * Restart requires that the secondary CPUs stop performing any activity * while the primary CPU resets the system. Systems with multiple CPUs must * provide a HW restart implementation, to ensure that all CPUs reset at once. * This is required so that any code running after reset on the primary CPU * doesn't have to co-ordinate with other CPUs to ensure they aren't still * executing pre-reset code, and using RAM that the primary CPU's code wishes * to use. Implementing such co-ordination would be essentially impossible. */void machine_restart(char *cmd){ /* Disable interrupts first */ local_irq_disable(); smp_send_stop(); /* * UpdateCapsule() depends on the system being reset via * ResetSystem(). */ if (efi_enabled(EFI_RUNTIME_SERVICES)) efi_reboot(reboot_mode, NULL); /* Now call the architecture specific reboot code. */ do_kernel_restart(cmd); /* * Whoops - the architecture was unable to reboot. */ printk("Reboot failed -- System halted\n"); while (1);} #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = strstatic const char *const btypes[] = { bstr(NONE, "--"), bstr( JC, "jc"), bstr( C, "-c"), bstr( J , "j-")};#undef bstr static void print_pstate(struct pt_regs *regs){ u64 pstate = regs->pstate; if (compat_user_mode(regs)) { printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n", pstate, pstate & PSR_AA32_N_BIT ? 'N' : 'n', pstate & PSR_AA32_Z_BIT ? 'Z' : 'z', pstate & PSR_AA32_C_BIT ? 'C' : 'c', pstate & PSR_AA32_V_BIT ? 'V' : 'v', pstate & PSR_AA32_Q_BIT ? 'Q' : 'q', pstate & PSR_AA32_T_BIT ? "T32" : "A32", pstate & PSR_AA32_E_BIT ? "BE" : "LE", pstate & PSR_AA32_A_BIT ? 'A' : 'a', pstate & PSR_AA32_I_BIT ? 'I' : 'i', pstate & PSR_AA32_F_BIT ? 'F' : 'f', pstate & PSR_AA32_DIT_BIT ? '+' : '-', pstate & PSR_AA32_SSBS_BIT ? '+' : '-'); } else { const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', pstate & PSR_C_BIT ? 'C' : 'c', pstate & PSR_V_BIT ? 'V' : 'v', pstate & PSR_D_BIT ? 'D' : 'd', pstate & PSR_A_BIT ? 'A' : 'a', pstate & PSR_I_BIT ? 'I' : 'i', pstate & PSR_F_BIT ? 'F' : 'f', pstate & PSR_PAN_BIT ? '+' : '-', pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); }} void __show_regs(struct pt_regs *regs){ int i, top_reg; u64 lr, sp; if (compat_user_mode(regs)) { lr = regs->compat_lr; sp = regs->compat_sp; top_reg = 12; } else { lr = regs->regs[30]; sp = regs->sp; top_reg = 29; } show_regs_print_info(KERN_DEFAULT); print_pstate(regs); if (!user_mode(regs)) { printk("pc : %pS\n", (void *)regs->pc); printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr)); } else { printk("pc : %016llx\n", regs->pc); printk("lr : %016llx\n", lr); } printk("sp : %016llx\n", sp); if (system_uses_irq_prio_masking()) printk("pmr_save: %08llx\n", regs->pmr_save); i = top_reg; while (i >= 0) { printk("x%-2d: %016llx", i, regs->regs[i]); while (i-- % 3) pr_cont(" x%-2d: %016llx", i, regs->regs[i]); pr_cont("\n"); }} void show_regs(struct pt_regs *regs){ __show_regs(regs); dump_backtrace(regs, NULL, KERN_DEFAULT);} static void tls_thread_flush(void){ write_sysreg(0, tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(0, SYS_TPIDR2_EL0); if (is_compat_task()) { current->thread.uw.tp_value = 0; /* * We need to ensure ordering between the shadow state and the * hardware state, so that we don't corrupt the hardware state * with a stale shadow state during context switch. */ barrier(); write_sysreg(0, tpidrro_el0); }} static void flush_tagged_addr_state(void){ if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI)) clear_thread_flag(TIF_TAGGED_ADDR);} void flush_thread(void){ fpsimd_flush_thread(); tls_thread_flush(); flush_ptrace_hw_breakpoint(current); flush_tagged_addr_state();} void arch_release_task_struct(struct task_struct *tsk){ fpsimd_release_task(tsk);} int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src){ if (current->mm) fpsimd_preserve_current_state(); *dst = *src; /* We rely on the above assignment to initialize dst's thread_flags: */ BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); /* * Detach src's sve_state (if any) from dst so that it does not * get erroneously used or freed prematurely. dst's copies * will be allocated on demand later on if dst uses SVE. * For consistency, also clear TIF_SVE here: this could be done * later in copy_process(), but to avoid tripping up future * maintainers it is best not to leave TIF flags and buffers in * an inconsistent state, even temporarily. */ dst->thread.sve_state = NULL; clear_tsk_thread_flag(dst, TIF_SVE); /* * In the unlikely event that we create a new thread with ZA * enabled we should retain the ZA state so duplicate it here. * This may be shortly freed if we exec() or if CLONE_SETTLS * but it's simpler to do it here. To avoid confusing the rest * of the code ensure that we have a sve_state allocated * whenever za_state is allocated. */ if (thread_za_enabled(&src->thread)) { dst->thread.sve_state = kzalloc(sve_state_size(src), GFP_KERNEL); if (!dst->thread.sve_state) return -ENOMEM; dst->thread.za_state = kmemdup(src->thread.za_state, za_state_size(src), GFP_KERNEL); if (!dst->thread.za_state) { kfree(dst->thread.sve_state); dst->thread.sve_state = NULL; return -ENOMEM; } } else { dst->thread.za_state = NULL; clear_tsk_thread_flag(dst, TIF_SME); } dst->thread.fp_type = FP_STATE_FPSIMD; /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); return 0;} asmlinkage void ret_from_fork(void) asm("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args){ unsigned long clone_flags = args->flags; unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); /* * In case p was allocated the same task_struct pointer as some * other recently-exited task, make sure p is disassociated from * any cpu that may have run that now-exited task recently. * Otherwise we could erroneously skip reloading the FPSIMD * registers for p. */ fpsimd_flush_task_state(p); ptrauth_thread_init_kernel(p); if (likely(!args->fn)) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; /* * Read the current TLS pointer from tpidr_el0 as it may be * out-of-sync with the saved value. */ *task_user_tls(p) = read_sysreg(tpidr_el0); if (system_supports_tpidr2()) p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; else childregs->sp = stack_start; } /* * If a TLS pointer was passed to clone, use it for the new * thread. We also reset TPIDR2 if it's in use. */ if (clone_flags & CLONE_SETTLS) { p->thread.uw.tp_value = tls; p->thread.tpidr2_el0 = 0; } } else { /* * A kthread has no context to ERET to, so ensure any buggy * ERET is treated as an illegal exception return. * * When a user task is created from a kthread, childregs will * be initialized by start_thread() or start_compat_thread(). */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT; p->thread.cpu_context.x19 = (unsigned long)args->fn; p->thread.cpu_context.x20 = (unsigned long)args->fn_arg; } p->thread.cpu_context.pc = (unsigned long)ret_from_fork; p->thread.cpu_context.sp = (unsigned long)childregs; /* * For the benefit of the unwinder, set up childregs->stackframe * as the final frame for the new task. */ p->thread.cpu_context.fp = (unsigned long)childregs->stackframe; ptrace_hw_copy_thread(p); return 0;} void tls_preserve_current_state(void){ *task_user_tls(current) = read_sysreg(tpidr_el0); if (system_supports_tpidr2() && !is_compat_task()) current->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0);} static void tls_thread_switch(struct task_struct *next){ tls_preserve_current_state(); if (is_compat_thread(task_thread_info(next))) write_sysreg(next->thread.uw.tp_value, tpidrro_el0); else if (!arm64_kernel_unmapped_at_el0()) write_sysreg(0, tpidrro_el0); write_sysreg(*task_user_tls(next), tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0);} /* * Force SSBS state on context-switch, since it may be lost after migrating * from a CPU which treats the bit as RES0 in a heterogeneous system. */static void ssbs_thread_switch(struct task_struct *next){ /* * Nothing to do for kernel threads, but 'regs' may be junk * (e.g. idle task) so check the flags and bail early. */ if (unlikely(next->flags & PF_KTHREAD)) return; /* * If all CPUs implement the SSBS extension, then we just need to * context-switch the PSTATE field. */ if (cpus_have_const_cap(ARM64_SSBS)) return; spectre_v4_enable_task_mitigation(next);} /* * We store our current task in sp_el0, which is clobbered by userspace. Keep a * shadow copy so that we can restore this upon entry from userspace. * * This is *only* for exception entry from EL0, and is not valid until we * __switch_to() a user task. */DEFINE_PER_CPU(struct task_struct *, __entry_task); static void entry_task_switch(struct task_struct *next){ __this_cpu_write(__entry_task, next);} /* * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT. * Ensure access is disabled when switching to a 32bit task, ensure * access is enabled when switching to a 64bit task. */static void erratum_1418040_thread_switch(struct task_struct *next){ if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) || !this_cpu_has_cap(ARM64_WORKAROUND_1418040)) return; if (is_compat_thread(task_thread_info(next))) sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0); else sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);} static void erratum_1418040_new_exec(void){ preempt_disable(); erratum_1418040_thread_switch(current); preempt_enable();} /*Lines 1–500 of 759. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Based on arch/arm/kernel/process.c * * Original Copyright (C) 1995 Linus Torvalds * Copyright (C) 1996-2000 Russell King - Converted to ARM. * Copyright (C) 2012 ARM Ltd. */ #include <linux/compat.h> #include <linux/efi.h> #include <linux/elf.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/sched/task.h> #include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mman.h> #include <linux/mm.h> #include <linux/nospec.h> #include <linux/stddef.h> #include <linux/sysctl.h> #include <linux/unistd.h> #include <linux/user.h> #include <linux/delay.h> #include <linux/reboot.h> #include <linux/interrupt.h> #include <linux/init.h> #include <linux/cpu.h> #include <linux/elfcore.h> #include <linux/pm.h> #include <linux/tick.h> #include <linux/utsname.h> #include <linux/uaccess.h> #include <linux/random.h> #include <linux/hw_breakpoint.h> #include <linux/personality.h> #include <linux/notifier.h> #include <trace/events/power.h> #include <linux/percpu.h> #include <linux/thread_info.h> #include <linux/prctl.h> #include <linux/stacktrace.h> #include <asm/alternative.h> #include <asm/compat.h> #include <asm/cpufeature.h> #include <asm/cacheflush.h> #include <asm/exec.h> #include <asm/fpsimd.h> #include <asm/mmu_context.h> #include <asm/mte.h> #include <asm/processor.h> #include <asm/pointer_auth.h> #include <asm/stacktrace.h> #include <asm/switch_to.h> #include <asm/system_misc.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> unsigned long __stack_chk_guard __ro_after_init; EXPORT_SYMBOL(__stack_chk_guard); #endif /* * Function pointers to optional machine specific functions */ void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPU void arch_cpu_idle_dead(void) { cpu_die(); } #endif /* * Called by kexec, immediately prior to machine_kexec(). * * This must completely disable all secondary CPUs; simply causing those CPUs * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. */ void machine_shutdown(void) { smp_shutdown_nonboot_cpus(reboot_cpu); } /* * Halting simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. */ void machine_halt(void) { local_irq_disable(); smp_send_stop(); while (1); } /* * Power-off simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. When the system power is turned off, it will take all CPUs * with it. */ void machine_power_off(void) { local_irq_disable(); smp_send_stop(); do_kernel_power_off(); } /* * Restart requires that the secondary CPUs stop performing any activity * while the primary CPU resets the system. Systems with multiple CPUs must * provide a HW restart implementation, to ensure that all CPUs reset at once. * This is required so that any code running after reset on the primary CPU * doesn't have to co-ordinate with other CPUs to ensure they aren't still * executing pre-reset code, and using RAM that the primary CPU's code wishes * to use. Implementing such co-ordination would be essentially impossible. */ void machine_restart(char *cmd) { /* Disable interrupts first */ local_irq_disable(); smp_send_stop(); /* * UpdateCapsule() depends on the system being reset via * ResetSystem(). */ if (efi_enabled(EFI_RUNTIME_SERVICES)) efi_reboot(reboot_mode, NULL); /* Now call the architecture specific reboot code. */ do_kernel_restart(cmd); /* * Whoops - the architecture was unable to reboot. */ printk("Reboot failed -- System halted\n"); while (1); } #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str static const char *const btypes[] = { bstr(NONE, "--"), bstr( JC, "jc"), bstr( C, "-c"), bstr( J , "j-") }; #undef bstr static void print_pstate(struct pt_regs *regs) { u64 pstate = regs->pstate; if (compat_user_mode(regs)) { printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n", pstate, pstate & PSR_AA32_N_BIT ? 'N' : 'n', pstate & PSR_AA32_Z_BIT ? 'Z' : 'z', pstate & PSR_AA32_C_BIT ? 'C' : 'c', pstate & PSR_AA32_V_BIT ? 'V' : 'v', pstate & PSR_AA32_Q_BIT ? 'Q' : 'q', pstate & PSR_AA32_T_BIT ? "T32" : "A32", pstate & PSR_AA32_E_BIT ? "BE" : "LE", pstate & PSR_AA32_A_BIT ? 'A' : 'a', pstate & PSR_AA32_I_BIT ? 'I' : 'i', pstate & PSR_AA32_F_BIT ? 'F' : 'f', pstate & PSR_AA32_DIT_BIT ? '+' : '-', pstate & PSR_AA32_SSBS_BIT ? '+' : '-'); } else { const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', pstate & PSR_C_BIT ? 'C' : 'c', pstate & PSR_V_BIT ? 'V' : 'v', pstate & PSR_D_BIT ? 'D' : 'd', pstate & PSR_A_BIT ? 'A' : 'a', pstate & PSR_I_BIT ? 'I' : 'i', pstate & PSR_F_BIT ? 'F' : 'f', pstate & PSR_PAN_BIT ? '+' : '-', pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); } } void __show_regs(struct pt_regs *regs) { int i, top_reg; u64 lr, sp; if (compat_user_mode(regs)) { lr = regs->compat_lr; sp = regs->compat_sp; top_reg = 12; } else { lr = regs->regs[30]; sp = regs->sp; top_reg = 29; } show_regs_print_info(KERN_DEFAULT); print_pstate(regs); if (!user_mode(regs)) { printk("pc : %pS\n", (void *)regs->pc); printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr)); } else { printk("pc : %016llx\n", regs->pc); printk("lr : %016llx\n", lr); } printk("sp : %016llx\n", sp); if (system_uses_irq_prio_masking()) printk("pmr_save: %08llx\n", regs->pmr_save); i = top_reg; while (i >= 0) { printk("x%-2d: %016llx", i, regs->regs[i]); while (i-- % 3) pr_cont(" x%-2d: %016llx", i, regs->regs[i]); pr_cont("\n"); } } void show_regs(struct pt_regs *regs) { __show_regs(regs); dump_backtrace(regs, NULL, KERN_DEFAULT); } static void tls_thread_flush(void) { write_sysreg(0, tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(0, SYS_TPIDR2_EL0); if (is_compat_task()) { current->thread.uw.tp_value = 0; /* * We need to ensure ordering between the shadow state and the * hardware state, so that we don't corrupt the hardware state * with a stale shadow state during context switch. */ barrier(); write_sysreg(0, tpidrro_el0); } } static void flush_tagged_addr_state(void) { if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI)) clear_thread_flag(TIF_TAGGED_ADDR); } void flush_thread(void) { fpsimd_flush_thread(); tls_thread_flush(); flush_ptrace_hw_breakpoint(current); flush_tagged_addr_state(); } void arch_release_task_struct(struct task_struct *tsk) { fpsimd_release_task(tsk); } int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { if (current->mm) fpsimd_preserve_current_state(); *dst = *src; /* We rely on the above assignment to initialize dst's thread_flags: */ BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK)); /* * Detach src's sve_state (if any) from dst so that it does not * get erroneously used or freed prematurely. dst's copies * will be allocated on demand later on if dst uses SVE. * For consistency, also clear TIF_SVE here: this could be done * later in copy_process(), but to avoid tripping up future * maintainers it is best not to leave TIF flags and buffers in * an inconsistent state, even temporarily. */ dst->thread.sve_state = NULL; clear_tsk_thread_flag(dst, TIF_SVE); /* * In the unlikely event that we create a new thread with ZA * enabled we should retain the ZA state so duplicate it here. * This may be shortly freed if we exec() or if CLONE_SETTLS * but it's simpler to do it here. To avoid confusing the rest * of the code ensure that we have a sve_state allocated * whenever za_state is allocated. */ if (thread_za_enabled(&src->thread)) { dst->thread.sve_state = kzalloc(sve_state_size(src), GFP_KERNEL); if (!dst->thread.sve_state) return -ENOMEM; dst->thread.za_state = kmemdup(src->thread.za_state, za_state_size(src), GFP_KERNEL); if (!dst->thread.za_state) { kfree(dst->thread.sve_state); dst->thread.sve_state = NULL; return -ENOMEM; } } else { dst->thread.za_state = NULL; clear_tsk_thread_flag(dst, TIF_SME); } dst->thread.fp_type = FP_STATE_FPSIMD; /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); return 0; } asmlinkage void ret_from_fork(void) asm("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { unsigned long clone_flags = args->flags; unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); /* * In case p was allocated the same task_struct pointer as some * other recently-exited task, make sure p is disassociated from * any cpu that may have run that now-exited task recently. * Otherwise we could erroneously skip reloading the FPSIMD * registers for p. */ fpsimd_flush_task_state(p); ptrauth_thread_init_kernel(p); if (likely(!args->fn)) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; /* * Read the current TLS pointer from tpidr_el0 as it may be * out-of-sync with the saved value. */ *task_user_tls(p) = read_sysreg(tpidr_el0); if (system_supports_tpidr2()) p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; else childregs->sp = stack_start; } /* * If a TLS pointer was passed to clone, use it for the new * thread. We also reset TPIDR2 if it's in use. */ if (clone_flags & CLONE_SETTLS) { p->thread.uw.tp_value = tls; p->thread.tpidr2_el0 = 0; } } else { /* * A kthread has no context to ERET to, so ensure any buggy * ERET is treated as an illegal exception return. * * When a user task is created from a kthread, childregs will * be initialized by start_thread() or start_compat_thread(). */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT; p->thread.cpu_context.x19 = (unsigned long)args->fn; p->thread.cpu_context.x20 = (unsigned long)args->fn_arg; } p->thread.cpu_context.pc = (unsigned long)ret_from_fork; p->thread.cpu_context.sp = (unsigned long)childregs; /* * For the benefit of the unwinder, set up childregs->stackframe * as the final frame for the new task. */ p->thread.cpu_context.fp = (unsigned long)childregs->stackframe; ptrace_hw_copy_thread(p); return 0; } void tls_preserve_current_state(void) { *task_user_tls(current) = read_sysreg(tpidr_el0); if (system_supports_tpidr2() && !is_compat_task()) current->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); } static void tls_thread_switch(struct task_struct *next) { tls_preserve_current_state(); if (is_compat_thread(task_thread_info(next))) write_sysreg(next->thread.uw.tp_value, tpidrro_el0); else if (!arm64_kernel_unmapped_at_el0()) write_sysreg(0, tpidrro_el0); write_sysreg(*task_user_tls(next), tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(next->thread.tpidr2_el0, SYS_TPIDR2_EL0); } /* * Force SSBS state on context-switch, since it may be lost after migrating * from a CPU which treats the bit as RES0 in a heterogeneous system. */ static void ssbs_thread_switch(struct task_struct *next) { /* * Nothing to do for kernel threads, but 'regs' may be junk * (e.g. idle task) so check the flags and bail early. */ if (unlikely(next->flags & PF_KTHREAD)) return; /* * If all CPUs implement the SSBS extension, then we just need to * context-switch the PSTATE field. */ if (cpus_have_const_cap(ARM64_SSBS)) return; spectre_v4_enable_task_mitigation(next); } /* * We store our current task in sp_el0, which is clobbered by userspace. Keep a * shadow copy so that we can restore this upon entry from userspace. * * This is *only* for exception entry from EL0, and is not valid until we * __switch_to() a user task. */ DEFINE_PER_CPU(struct task_struct *, __entry_task); static void entry_task_switch(struct task_struct *next) { __this_cpu_write(__entry_task, next); } /* * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT. * Ensure access is disabled when switching to a 32bit task, ensure * access is enabled when switching to a 64bit task. */ static void erratum_1418040_thread_switch(struct task_struct *next) { if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) || !this_cpu_has_cap(ARM64_WORKAROUND_1418040)) return; if (is_compat_thread(task_thread_info(next))) sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0); else sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN); } static void erratum_1418040_new_exec(void) { preempt_disable(); erratum_1418040_thread_switch(current); preempt_enable(); } /*