// SPDX-License-Identifier: GPL-2.0-only/* * Based on arch/arm/kernel/process.c * * Original Copyright (C) 1995 Linus Torvalds * Copyright (C) 1996-2000 Russell King - Converted to ARM. * Copyright (C) 2012 ARM Ltd. */#include <linux/compat.h>#include <linux/efi.h>#include <linux/elf.h>#include <linux/export.h>#include <linux/sched.h>#include <linux/sched/debug.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/kernel.h>#include <linux/mman.h>#include <linux/mm.h>#include <linux/nospec.h>#include <linux/stddef.h>#include <linux/sysctl.h>#include <linux/unistd.h>#include <linux/user.h>#include <linux/delay.h>#include <linux/reboot.h>#include <linux/interrupt.h>#include <linux/init.h>#include <linux/cpumask.h>#include <linux/cpu.h>#include <linux/elfcore.h>#include <linux/pm.h>#include <linux/tick.h>#include <linux/utsname.h>#include <linux/uaccess.h>#include <linux/random.h>#include <linux/hw_breakpoint.h>#include <linux/personality.h>#include <linux/notifier.h>#include <trace/events/power.h>#include <linux/percpu.h>#include <linux/thread_info.h>#include <linux/prctl.h>#include <linux/stacktrace.h> #include <asm/alternative.h>#include <asm/arch_timer.h>#include <asm/compat.h>#include <asm/cpufeature.h>#include <asm/cacheflush.h>#include <asm/exec.h>#include <asm/fpsimd.h>#include <asm/gcs.h>#include <asm/mmu_context.h>#include <asm/mpam.h>#include <asm/mte.h>#include <asm/processor.h>#include <asm/pointer_auth.h>#include <asm/stacktrace.h>#include <asm/switch_to.h>#include <asm/system_misc.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)#include <linux/stackprotector.h>unsigned long __stack_chk_guard __ro_after_init;EXPORT_SYMBOL(__stack_chk_guard);#endif /* * Function pointers to optional machine specific functions */void (*pm_power_off)(void);EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPUvoid __noreturn arch_cpu_idle_dead(void){ cpu_die();}#endif /* * Called by kexec, immediately prior to machine_kexec(). * * This must completely disable all secondary CPUs; simply causing those CPUs * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. */void machine_shutdown(void){ smp_shutdown_nonboot_cpus(reboot_cpu);} /* * Halting simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. */void machine_halt(void){ local_irq_disable(); smp_send_stop(); while (1);} /* * Power-off simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. When the system power is turned off, it will take all CPUs * with it. */void machine_power_off(void){ local_irq_disable(); smp_send_stop(); do_kernel_power_off();} /* * Restart requires that the secondary CPUs stop performing any activity * while the primary CPU resets the system. Systems with multiple CPUs must * provide a HW restart implementation, to ensure that all CPUs reset at once. * This is required so that any code running after reset on the primary CPU * doesn't have to co-ordinate with other CPUs to ensure they aren't still * executing pre-reset code, and using RAM that the primary CPU's code wishes * to use. Implementing such co-ordination would be essentially impossible. */void machine_restart(char *cmd){ /* Disable interrupts first */ local_irq_disable(); smp_send_stop(); /* * UpdateCapsule() depends on the system being reset via * ResetSystem(). */ if (efi_enabled(EFI_RUNTIME_SERVICES)) efi_reboot(reboot_mode, NULL); /* Now call the architecture specific reboot code. */ do_kernel_restart(cmd); /* * Whoops - the architecture was unable to reboot. */ printk("Reboot failed -- System halted\n"); while (1);} #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = strstatic const char *const btypes[] = { bstr(NONE, "--"), bstr( JC, "jc"), bstr( C, "-c"), bstr( J , "j-")};#undef bstr static void print_pstate(struct pt_regs *regs){ u64 pstate = regs->pstate; if (compat_user_mode(regs)) { printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n", pstate, pstate & PSR_AA32_N_BIT ? 'N' : 'n', pstate & PSR_AA32_Z_BIT ? 'Z' : 'z', pstate & PSR_AA32_C_BIT ? 'C' : 'c', pstate & PSR_AA32_V_BIT ? 'V' : 'v', pstate & PSR_AA32_Q_BIT ? 'Q' : 'q', pstate & PSR_AA32_T_BIT ? "T32" : "A32", pstate & PSR_AA32_E_BIT ? "BE" : "LE", pstate & PSR_AA32_A_BIT ? 'A' : 'a', pstate & PSR_AA32_I_BIT ? 'I' : 'i', pstate & PSR_AA32_F_BIT ? 'F' : 'f', pstate & PSR_AA32_DIT_BIT ? '+' : '-', pstate & PSR_AA32_SSBS_BIT ? '+' : '-'); } else { const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', pstate & PSR_C_BIT ? 'C' : 'c', pstate & PSR_V_BIT ? 'V' : 'v', pstate & PSR_D_BIT ? 'D' : 'd', pstate & PSR_A_BIT ? 'A' : 'a', pstate & PSR_I_BIT ? 'I' : 'i', pstate & PSR_F_BIT ? 'F' : 'f', pstate & PSR_PAN_BIT ? '+' : '-', pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); }} void __show_regs(struct pt_regs *regs){ int i, top_reg; u64 lr, sp; if (compat_user_mode(regs)) { lr = regs->compat_lr; sp = regs->compat_sp; top_reg = 12; } else { lr = regs->regs[30]; sp = regs->sp; top_reg = 29; } show_regs_print_info(KERN_DEFAULT); print_pstate(regs); if (!user_mode(regs)) { printk("pc : %pS\n", (void *)regs->pc); printk("lr : %pS\n", (void *)ptrauth_strip_kernel_insn_pac(lr)); } else { printk("pc : %016llx\n", regs->pc); printk("lr : %016llx\n", lr); } printk("sp : %016llx\n", sp); if (system_uses_irq_prio_masking()) printk("pmr: %08x\n", regs->pmr); i = top_reg; while (i >= 0) { printk("x%-2d: %016llx", i, regs->regs[i]); while (i-- % 3) pr_cont(" x%-2d: %016llx", i, regs->regs[i]); pr_cont("\n"); }} void show_regs(struct pt_regs *regs){ __show_regs(regs); dump_backtrace(regs, NULL, KERN_DEFAULT);} static void tls_thread_flush(void){ write_sysreg(0, tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(0, SYS_TPIDR2_EL0); if (is_compat_task()) { current->thread.uw.tp_value = 0; /* * We need to ensure ordering between the shadow state and the * hardware state, so that we don't corrupt the hardware state * with a stale shadow state during context switch. */ barrier(); write_sysreg(0, tpidrro_el0); }} static void flush_tagged_addr_state(void){ if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI)) clear_thread_flag(TIF_TAGGED_ADDR);} static void flush_poe(void){ if (!system_supports_poe()) return; write_sysreg_s(POR_EL0_INIT, SYS_POR_EL0);} #ifdef CONFIG_ARM64_GCS static void flush_gcs(void){ if (!system_supports_gcs()) return; current->thread.gcspr_el0 = 0; current->thread.gcs_base = 0; current->thread.gcs_size = 0; current->thread.gcs_el0_mode = 0; current->thread.gcs_el0_locked = 0; write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1); write_sysreg_s(0, SYS_GCSPR_EL0);} static int copy_thread_gcs(struct task_struct *p, const struct kernel_clone_args *args){ unsigned long gcs; if (!system_supports_gcs()) return 0; p->thread.gcs_base = 0; p->thread.gcs_size = 0; p->thread.gcs_el0_mode = current->thread.gcs_el0_mode; p->thread.gcs_el0_locked = current->thread.gcs_el0_locked; gcs = gcs_alloc_thread_stack(p, args); if (IS_ERR_VALUE(gcs)) return PTR_ERR((void *)gcs); return 0;} #else static void flush_gcs(void) { }static int copy_thread_gcs(struct task_struct *p, const struct kernel_clone_args *args){ return 0;} #endif void flush_thread(void){ fpsimd_flush_thread(); tls_thread_flush(); flush_ptrace_hw_breakpoint(current); flush_tagged_addr_state(); flush_poe(); flush_gcs();} void arch_release_task_struct(struct task_struct *tsk){ fpsimd_release_task(tsk);} int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src){ /* * The current/src task's FPSIMD state may or may not be live, and may * have been altered by ptrace after entry to the kernel. Save the * effective FPSIMD state so that this will be copied into dst. */ fpsimd_save_and_flush_current_state(); fpsimd_sync_from_effective_state(src); *dst = *src; /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. */ dst->thread.fp_type = FP_STATE_FPSIMD; dst->thread.sve_state = NULL; clear_tsk_thread_flag(dst, TIF_SVE); task_smstop_sm(dst); /* * Drop stale reference to src's sme_state and ensure dst has ZA * disabled. * * When necessary, ZA will be inherited later in copy_thread_za(). */ dst->thread.sme_state = NULL; clear_tsk_thread_flag(dst, TIF_SME); dst->thread.svcr &= ~SVCR_ZA_MASK; /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); return 0;} static int copy_thread_za(struct task_struct *dst, struct task_struct *src){ if (!thread_za_enabled(&src->thread)) return 0; dst->thread.sve_state = kzalloc(sve_state_size(src), GFP_KERNEL); if (!dst->thread.sve_state) return -ENOMEM; dst->thread.sme_state = kmemdup(src->thread.sme_state, sme_state_size(src), GFP_KERNEL); if (!dst->thread.sme_state) { kfree(dst->thread.sve_state); dst->thread.sve_state = NULL; return -ENOMEM; } set_tsk_thread_flag(dst, TIF_SME); dst->thread.svcr |= SVCR_ZA_MASK; return 0;} asmlinkage void ret_from_fork(void) asm("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args){ u64 clone_flags = args->flags; unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); int ret; memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); /* * In case p was allocated the same task_struct pointer as some * other recently-exited task, make sure p is disassociated from * any cpu that may have run that now-exited task recently. * Otherwise we could erroneously skip reloading the FPSIMD * registers for p. */ fpsimd_flush_task_state(p); ptrauth_thread_init_kernel(p); if (likely(!args->fn)) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; /* * Read the current TLS pointer from tpidr_el0 as it may be * out-of-sync with the saved value. */ *task_user_tls(p) = read_sysreg(tpidr_el0); if (system_supports_poe()) p->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; else childregs->sp = stack_start; } /* * Due to the AAPCS64 "ZA lazy saving scheme", PSTATE.ZA and * TPIDR2 need to be manipulated as a pair, and either both * need to be inherited or both need to be reset. * * Within a process, child threads must not inherit their * parent's TPIDR2 value or they may clobber their parent's * stack at some later point. * * When a process is fork()'d, the child must inherit ZA and * TPIDR2 from its parent in case there was dormant ZA state. * * Use CLONE_VM to determine when the child will share the * address space with the parent, and cannot safely inherit the * state. */ if (system_supports_sme()) { if (!(clone_flags & CLONE_VM)) { p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); ret = copy_thread_za(p, current); if (ret) return ret; } else { p->thread.tpidr2_el0 = 0; WARN_ON_ONCE(p->thread.svcr & SVCR_ZA_MASK); } } /* * If a TLS pointer was passed to clone, use it for the new * thread. */ if (clone_flags & CLONE_SETTLS) p->thread.uw.tp_value = tls; ret = copy_thread_gcs(p, args); if (ret != 0) return ret; } else { /* * A kthread has no context to ERET to, so ensure any buggy * ERET is treated as an illegal exception return. * * When a user task is created from a kthread, childregs will * be initialized by start_thread() or start_compat_thread(). */ memset(childregs, 0, sizeof(struct pt_regs));Lines 1–500 of 1005. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * Based on arch/arm/kernel/process.c * * Original Copyright (C) 1995 Linus Torvalds * Copyright (C) 1996-2000 Russell King - Converted to ARM. * Copyright (C) 2012 ARM Ltd. */ #include <linux/compat.h> #include <linux/efi.h> #include <linux/elf.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/sched/debug.h> #include <linux/sched/task.h> #include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mman.h> #include <linux/mm.h> #include <linux/nospec.h> #include <linux/stddef.h> #include <linux/sysctl.h> #include <linux/unistd.h> #include <linux/user.h> #include <linux/delay.h> #include <linux/reboot.h> #include <linux/interrupt.h> #include <linux/init.h> #include <linux/cpumask.h> #include <linux/cpu.h> #include <linux/elfcore.h> #include <linux/pm.h> #include <linux/tick.h> #include <linux/utsname.h> #include <linux/uaccess.h> #include <linux/random.h> #include <linux/hw_breakpoint.h> #include <linux/personality.h> #include <linux/notifier.h> #include <trace/events/power.h> #include <linux/percpu.h> #include <linux/thread_info.h> #include <linux/prctl.h> #include <linux/stacktrace.h> #include <asm/alternative.h> #include <asm/arch_timer.h> #include <asm/compat.h> #include <asm/cpufeature.h> #include <asm/cacheflush.h> #include <asm/exec.h> #include <asm/fpsimd.h> #include <asm/gcs.h> #include <asm/mmu_context.h> #include <asm/mpam.h> #include <asm/mte.h> #include <asm/processor.h> #include <asm/pointer_auth.h> #include <asm/stacktrace.h> #include <asm/switch_to.h> #include <asm/system_misc.h> #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> unsigned long __stack_chk_guard __ro_after_init; EXPORT_SYMBOL(__stack_chk_guard); #endif /* * Function pointers to optional machine specific functions */ void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); #ifdef CONFIG_HOTPLUG_CPU void __noreturn arch_cpu_idle_dead(void) { cpu_die(); } #endif /* * Called by kexec, immediately prior to machine_kexec(). * * This must completely disable all secondary CPUs; simply causing those CPUs * to execute e.g. a RAM-based pin loop is not sufficient. This allows the * kexec'd kernel to use any and all RAM as it sees fit, without having to * avoid any code or data used by any SW CPU pin loop. The CPU hotplug * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this. */ void machine_shutdown(void) { smp_shutdown_nonboot_cpus(reboot_cpu); } /* * Halting simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. */ void machine_halt(void) { local_irq_disable(); smp_send_stop(); while (1); } /* * Power-off simply requires that the secondary CPUs stop performing any * activity (executing tasks, handling interrupts). smp_send_stop() * achieves this. When the system power is turned off, it will take all CPUs * with it. */ void machine_power_off(void) { local_irq_disable(); smp_send_stop(); do_kernel_power_off(); } /* * Restart requires that the secondary CPUs stop performing any activity * while the primary CPU resets the system. Systems with multiple CPUs must * provide a HW restart implementation, to ensure that all CPUs reset at once. * This is required so that any code running after reset on the primary CPU * doesn't have to co-ordinate with other CPUs to ensure they aren't still * executing pre-reset code, and using RAM that the primary CPU's code wishes * to use. Implementing such co-ordination would be essentially impossible. */ void machine_restart(char *cmd) { /* Disable interrupts first */ local_irq_disable(); smp_send_stop(); /* * UpdateCapsule() depends on the system being reset via * ResetSystem(). */ if (efi_enabled(EFI_RUNTIME_SERVICES)) efi_reboot(reboot_mode, NULL); /* Now call the architecture specific reboot code. */ do_kernel_restart(cmd); /* * Whoops - the architecture was unable to reboot. */ printk("Reboot failed -- System halted\n"); while (1); } #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str static const char *const btypes[] = { bstr(NONE, "--"), bstr( JC, "jc"), bstr( C, "-c"), bstr( J , "j-") }; #undef bstr static void print_pstate(struct pt_regs *regs) { u64 pstate = regs->pstate; if (compat_user_mode(regs)) { printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c %cDIT %cSSBS)\n", pstate, pstate & PSR_AA32_N_BIT ? 'N' : 'n', pstate & PSR_AA32_Z_BIT ? 'Z' : 'z', pstate & PSR_AA32_C_BIT ? 'C' : 'c', pstate & PSR_AA32_V_BIT ? 'V' : 'v', pstate & PSR_AA32_Q_BIT ? 'Q' : 'q', pstate & PSR_AA32_T_BIT ? "T32" : "A32", pstate & PSR_AA32_E_BIT ? "BE" : "LE", pstate & PSR_AA32_A_BIT ? 'A' : 'a', pstate & PSR_AA32_I_BIT ? 'I' : 'i', pstate & PSR_AA32_F_BIT ? 'F' : 'f', pstate & PSR_AA32_DIT_BIT ? '+' : '-', pstate & PSR_AA32_SSBS_BIT ? '+' : '-'); } else { const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >> PSR_BTYPE_SHIFT]; printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO %cDIT %cSSBS BTYPE=%s)\n", pstate, pstate & PSR_N_BIT ? 'N' : 'n', pstate & PSR_Z_BIT ? 'Z' : 'z', pstate & PSR_C_BIT ? 'C' : 'c', pstate & PSR_V_BIT ? 'V' : 'v', pstate & PSR_D_BIT ? 'D' : 'd', pstate & PSR_A_BIT ? 'A' : 'a', pstate & PSR_I_BIT ? 'I' : 'i', pstate & PSR_F_BIT ? 'F' : 'f', pstate & PSR_PAN_BIT ? '+' : '-', pstate & PSR_UAO_BIT ? '+' : '-', pstate & PSR_TCO_BIT ? '+' : '-', pstate & PSR_DIT_BIT ? '+' : '-', pstate & PSR_SSBS_BIT ? '+' : '-', btype_str); } } void __show_regs(struct pt_regs *regs) { int i, top_reg; u64 lr, sp; if (compat_user_mode(regs)) { lr = regs->compat_lr; sp = regs->compat_sp; top_reg = 12; } else { lr = regs->regs[30]; sp = regs->sp; top_reg = 29; } show_regs_print_info(KERN_DEFAULT); print_pstate(regs); if (!user_mode(regs)) { printk("pc : %pS\n", (void *)regs->pc); printk("lr : %pS\n", (void *)ptrauth_strip_kernel_insn_pac(lr)); } else { printk("pc : %016llx\n", regs->pc); printk("lr : %016llx\n", lr); } printk("sp : %016llx\n", sp); if (system_uses_irq_prio_masking()) printk("pmr: %08x\n", regs->pmr); i = top_reg; while (i >= 0) { printk("x%-2d: %016llx", i, regs->regs[i]); while (i-- % 3) pr_cont(" x%-2d: %016llx", i, regs->regs[i]); pr_cont("\n"); } } void show_regs(struct pt_regs *regs) { __show_regs(regs); dump_backtrace(regs, NULL, KERN_DEFAULT); } static void tls_thread_flush(void) { write_sysreg(0, tpidr_el0); if (system_supports_tpidr2()) write_sysreg_s(0, SYS_TPIDR2_EL0); if (is_compat_task()) { current->thread.uw.tp_value = 0; /* * We need to ensure ordering between the shadow state and the * hardware state, so that we don't corrupt the hardware state * with a stale shadow state during context switch. */ barrier(); write_sysreg(0, tpidrro_el0); } } static void flush_tagged_addr_state(void) { if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI)) clear_thread_flag(TIF_TAGGED_ADDR); } static void flush_poe(void) { if (!system_supports_poe()) return; write_sysreg_s(POR_EL0_INIT, SYS_POR_EL0); } #ifdef CONFIG_ARM64_GCS static void flush_gcs(void) { if (!system_supports_gcs()) return; current->thread.gcspr_el0 = 0; current->thread.gcs_base = 0; current->thread.gcs_size = 0; current->thread.gcs_el0_mode = 0; current->thread.gcs_el0_locked = 0; write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1); write_sysreg_s(0, SYS_GCSPR_EL0); } static int copy_thread_gcs(struct task_struct *p, const struct kernel_clone_args *args) { unsigned long gcs; if (!system_supports_gcs()) return 0; p->thread.gcs_base = 0; p->thread.gcs_size = 0; p->thread.gcs_el0_mode = current->thread.gcs_el0_mode; p->thread.gcs_el0_locked = current->thread.gcs_el0_locked; gcs = gcs_alloc_thread_stack(p, args); if (IS_ERR_VALUE(gcs)) return PTR_ERR((void *)gcs); return 0; } #else static void flush_gcs(void) { } static int copy_thread_gcs(struct task_struct *p, const struct kernel_clone_args *args) { return 0; } #endif void flush_thread(void) { fpsimd_flush_thread(); tls_thread_flush(); flush_ptrace_hw_breakpoint(current); flush_tagged_addr_state(); flush_poe(); flush_gcs(); } void arch_release_task_struct(struct task_struct *tsk) { fpsimd_release_task(tsk); } int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { /* * The current/src task's FPSIMD state may or may not be live, and may * have been altered by ptrace after entry to the kernel. Save the * effective FPSIMD state so that this will be copied into dst. */ fpsimd_save_and_flush_current_state(); fpsimd_sync_from_effective_state(src); *dst = *src; /* * Drop stale reference to src's sve_state and convert dst to * non-streaming FPSIMD mode. */ dst->thread.fp_type = FP_STATE_FPSIMD; dst->thread.sve_state = NULL; clear_tsk_thread_flag(dst, TIF_SVE); task_smstop_sm(dst); /* * Drop stale reference to src's sme_state and ensure dst has ZA * disabled. * * When necessary, ZA will be inherited later in copy_thread_za(). */ dst->thread.sme_state = NULL; clear_tsk_thread_flag(dst, TIF_SME); dst->thread.svcr &= ~SVCR_ZA_MASK; /* clear any pending asynchronous tag fault raised by the parent */ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT); return 0; } static int copy_thread_za(struct task_struct *dst, struct task_struct *src) { if (!thread_za_enabled(&src->thread)) return 0; dst->thread.sve_state = kzalloc(sve_state_size(src), GFP_KERNEL); if (!dst->thread.sve_state) return -ENOMEM; dst->thread.sme_state = kmemdup(src->thread.sme_state, sme_state_size(src), GFP_KERNEL); if (!dst->thread.sme_state) { kfree(dst->thread.sve_state); dst->thread.sve_state = NULL; return -ENOMEM; } set_tsk_thread_flag(dst, TIF_SME); dst->thread.svcr |= SVCR_ZA_MASK; return 0; } asmlinkage void ret_from_fork(void) asm("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { u64 clone_flags = args->flags; unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); int ret; memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); /* * In case p was allocated the same task_struct pointer as some * other recently-exited task, make sure p is disassociated from * any cpu that may have run that now-exited task recently. * Otherwise we could erroneously skip reloading the FPSIMD * registers for p. */ fpsimd_flush_task_state(p); ptrauth_thread_init_kernel(p); if (likely(!args->fn)) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; /* * Read the current TLS pointer from tpidr_el0 as it may be * out-of-sync with the saved value. */ *task_user_tls(p) = read_sysreg(tpidr_el0); if (system_supports_poe()) p->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; else childregs->sp = stack_start; } /* * Due to the AAPCS64 "ZA lazy saving scheme", PSTATE.ZA and * TPIDR2 need to be manipulated as a pair, and either both * need to be inherited or both need to be reset. * * Within a process, child threads must not inherit their * parent's TPIDR2 value or they may clobber their parent's * stack at some later point. * * When a process is fork()'d, the child must inherit ZA and * TPIDR2 from its parent in case there was dormant ZA state. * * Use CLONE_VM to determine when the child will share the * address space with the parent, and cannot safely inherit the * state. */ if (system_supports_sme()) { if (!(clone_flags & CLONE_VM)) { p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); ret = copy_thread_za(p, current); if (ret) return ret; } else { p->thread.tpidr2_el0 = 0; WARN_ON_ONCE(p->thread.svcr & SVCR_ZA_MASK); } } /* * If a TLS pointer was passed to clone, use it for the new * thread. */ if (clone_flags & CLONE_SETTLS) p->thread.uw.tp_value = tls; ret = copy_thread_gcs(p, args); if (ret != 0) return ret; } else { /* * A kthread has no context to ERET to, so ensure any buggy * ERET is treated as an illegal exception return. * * When a user task is created from a kthread, childregs will * be initialized by start_thread() or start_compat_thread(). */ memset(childregs, 0, sizeof(struct pt_regs));