// SPDX-License-Identifier: GPL-2.0#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/errno.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/prctl.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/sched/idle.h>#include <linux/sched/debug.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/init.h>#include <linux/export.h>#include <linux/pm.h>#include <linux/tick.h>#include <linux/random.h>#include <linux/user-return-notifier.h>#include <linux/dmi.h>#include <linux/utsname.h>#include <linux/stackprotector.h>#include <linux/cpuidle.h>#include <linux/acpi.h>#include <linux/elf-randomize.h>#include <linux/static_call.h>#include <trace/events/power.h>#include <linux/hw_breakpoint.h>#include <linux/entry-common.h>#include <asm/cpu.h>#include <asm/cpuid/api.h>#include <asm/apic.h>#include <linux/uaccess.h>#include <asm/mwait.h>#include <asm/fpu/api.h>#include <asm/fpu/sched.h>#include <asm/fpu/xstate.h>#include <asm/debugreg.h>#include <asm/nmi.h>#include <asm/tlbflush.h>#include <asm/mce.h>#include <asm/vm86.h>#include <asm/switch_to.h>#include <asm/desc.h>#include <asm/prctl.h>#include <asm/spec-ctrl.h>#include <asm/io_bitmap.h>#include <asm/proto.h>#include <asm/frame.h>#include <asm/unwind.h>#include <asm/tdx.h>#include <asm/mmu_context.h>#include <asm/msr.h>#include <asm/shstk.h> #include "process.h" /* * per-CPU TSS segments. Threads are completely 'soft' on Linux, * no more per-task TSS's. The TSS size is kept cacheline-aligned * so they are allowed to end up in the .data..cacheline_aligned * section. Since TSS's are completely CPU-local, we want them * on exact cacheline boundaries, to eliminate cacheline ping-pong. */__visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = {	.x86_tss = {		/*		 * .sp0 is only used when entering ring 0 from a lower		 * privilege level.  Since the init task never runs anything		 * but ring 0 code, there is no need for a valid value here.		 * Poison it.		 */		.sp0 = (1UL << (BITS_PER_LONG-1)) + 1, #ifdef CONFIG_X86_32		.sp1 = TOP_OF_INIT_STACK, 		.ss0 = __KERNEL_DS,		.ss1 = __KERNEL_CS,#endif		.io_bitmap_base	= IO_BITMAP_OFFSET_INVALID,	 },};EXPORT_PER_CPU_SYMBOL(cpu_tss_rw); DEFINE_PER_CPU(bool, __tss_limit_invalid);EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid); /* * The cache may be in an incoherent state and needs flushing during kexec. * E.g., on SME/TDX platforms, dirty cacheline aliases with and without * encryption bit(s) can coexist and the cache needs to be flushed before * booting to the new kernel to avoid the silent memory corruption due to * dirty cachelines with different encryption property being written back * to the memory. */DEFINE_PER_CPU(bool, cache_state_incoherent); /* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. */int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src){	/* fpu_clone() will initialize the "dst_fpu" memory */	memcpy_and_pad(dst, arch_task_struct_size, src, sizeof(*dst), 0); #ifdef CONFIG_VM86	dst->thread.vm86 = NULL;#endif 	return 0;} #ifdef CONFIG_X86_64void arch_release_task_struct(struct task_struct *tsk){	if (fpu_state_size_dynamic() && !(tsk->flags & (PF_KTHREAD | PF_USER_WORKER)))		fpstate_free(x86_task_fpu(tsk));}#endif /* * Free thread data structures etc.. */void exit_thread(struct task_struct *tsk){	struct thread_struct *t = &tsk->thread; 	if (test_thread_flag(TIF_IO_BITMAP))		io_bitmap_exit(tsk); 	free_vm86(t); 	shstk_free(tsk);	fpu__drop(tsk);} static int set_new_tls(struct task_struct *p, unsigned long tls){	struct user_desc __user *utls = (struct user_desc __user *)tls; 	if (in_ia32_syscall())		return do_set_thread_area(p, -1, utls, 0);	else		return do_set_thread_area_64(p, ARCH_SET_FS, tls);} __visible void ret_from_fork(struct task_struct *prev, struct pt_regs *regs,				     int (*fn)(void *), void *fn_arg){	schedule_tail(prev); 	/* Is this a kernel thread? */	if (unlikely(fn)) {		fn(fn_arg);		/*		 * A kernel thread is allowed to return here after successfully		 * calling kernel_execve().  Exit to userspace to complete the		 * execve() syscall.		 */		regs->ax = 0;	} 	syscall_exit_to_user_mode(regs);} int copy_thread(struct task_struct *p, const struct kernel_clone_args *args){	u64 clone_flags = args->flags;	unsigned long sp = args->stack;	unsigned long tls = args->tls;	struct inactive_task_frame *frame;	struct fork_frame *fork_frame;	struct pt_regs *childregs;	unsigned long new_ssp;	int ret = 0; 	childregs = task_pt_regs(p);	fork_frame = container_of(childregs, struct fork_frame, regs);	frame = &fork_frame->frame; 	frame->bp = encode_frame_pointer(childregs);	frame->ret_addr = (unsigned long) ret_from_fork_asm;	p->thread.sp = (unsigned long) fork_frame;	p->thread.io_bitmap = NULL;	clear_tsk_thread_flag(p, TIF_IO_BITMAP);	p->thread.iopl_warn = 0;	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); #ifdef CONFIG_X86_64	current_save_fsgs();	p->thread.fsindex = current->thread.fsindex;	p->thread.fsbase = current->thread.fsbase;	p->thread.gsindex = current->thread.gsindex;	p->thread.gsbase = current->thread.gsbase; 	savesegment(es, p->thread.es);	savesegment(ds, p->thread.ds); 	if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM)		set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags);#else	p->thread.sp0 = (unsigned long) (childregs + 1);	savesegment(gs, p->thread.gs);	/*	 * Clear all status flags including IF and set fixed bit. 64bit	 * does not have this initialization as the frame does not contain	 * flags. The flags consistency (especially vs. AC) is there	 * ensured via objtool, which lacks 32bit support.	 */	frame->flags = X86_EFLAGS_FIXED;#endif 	/*	 * Allocate a new shadow stack for thread if needed. If shadow stack,	 * is disabled, new_ssp will remain 0, and fpu_clone() will know not to	 * update it.	 */	new_ssp = shstk_alloc_thread_stack(p, clone_flags, args->stack_size);	if (IS_ERR_VALUE(new_ssp))		return PTR_ERR((void *)new_ssp); 	fpu_clone(p, clone_flags, args->fn, new_ssp); 	/* Kernel thread ? */	if (unlikely(p->flags & PF_KTHREAD)) {		p->thread.pkru = pkru_get_init_value();		memset(childregs, 0, sizeof(struct pt_regs));		kthread_frame_init(frame, args->fn, args->fn_arg);		return 0;	} 	/*	 * Clone current's PKRU value from hardware. tsk->thread.pkru	 * is only valid when scheduled out.	 */	p->thread.pkru = read_pkru(); 	frame->bx = 0;	*childregs = *current_pt_regs();	childregs->ax = 0;	if (sp)		childregs->sp = sp; 	if (unlikely(args->fn)) {		/*		 * A user space thread, but it doesn't return to		 * ret_after_fork().		 *		 * In order to indicate that to tools like gdb,		 * we reset the stack and instruction pointers.		 *		 * It does the same kernel frame setup to return to a kernel		 * function that a kernel thread does.		 */		childregs->sp = 0;		childregs->ip = 0;		kthread_frame_init(frame, args->fn, args->fn_arg);		return 0;	} 	/* Set a new TLS for the child thread? */	if (clone_flags & CLONE_SETTLS)		ret = set_new_tls(p, tls); 	if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))		io_bitmap_share(p); 	return ret;} static void pkru_flush_thread(void){	/*	 * If PKRU is enabled the default PKRU value has to be loaded into	 * the hardware right here (similar to context switch).	 */	pkru_write_default();} void flush_thread(void){	struct task_struct *tsk = current; 	flush_ptrace_hw_breakpoint(tsk);	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 	fpu_flush_thread();	pkru_flush_thread();} void disable_TSC(void){	preempt_disable();	if (!test_and_set_thread_flag(TIF_NOTSC))		/*		 * Must flip the CPU state synchronously with		 * TIF_NOTSC in the current running context.		 */		cr4_set_bits(X86_CR4_TSD);	preempt_enable();} static void enable_TSC(void){	preempt_disable();	if (test_and_clear_thread_flag(TIF_NOTSC))		/*		 * Must flip the CPU state synchronously with		 * TIF_NOTSC in the current running context.		 */		cr4_clear_bits(X86_CR4_TSD);	preempt_enable();} int get_tsc_mode(unsigned long adr){	unsigned int val; 	if (test_thread_flag(TIF_NOTSC))		val = PR_TSC_SIGSEGV;	else		val = PR_TSC_ENABLE; 	return put_user(val, (unsigned int __user *)adr);} int set_tsc_mode(unsigned int val){	if (val == PR_TSC_SIGSEGV)		disable_TSC();	else if (val == PR_TSC_ENABLE)		enable_TSC();	else		return -EINVAL; 	return 0;} DEFINE_PER_CPU(u64, msr_misc_features_shadow); static void set_cpuid_faulting(bool on){ 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {		u64 msrval; 		msrval = this_cpu_read(msr_misc_features_shadow);		msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;		msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);		this_cpu_write(msr_misc_features_shadow, msrval);		wrmsrq(MSR_MISC_FEATURES_ENABLES, msrval);	} else if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {		if (on)			msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_CPUID_USER_DIS_BIT);		else			msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_CPUID_USER_DIS_BIT);	}} static void disable_cpuid(void){	preempt_disable();	if (!test_and_set_thread_flag(TIF_NOCPUID)) {		/*		 * Must flip the CPU state synchronously with		 * TIF_NOCPUID in the current running context.		 */		set_cpuid_faulting(true);	}	preempt_enable();} static void enable_cpuid(void){	preempt_disable();	if (test_and_clear_thread_flag(TIF_NOCPUID)) {		/*		 * Must flip the CPU state synchronously with		 * TIF_NOCPUID in the current running context.		 */		set_cpuid_faulting(false);	}	preempt_enable();} static int get_cpuid_mode(void){	return !test_thread_flag(TIF_NOCPUID);} static int set_cpuid_mode(unsigned long cpuid_enabled){	if (!boot_cpu_has(X86_FEATURE_CPUID_FAULT))		return -ENODEV; 	if (cpuid_enabled)		enable_cpuid();	else		disable_cpuid(); 	return 0;} /* * Called immediately after a successful exec. */void arch_setup_new_exec(void){	/* If cpuid was previously disabled for this task, re-enable it. */	if (test_thread_flag(TIF_NOCPUID))		enable_cpuid(); 	/*	 * Don't inherit TIF_SSBD across exec boundary when	 * PR_SPEC_DISABLE_NOEXEC is used.	 */	if (test_thread_flag(TIF_SSBD) &&	    task_spec_ssb_noexec(current)) {		clear_thread_flag(TIF_SSBD);		task_clear_spec_ssb_disable(current);		task_clear_spec_ssb_noexec(current);		speculation_ctrl_update(read_thread_flags());	} 	mm_reset_untag_mask(current->mm);} #ifdef CONFIG_X86_IOPL_IOPERMstatic inline void switch_to_bitmap(unsigned long tifp){	/*	 * Invalidate I/O bitmap if the previous task used it. This prevents	 * any possible leakage of an active I/O bitmap.	 *	 * If the next task has an I/O bitmap it will handle it on exit to	 * user mode.	 */	if (tifp & _TIF_IO_BITMAP)		tss_invalidate_io_bitmap();} static void tss_copy_io_bitmap(struct tss_struct *tss, struct io_bitmap *iobm){	/*	 * Copy at least the byte range of the incoming tasks bitmap which	 * covers the permitted I/O ports.	 *	 * If the previous task which used an I/O bitmap had more bits	 * permitted, then the copy needs to cover those as well so they	 * get turned off.	 */	memcpy(tss->io_bitmap.bitmap, iobm->bitmap,	       max(tss->io_bitmap.prev_max, iobm->max)); 	/*	 * Store the new max and the sequence number of this bitmap	 * and a pointer to the bitmap itself.	 */	tss->io_bitmap.prev_max = iobm->max;	tss->io_bitmap.prev_sequence = iobm->sequence;} /** * native_tss_update_io_bitmap - Update I/O bitmap before exiting to user mode */void native_tss_update_io_bitmap(void){	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);	struct thread_struct *t = &current->thread;	u16 *base = &tss->x86_tss.io_bitmap_base; 	if (!test_thread_flag(TIF_IO_BITMAP)) {		native_tss_invalidate_io_bitmap();		return;	} 	if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) {		*base = IO_BITMAP_OFFSET_VALID_ALL;	} else {		struct io_bitmap *iobm = t->io_bitmap; 		if (WARN_ON_ONCE(!iobm)) {			clear_thread_flag(TIF_IO_BITMAP);			native_tss_invalidate_io_bitmap();			return;		} 		/*		 * Only copy bitmap data when the sequence number differs. The		 * update time is accounted to the incoming task.		 */		if (tss->io_bitmap.prev_sequence != iobm->sequence)			tss_copy_io_bitmap(tss, iobm); 		/* Enable the bitmap */		*base = IO_BITMAP_OFFSET_VALID_MAP;

Lines 1–500 of 1088. Load more