// SPDX-License-Identifier: GPL-2.0#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/errno.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/smp.h>#include <linux/prctl.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/sched/idle.h>#include <linux/sched/debug.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/init.h>#include <linux/export.h>#include <linux/pm.h>#include <linux/tick.h>#include <linux/random.h>#include <linux/user-return-notifier.h>#include <linux/dmi.h>#include <linux/utsname.h>#include <linux/stackprotector.h>#include <linux/cpuidle.h>#include <linux/acpi.h>#include <linux/elf-randomize.h>#include <trace/events/power.h>#include <linux/hw_breakpoint.h>#include <asm/cpu.h>#include <asm/apic.h>#include <linux/uaccess.h>#include <asm/mwait.h>#include <asm/fpu/internal.h>#include <asm/debugreg.h>#include <asm/nmi.h>#include <asm/tlbflush.h>#include <asm/mce.h>#include <asm/vm86.h>#include <asm/switch_to.h>#include <asm/desc.h>#include <asm/prctl.h>#include <asm/spec-ctrl.h>#include <asm/io_bitmap.h>#include <asm/proto.h>#include <asm/frame.h> #include "process.h" /* * per-CPU TSS segments. Threads are completely 'soft' on Linux, * no more per-task TSS's. The TSS size is kept cacheline-aligned * so they are allowed to end up in the .data..cacheline_aligned * section. Since TSS's are completely CPU-local, we want them * on exact cacheline boundaries, to eliminate cacheline ping-pong. */__visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = {	.x86_tss = {		/*		 * .sp0 is only used when entering ring 0 from a lower		 * privilege level.  Since the init task never runs anything		 * but ring 0 code, there is no need for a valid value here.		 * Poison it.		 */		.sp0 = (1UL << (BITS_PER_LONG-1)) + 1, #ifdef CONFIG_X86_32		.sp1 = TOP_OF_INIT_STACK, 		.ss0 = __KERNEL_DS,		.ss1 = __KERNEL_CS,#endif		.io_bitmap_base	= IO_BITMAP_OFFSET_INVALID,	 },};EXPORT_PER_CPU_SYMBOL(cpu_tss_rw); DEFINE_PER_CPU(bool, __tss_limit_invalid);EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid); /* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. */int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src){	memcpy(dst, src, arch_task_struct_size);#ifdef CONFIG_VM86	dst->thread.vm86 = NULL;#endif 	return fpu__copy(dst, src);} /* * Free thread data structures etc.. */void exit_thread(struct task_struct *tsk){	struct thread_struct *t = &tsk->thread;	struct fpu *fpu = &t->fpu; 	if (test_thread_flag(TIF_IO_BITMAP))		io_bitmap_exit(tsk); 	free_vm86(t); 	fpu__drop(fpu);} static int set_new_tls(struct task_struct *p, unsigned long tls){	struct user_desc __user *utls = (struct user_desc __user *)tls; 	if (in_ia32_syscall())		return do_set_thread_area(p, -1, utls, 0);	else		return do_set_thread_area_64(p, ARCH_SET_FS, tls);} int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg,		struct task_struct *p, unsigned long tls){	struct inactive_task_frame *frame;	struct fork_frame *fork_frame;	struct pt_regs *childregs;	int ret = 0; 	childregs = task_pt_regs(p);	fork_frame = container_of(childregs, struct fork_frame, regs);	frame = &fork_frame->frame; 	frame->bp = encode_frame_pointer(childregs);	frame->ret_addr = (unsigned long) ret_from_fork;	p->thread.sp = (unsigned long) fork_frame;	p->thread.io_bitmap = NULL;	memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); #ifdef CONFIG_X86_64	current_save_fsgs();	p->thread.fsindex = current->thread.fsindex;	p->thread.fsbase = current->thread.fsbase;	p->thread.gsindex = current->thread.gsindex;	p->thread.gsbase = current->thread.gsbase; 	savesegment(es, p->thread.es);	savesegment(ds, p->thread.ds);#else	p->thread.sp0 = (unsigned long) (childregs + 1);	/*	 * Clear all status flags including IF and set fixed bit. 64bit	 * does not have this initialization as the frame does not contain	 * flags. The flags consistency (especially vs. AC) is there	 * ensured via objtool, which lacks 32bit support.	 */	frame->flags = X86_EFLAGS_FIXED;#endif 	/* Kernel thread ? */	if (unlikely(p->flags & PF_KTHREAD)) {		memset(childregs, 0, sizeof(struct pt_regs));		kthread_frame_init(frame, sp, arg);		return 0;	} 	frame->bx = 0;	*childregs = *current_pt_regs();	childregs->ax = 0;	if (sp)		childregs->sp = sp; #ifdef CONFIG_X86_32	task_user_gs(p) = get_user_gs(current_pt_regs());#endif 	if (unlikely(p->flags & PF_IO_WORKER)) {		/*		 * An IO thread is a user space thread, but it doesn't		 * return to ret_after_fork().		 *		 * In order to indicate that to tools like gdb,		 * we reset the stack and instruction pointers.		 *		 * It does the same kernel frame setup to return to a kernel		 * function that a kernel thread does.		 */		childregs->sp = 0;		childregs->ip = 0;		kthread_frame_init(frame, sp, arg);		return 0;	} 	/* Set a new TLS for the child thread? */	if (clone_flags & CLONE_SETTLS)		ret = set_new_tls(p, tls); 	if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))		io_bitmap_share(p); 	return ret;} void flush_thread(void){	struct task_struct *tsk = current; 	flush_ptrace_hw_breakpoint(tsk);	memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 	fpu__clear_all(&tsk->thread.fpu);} void disable_TSC(void){	preempt_disable();	if (!test_and_set_thread_flag(TIF_NOTSC))		/*		 * Must flip the CPU state synchronously with		 * TIF_NOTSC in the current running context.		 */		cr4_set_bits(X86_CR4_TSD);	preempt_enable();} static void enable_TSC(void){	preempt_disable();	if (test_and_clear_thread_flag(TIF_NOTSC))		/*		 * Must flip the CPU state synchronously with		 * TIF_NOTSC in the current running context.		 */		cr4_clear_bits(X86_CR4_TSD);	preempt_enable();} int get_tsc_mode(unsigned long adr){	unsigned int val; 	if (test_thread_flag(TIF_NOTSC))		val = PR_TSC_SIGSEGV;	else		val = PR_TSC_ENABLE; 	return put_user(val, (unsigned int __user *)adr);} int set_tsc_mode(unsigned int val){	if (val == PR_TSC_SIGSEGV)		disable_TSC();	else if (val == PR_TSC_ENABLE)		enable_TSC();	else		return -EINVAL; 	return 0;} DEFINE_PER_CPU(u64, msr_misc_features_shadow); static void set_cpuid_faulting(bool on){	u64 msrval; 	msrval = this_cpu_read(msr_misc_features_shadow);	msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;	msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);	this_cpu_write(msr_misc_features_shadow, msrval);	wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);} static void disable_cpuid(void){	preempt_disable();	if (!test_and_set_thread_flag(TIF_NOCPUID)) {		/*		 * Must flip the CPU state synchronously with		 * TIF_NOCPUID in the current running context.		 */		set_cpuid_faulting(true);	}	preempt_enable();} static void enable_cpuid(void){	preempt_disable();	if (test_and_clear_thread_flag(TIF_NOCPUID)) {		/*		 * Must flip the CPU state synchronously with		 * TIF_NOCPUID in the current running context.		 */		set_cpuid_faulting(false);	}	preempt_enable();} static int get_cpuid_mode(void){	return !test_thread_flag(TIF_NOCPUID);} static int set_cpuid_mode(struct task_struct *task, unsigned long cpuid_enabled){	if (!boot_cpu_has(X86_FEATURE_CPUID_FAULT))		return -ENODEV; 	if (cpuid_enabled)		enable_cpuid();	else		disable_cpuid(); 	return 0;} /* * Called immediately after a successful exec. */void arch_setup_new_exec(void){	/* If cpuid was previously disabled for this task, re-enable it. */	if (test_thread_flag(TIF_NOCPUID))		enable_cpuid(); 	/*	 * Don't inherit TIF_SSBD across exec boundary when	 * PR_SPEC_DISABLE_NOEXEC is used.	 */	if (test_thread_flag(TIF_SSBD) &&	    task_spec_ssb_noexec(current)) {		clear_thread_flag(TIF_SSBD);		task_clear_spec_ssb_disable(current);		task_clear_spec_ssb_noexec(current);		speculation_ctrl_update(task_thread_info(current)->flags);	}} #ifdef CONFIG_X86_IOPL_IOPERMstatic inline void switch_to_bitmap(unsigned long tifp){	/*	 * Invalidate I/O bitmap if the previous task used it. This prevents	 * any possible leakage of an active I/O bitmap.	 *	 * If the next task has an I/O bitmap it will handle it on exit to	 * user mode.	 */	if (tifp & _TIF_IO_BITMAP)		tss_invalidate_io_bitmap();} static void tss_copy_io_bitmap(struct tss_struct *tss, struct io_bitmap *iobm){	/*	 * Copy at least the byte range of the incoming tasks bitmap which	 * covers the permitted I/O ports.	 *	 * If the previous task which used an I/O bitmap had more bits	 * permitted, then the copy needs to cover those as well so they	 * get turned off.	 */	memcpy(tss->io_bitmap.bitmap, iobm->bitmap,	       max(tss->io_bitmap.prev_max, iobm->max)); 	/*	 * Store the new max and the sequence number of this bitmap	 * and a pointer to the bitmap itself.	 */	tss->io_bitmap.prev_max = iobm->max;	tss->io_bitmap.prev_sequence = iobm->sequence;} /** * tss_update_io_bitmap - Update I/O bitmap before exiting to usermode */void native_tss_update_io_bitmap(void){	struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);	struct thread_struct *t = &current->thread;	u16 *base = &tss->x86_tss.io_bitmap_base; 	if (!test_thread_flag(TIF_IO_BITMAP)) {		native_tss_invalidate_io_bitmap();		return;	} 	if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) {		*base = IO_BITMAP_OFFSET_VALID_ALL;	} else {		struct io_bitmap *iobm = t->io_bitmap; 		/*		 * Only copy bitmap data when the sequence number differs. The		 * update time is accounted to the incoming task.		 */		if (tss->io_bitmap.prev_sequence != iobm->sequence)			tss_copy_io_bitmap(tss, iobm); 		/* Enable the bitmap */		*base = IO_BITMAP_OFFSET_VALID_MAP;	} 	/*	 * Make sure that the TSS limit is covering the IO bitmap. It might have	 * been cut down by a VMEXIT to 0x67 which would cause a subsequent I/O	 * access from user space to trigger a #GP because tbe bitmap is outside	 * the TSS limit.	 */	refresh_tss_limit();}#else /* CONFIG_X86_IOPL_IOPERM */static inline void switch_to_bitmap(unsigned long tifp) { }#endif #ifdef CONFIG_SMP struct ssb_state {	struct ssb_state	*shared_state;	raw_spinlock_t		lock;	unsigned int		disable_state;	unsigned long		local_state;}; #define LSTATE_SSB	0 static DEFINE_PER_CPU(struct ssb_state, ssb_state); void speculative_store_bypass_ht_init(void){	struct ssb_state *st = this_cpu_ptr(&ssb_state);	unsigned int this_cpu = smp_processor_id();	unsigned int cpu; 	st->local_state = 0; 	/*	 * Shared state setup happens once on the first bringup	 * of the CPU. It's not destroyed on CPU hotunplug.	 */	if (st->shared_state)		return; 	raw_spin_lock_init(&st->lock); 	/*	 * Go over HT siblings and check whether one of them has set up the	 * shared state pointer already.	 */	for_each_cpu(cpu, topology_sibling_cpumask(this_cpu)) {		if (cpu == this_cpu)			continue; 		if (!per_cpu(ssb_state, cpu).shared_state)			continue; 		/* Link it to the state of the sibling: */		st->shared_state = per_cpu(ssb_state, cpu).shared_state;		return;	} 	/*	 * First HT sibling to come up on the core.  Link shared state of	 * the first HT sibling to itself. The siblings on the same core	 * which come up later will see the shared state pointer and link	 * themselves to the state of this CPU.	 */	st->shared_state = st;} /* * Logic is: First HT sibling enables SSBD for both siblings in the core * and last sibling to disable it, disables it for the whole core. This how * MSR_SPEC_CTRL works in "hardware": * *  CORE_SPEC_CTRL = THREAD0_SPEC_CTRL | THREAD1_SPEC_CTRL */static __always_inline void amd_set_core_ssb_state(unsigned long tifn){	struct ssb_state *st = this_cpu_ptr(&ssb_state);	u64 msr = x86_amd_ls_cfg_base; 	if (!static_cpu_has(X86_FEATURE_ZEN)) {		msr |= ssbd_tif_to_amd_ls_cfg(tifn);		wrmsrl(MSR_AMD64_LS_CFG, msr);		return;	} 	if (tifn & _TIF_SSBD) {		/*		 * Since this can race with prctl(), block reentry on the		 * same CPU.		 */		if (__test_and_set_bit(LSTATE_SSB, &st->local_state))			return; 		msr |= x86_amd_ls_cfg_ssbd_mask; 		raw_spin_lock(&st->shared_state->lock);		/* First sibling enables SSBD: */

Lines 1–500 of 996. Load more