// SPDX-License-Identifier: GPL-2.0-only/* *  Copyright (C) 1995  Linus Torvalds * *  Pentium III FXSR, SSE support *	Gareth Hughes <gareth@valinux.com>, May 2000 * *  X86-64 port *	Andi Kleen. * *	CPU hotplug support - ashok.raj@intel.com */ /* * This file handles the architecture-dependent parts of process handling.. */ #include <linux/cpu.h>#include <linux/errno.h>#include <linux/sched.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/fs.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/elfcore.h>#include <linux/smp.h>#include <linux/slab.h>#include <linux/user.h>#include <linux/interrupt.h>#include <linux/delay.h>#include <linux/export.h>#include <linux/ptrace.h>#include <linux/notifier.h>#include <linux/kprobes.h>#include <linux/kdebug.h>#include <linux/prctl.h>#include <linux/uaccess.h>#include <linux/io.h>#include <linux/ftrace.h>#include <linux/syscalls.h> #include <asm/processor.h>#include <asm/fpu/internal.h>#include <asm/mmu_context.h>#include <asm/prctl.h>#include <asm/desc.h>#include <asm/proto.h>#include <asm/ia32.h>#include <asm/debugreg.h>#include <asm/switch_to.h>#include <asm/xen/hypervisor.h>#include <asm/vdso.h>#include <asm/resctrl.h>#include <asm/unistd.h>#include <asm/fsgsbase.h>#ifdef CONFIG_IA32_EMULATION/* Not included via unistd.h */#include <asm/unistd_32_ia32.h>#endif #include "process.h" /* Prints also some state that isn't saved in the pt_regs */void __show_regs(struct pt_regs *regs, enum show_regs_mode mode,		 const char *log_lvl){	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;	unsigned long d0, d1, d2, d3, d6, d7;	unsigned int fsindex, gsindex;	unsigned int ds, es; 	show_iret_regs(regs, log_lvl); 	if (regs->orig_ax != -1)		pr_cont(" ORIG_RAX: %016lx\n", regs->orig_ax);	else		pr_cont("\n"); 	printk("%sRAX: %016lx RBX: %016lx RCX: %016lx\n",	       log_lvl, regs->ax, regs->bx, regs->cx);	printk("%sRDX: %016lx RSI: %016lx RDI: %016lx\n",	       log_lvl, regs->dx, regs->si, regs->di);	printk("%sRBP: %016lx R08: %016lx R09: %016lx\n",	       log_lvl, regs->bp, regs->r8, regs->r9);	printk("%sR10: %016lx R11: %016lx R12: %016lx\n",	       log_lvl, regs->r10, regs->r11, regs->r12);	printk("%sR13: %016lx R14: %016lx R15: %016lx\n",	       log_lvl, regs->r13, regs->r14, regs->r15); 	if (mode == SHOW_REGS_SHORT)		return; 	if (mode == SHOW_REGS_USER) {		rdmsrl(MSR_FS_BASE, fs);		rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);		printk("%sFS:  %016lx GS:  %016lx\n",		       log_lvl, fs, shadowgs);		return;	} 	asm("movl %%ds,%0" : "=r" (ds));	asm("movl %%es,%0" : "=r" (es));	asm("movl %%fs,%0" : "=r" (fsindex));	asm("movl %%gs,%0" : "=r" (gsindex)); 	rdmsrl(MSR_FS_BASE, fs);	rdmsrl(MSR_GS_BASE, gs);	rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); 	cr0 = read_cr0();	cr2 = read_cr2();	cr3 = __read_cr3();	cr4 = __read_cr4(); 	printk("%sFS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",	       log_lvl, fs, fsindex, gs, gsindex, shadowgs);	printk("%sCS:  %04lx DS: %04x ES: %04x CR0: %016lx\n",		log_lvl, regs->cs, ds, es, cr0);	printk("%sCR2: %016lx CR3: %016lx CR4: %016lx\n",		log_lvl, cr2, cr3, cr4); 	get_debugreg(d0, 0);	get_debugreg(d1, 1);	get_debugreg(d2, 2);	get_debugreg(d3, 3);	get_debugreg(d6, 6);	get_debugreg(d7, 7); 	/* Only print out debug registers if they are in their non-default state. */	if (!((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&	    (d6 == DR6_RESERVED) && (d7 == 0x400))) {		printk("%sDR0: %016lx DR1: %016lx DR2: %016lx\n",		       log_lvl, d0, d1, d2);		printk("%sDR3: %016lx DR6: %016lx DR7: %016lx\n",		       log_lvl, d3, d6, d7);	} 	if (boot_cpu_has(X86_FEATURE_OSPKE))		printk("%sPKRU: %08x\n", log_lvl, read_pkru());} void release_thread(struct task_struct *dead_task){	WARN_ON(dead_task->mm);} enum which_selector {	FS,	GS}; /* * Out of line to be protected from kprobes and tracing. If this would be * traced or probed than any access to a per CPU variable happens with * the wrong GS. * * It is not used on Xen paravirt. When paravirt support is needed, it * needs to be renamed with native_ prefix. */static noinstr unsigned long __rdgsbase_inactive(void){	unsigned long gsbase; 	lockdep_assert_irqs_disabled(); 	if (!static_cpu_has(X86_FEATURE_XENPV)) {		native_swapgs();		gsbase = rdgsbase();		native_swapgs();	} else {		instrumentation_begin();		rdmsrl(MSR_KERNEL_GS_BASE, gsbase);		instrumentation_end();	} 	return gsbase;} /* * Out of line to be protected from kprobes and tracing. If this would be * traced or probed than any access to a per CPU variable happens with * the wrong GS. * * It is not used on Xen paravirt. When paravirt support is needed, it * needs to be renamed with native_ prefix. */static noinstr void __wrgsbase_inactive(unsigned long gsbase){	lockdep_assert_irqs_disabled(); 	if (!static_cpu_has(X86_FEATURE_XENPV)) {		native_swapgs();		wrgsbase(gsbase);		native_swapgs();	} else {		instrumentation_begin();		wrmsrl(MSR_KERNEL_GS_BASE, gsbase);		instrumentation_end();	}} /* * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are * not available.  The goal is to be reasonably fast on non-FSGSBASE systems. * It's forcibly inlined because it'll generate better code and this function * is hot. */static __always_inline void save_base_legacy(struct task_struct *prev_p,					     unsigned short selector,					     enum which_selector which){	if (likely(selector == 0)) {		/*		 * On Intel (without X86_BUG_NULL_SEG), the segment base could		 * be the pre-existing saved base or it could be zero.  On AMD		 * (with X86_BUG_NULL_SEG), the segment base could be almost		 * anything.		 *		 * This branch is very hot (it's hit twice on almost every		 * context switch between 64-bit programs), and avoiding		 * the RDMSR helps a lot, so we just assume that whatever		 * value is already saved is correct.  This matches historical		 * Linux behavior, so it won't break existing applications.		 *		 * To avoid leaking state, on non-X86_BUG_NULL_SEG CPUs, if we		 * report that the base is zero, it needs to actually be zero:		 * see the corresponding logic in load_seg_legacy.		 */	} else {		/*		 * If the selector is 1, 2, or 3, then the base is zero on		 * !X86_BUG_NULL_SEG CPUs and could be anything on		 * X86_BUG_NULL_SEG CPUs.  In the latter case, Linux		 * has never attempted to preserve the base across context		 * switches.		 *		 * If selector > 3, then it refers to a real segment, and		 * saving the base isn't necessary.		 */		if (which == FS)			prev_p->thread.fsbase = 0;		else			prev_p->thread.gsbase = 0;	}} static __always_inline void save_fsgs(struct task_struct *task){	savesegment(fs, task->thread.fsindex);	savesegment(gs, task->thread.gsindex);	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {		/*		 * If FSGSBASE is enabled, we can't make any useful guesses		 * about the base, and user code expects us to save the current		 * value.  Fortunately, reading the base directly is efficient.		 */		task->thread.fsbase = rdfsbase();		task->thread.gsbase = __rdgsbase_inactive();	} else {		save_base_legacy(task, task->thread.fsindex, FS);		save_base_legacy(task, task->thread.gsindex, GS);	}} /* * While a process is running,current->thread.fsbase and current->thread.gsbase * may not match the corresponding CPU registers (see save_base_legacy()). */void current_save_fsgs(void){	unsigned long flags; 	/* Interrupts need to be off for FSGSBASE */	local_irq_save(flags);	save_fsgs(current);	local_irq_restore(flags);}#if IS_ENABLED(CONFIG_KVM)EXPORT_SYMBOL_GPL(current_save_fsgs);#endif static __always_inline void loadseg(enum which_selector which,				    unsigned short sel){	if (which == FS)		loadsegment(fs, sel);	else		load_gs_index(sel);} static __always_inline void load_seg_legacy(unsigned short prev_index,					    unsigned long prev_base,					    unsigned short next_index,					    unsigned long next_base,					    enum which_selector which){	if (likely(next_index <= 3)) {		/*		 * The next task is using 64-bit TLS, is not using this		 * segment at all, or is having fun with arcane CPU features.		 */		if (next_base == 0) {			/*			 * Nasty case: on AMD CPUs, we need to forcibly zero			 * the base.			 */			if (static_cpu_has_bug(X86_BUG_NULL_SEG)) {				loadseg(which, __USER_DS);				loadseg(which, next_index);			} else {				/*				 * We could try to exhaustively detect cases				 * under which we can skip the segment load,				 * but there's really only one case that matters				 * for performance: if both the previous and				 * next states are fully zeroed, we can skip				 * the load.				 *				 * (This assumes that prev_base == 0 has no				 * false positives.  This is the case on				 * Intel-style CPUs.)				 */				if (likely(prev_index | next_index | prev_base))					loadseg(which, next_index);			}		} else {			if (prev_index != next_index)				loadseg(which, next_index);			wrmsrl(which == FS ? MSR_FS_BASE : MSR_KERNEL_GS_BASE,			       next_base);		}	} else {		/*		 * The next task is using a real segment.  Loading the selector		 * is sufficient.		 */		loadseg(which, next_index);	}} static __always_inline void x86_fsgsbase_load(struct thread_struct *prev,					      struct thread_struct *next){	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {		/* Update the FS and GS selectors if they could have changed. */		if (unlikely(prev->fsindex || next->fsindex))			loadseg(FS, next->fsindex);		if (unlikely(prev->gsindex || next->gsindex))			loadseg(GS, next->gsindex); 		/* Update the bases. */		wrfsbase(next->fsbase);		__wrgsbase_inactive(next->gsbase);	} else {		load_seg_legacy(prev->fsindex, prev->fsbase,				next->fsindex, next->fsbase, FS);		load_seg_legacy(prev->gsindex, prev->gsbase,				next->gsindex, next->gsbase, GS);	}} unsigned long x86_fsgsbase_read_task(struct task_struct *task,				     unsigned short selector){	unsigned short idx = selector >> 3;	unsigned long base; 	if (likely((selector & SEGMENT_TI_MASK) == 0)) {		if (unlikely(idx >= GDT_ENTRIES))			return 0; 		/*		 * There are no user segments in the GDT with nonzero bases		 * other than the TLS segments.		 */		if (idx < GDT_ENTRY_TLS_MIN || idx > GDT_ENTRY_TLS_MAX)			return 0; 		idx -= GDT_ENTRY_TLS_MIN;		base = get_desc_base(&task->thread.tls_array[idx]);	} else {#ifdef CONFIG_MODIFY_LDT_SYSCALL		struct ldt_struct *ldt; 		/*		 * If performance here mattered, we could protect the LDT		 * with RCU.  This is a slow path, though, so we can just		 * take the mutex.		 */		mutex_lock(&task->mm->context.lock);		ldt = task->mm->context.ldt;		if (unlikely(!ldt || idx >= ldt->nr_entries))			base = 0;		else			base = get_desc_base(ldt->entries + idx);		mutex_unlock(&task->mm->context.lock);#else		base = 0;#endif	} 	return base;} unsigned long x86_gsbase_read_cpu_inactive(void){	unsigned long gsbase; 	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {		unsigned long flags; 		local_irq_save(flags);		gsbase = __rdgsbase_inactive();		local_irq_restore(flags);	} else {		rdmsrl(MSR_KERNEL_GS_BASE, gsbase);	} 	return gsbase;} void x86_gsbase_write_cpu_inactive(unsigned long gsbase){	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {		unsigned long flags; 		local_irq_save(flags);		__wrgsbase_inactive(gsbase);		local_irq_restore(flags);	} else {		wrmsrl(MSR_KERNEL_GS_BASE, gsbase);	}} unsigned long x86_fsbase_read_task(struct task_struct *task){	unsigned long fsbase; 	if (task == current)		fsbase = x86_fsbase_read_cpu();	else if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||		 (task->thread.fsindex == 0))		fsbase = task->thread.fsbase;	else		fsbase = x86_fsgsbase_read_task(task, task->thread.fsindex); 	return fsbase;} unsigned long x86_gsbase_read_task(struct task_struct *task){	unsigned long gsbase; 	if (task == current)		gsbase = x86_gsbase_read_cpu_inactive();	else if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||		 (task->thread.gsindex == 0))		gsbase = task->thread.gsbase;	else		gsbase = x86_fsgsbase_read_task(task, task->thread.gsindex); 	return gsbase;} void x86_fsbase_write_task(struct task_struct *task, unsigned long fsbase){	WARN_ON_ONCE(task == current); 	task->thread.fsbase = fsbase;} void x86_gsbase_write_task(struct task_struct *task, unsigned long gsbase){	WARN_ON_ONCE(task == current); 	task->thread.gsbase = gsbase;} static voidstart_thread_common(struct pt_regs *regs, unsigned long new_ip,		    unsigned long new_sp,		    unsigned int _cs, unsigned int _ss, unsigned int _ds){	WARN_ON_ONCE(regs != current_pt_regs()); 	if (static_cpu_has(X86_BUG_NULL_SEG)) {		/* Loading zero below won't clear the base. */		loadsegment(fs, __USER_DS);		load_gs_index(__USER_DS);	} 	loadsegment(fs, 0);	loadsegment(es, _ds);	loadsegment(ds, _ds);	load_gs_index(0); 	regs->ip		= new_ip;	regs->sp		= new_sp;	regs->cs		= _cs;

Lines 1–500 of 837. Load more