// SPDX-License-Identifier: GPL-2.0-only/* *  Copyright (C) 1995  Linus Torvalds * *  Pentium III FXSR, SSE support *	Gareth Hughes <gareth@valinux.com>, May 2000 * *  X86-64 port *	Andi Kleen. * *	CPU hotplug support - ashok.raj@intel.com */ /* * This file handles the architecture-dependent parts of process handling.. */ #include <linux/cpu.h>#include <linux/errno.h>#include <linux/sched.h>#include <linux/sched/task.h>#include <linux/sched/task_stack.h>#include <linux/fs.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/elfcore.h>#include <linux/smp.h>#include <linux/slab.h>#include <linux/user.h>#include <linux/interrupt.h>#include <linux/delay.h>#include <linux/export.h>#include <linux/kvm_types.h>#include <linux/ptrace.h>#include <linux/notifier.h>#include <linux/kprobes.h>#include <linux/kdebug.h>#include <linux/prctl.h>#include <linux/uaccess.h>#include <linux/io.h>#include <linux/ftrace.h>#include <linux/syscalls.h>#include <linux/iommu.h> #include <asm/processor.h>#include <asm/pkru.h>#include <asm/fpu/sched.h>#include <asm/mmu_context.h>#include <asm/prctl.h>#include <asm/desc.h>#include <asm/proto.h>#include <asm/ia32.h>#include <asm/debugreg.h>#include <asm/switch_to.h>#include <asm/xen/hypervisor.h>#include <asm/vdso.h>#include <asm/resctrl.h>#include <asm/unistd.h>#include <asm/fsgsbase.h>#include <asm/fred.h>#include <asm/msr.h>#ifdef CONFIG_IA32_EMULATION/* Not included via unistd.h */#include <asm/unistd_32_ia32.h>#endif #include "process.h" /* Prints also some state that isn't saved in the pt_regs */void __show_regs(struct pt_regs *regs, enum show_regs_mode mode,		 const char *log_lvl){	unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;	unsigned long d0, d1, d2, d3, d6, d7;	unsigned int fsindex, gsindex;	unsigned int ds, es; 	show_iret_regs(regs, log_lvl); 	if (regs->orig_ax != -1)		pr_cont(" ORIG_RAX: %016lx\n", regs->orig_ax);	else		pr_cont("\n"); 	printk("%sRAX: %016lx RBX: %016lx RCX: %016lx\n",	       log_lvl, regs->ax, regs->bx, regs->cx);	printk("%sRDX: %016lx RSI: %016lx RDI: %016lx\n",	       log_lvl, regs->dx, regs->si, regs->di);	printk("%sRBP: %016lx R08: %016lx R09: %016lx\n",	       log_lvl, regs->bp, regs->r8, regs->r9);	printk("%sR10: %016lx R11: %016lx R12: %016lx\n",	       log_lvl, regs->r10, regs->r11, regs->r12);	printk("%sR13: %016lx R14: %016lx R15: %016lx\n",	       log_lvl, regs->r13, regs->r14, regs->r15); 	if (mode == SHOW_REGS_SHORT)		return; 	if (mode == SHOW_REGS_USER) {		rdmsrq(MSR_FS_BASE, fs);		rdmsrq(MSR_KERNEL_GS_BASE, shadowgs);		printk("%sFS:  %016lx GS:  %016lx\n",		       log_lvl, fs, shadowgs);		return;	} 	savesegment(ds, ds);	savesegment(es, es);	savesegment(fs, fsindex);	savesegment(gs, gsindex); 	rdmsrq(MSR_FS_BASE, fs);	rdmsrq(MSR_GS_BASE, gs);	rdmsrq(MSR_KERNEL_GS_BASE, shadowgs); 	cr0 = read_cr0();	cr2 = read_cr2();	cr3 = __read_cr3();	cr4 = __read_cr4(); 	printk("%sFS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",	       log_lvl, fs, fsindex, gs, gsindex, shadowgs);	printk("%sCS:  %04x DS: %04x ES: %04x CR0: %016lx\n",		log_lvl, regs->cs, ds, es, cr0);	printk("%sCR2: %016lx CR3: %016lx CR4: %016lx\n",		log_lvl, cr2, cr3, cr4); 	get_debugreg(d0, 0);	get_debugreg(d1, 1);	get_debugreg(d2, 2);	get_debugreg(d3, 3);	get_debugreg(d6, 6);	get_debugreg(d7, 7); 	/* Only print out debug registers if they are in their non-default state. */	if (!((d0 == 0) && (d1 == 0) && (d2 == 0) && (d3 == 0) &&	    (d6 == DR6_RESERVED) && (d7 == DR7_FIXED_1))) {		printk("%sDR0: %016lx DR1: %016lx DR2: %016lx\n",		       log_lvl, d0, d1, d2);		printk("%sDR3: %016lx DR6: %016lx DR7: %016lx\n",		       log_lvl, d3, d6, d7);	} 	if (cr4 & X86_CR4_PKE)		printk("%sPKRU: %08x\n", log_lvl, read_pkru());} void release_thread(struct task_struct *dead_task){	WARN_ON(dead_task->mm);} enum which_selector {	FS,	GS}; /* * Out of line to be protected from kprobes and tracing. If this would be * traced or probed than any access to a per CPU variable happens with * the wrong GS. * * It is not used on Xen paravirt. When paravirt support is needed, it * needs to be renamed with native_ prefix. */static noinstr unsigned long __rdgsbase_inactive(void){	unsigned long gsbase; 	lockdep_assert_irqs_disabled(); 	/*	 * SWAPGS is no longer needed thus NOT allowed with FRED because	 * FRED transitions ensure that an operating system can _always_	 * operate with its own GS base address:	 * - For events that occur in ring 3, FRED event delivery swaps	 *   the GS base address with the IA32_KERNEL_GS_BASE MSR.	 * - ERETU (the FRED transition that returns to ring 3) also swaps	 *   the GS base address with the IA32_KERNEL_GS_BASE MSR.	 *	 * And the operating system can still setup the GS segment for a	 * user thread without the need of loading a user thread GS with:	 * - Using LKGS, available with FRED, to modify other attributes	 *   of the GS segment without compromising its ability always to	 *   operate with its own GS base address.	 * - Accessing the GS segment base address for a user thread as	 *   before using RDMSR or WRMSR on the IA32_KERNEL_GS_BASE MSR.	 *	 * Note, LKGS loads the GS base address into the IA32_KERNEL_GS_BASE	 * MSR instead of the GS segment’s descriptor cache. As such, the	 * operating system never changes its runtime GS base address.	 */	if (!cpu_feature_enabled(X86_FEATURE_FRED) &&	    !cpu_feature_enabled(X86_FEATURE_XENPV)) {		native_swapgs();		gsbase = rdgsbase();		native_swapgs();	} else {		instrumentation_begin();		rdmsrq(MSR_KERNEL_GS_BASE, gsbase);		instrumentation_end();	} 	return gsbase;} /* * Out of line to be protected from kprobes and tracing. If this would be * traced or probed than any access to a per CPU variable happens with * the wrong GS. * * It is not used on Xen paravirt. When paravirt support is needed, it * needs to be renamed with native_ prefix. */static noinstr void __wrgsbase_inactive(unsigned long gsbase){	lockdep_assert_irqs_disabled(); 	if (!cpu_feature_enabled(X86_FEATURE_FRED) &&	    !cpu_feature_enabled(X86_FEATURE_XENPV)) {		native_swapgs();		wrgsbase(gsbase);		native_swapgs();	} else {		instrumentation_begin();		wrmsrq(MSR_KERNEL_GS_BASE, gsbase);		instrumentation_end();	}} /* * Saves the FS or GS base for an outgoing thread if FSGSBASE extensions are * not available.  The goal is to be reasonably fast on non-FSGSBASE systems. * It's forcibly inlined because it'll generate better code and this function * is hot. */static __always_inline void save_base_legacy(struct task_struct *prev_p,					     unsigned short selector,					     enum which_selector which){	if (likely(selector == 0)) {		/*		 * On Intel (without X86_BUG_NULL_SEG), the segment base could		 * be the pre-existing saved base or it could be zero.  On AMD		 * (with X86_BUG_NULL_SEG), the segment base could be almost		 * anything.		 *		 * This branch is very hot (it's hit twice on almost every		 * context switch between 64-bit programs), and avoiding		 * the RDMSR helps a lot, so we just assume that whatever		 * value is already saved is correct.  This matches historical		 * Linux behavior, so it won't break existing applications.		 *		 * To avoid leaking state, on non-X86_BUG_NULL_SEG CPUs, if we		 * report that the base is zero, it needs to actually be zero:		 * see the corresponding logic in load_seg_legacy.		 */	} else {		/*		 * If the selector is 1, 2, or 3, then the base is zero on		 * !X86_BUG_NULL_SEG CPUs and could be anything on		 * X86_BUG_NULL_SEG CPUs.  In the latter case, Linux		 * has never attempted to preserve the base across context		 * switches.		 *		 * If selector > 3, then it refers to a real segment, and		 * saving the base isn't necessary.		 */		if (which == FS)			prev_p->thread.fsbase = 0;		else			prev_p->thread.gsbase = 0;	}} static __always_inline void save_fsgs(struct task_struct *task){	savesegment(fs, task->thread.fsindex);	savesegment(gs, task->thread.gsindex);	if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) {		/*		 * If FSGSBASE is enabled, we can't make any useful guesses		 * about the base, and user code expects us to save the current		 * value.  Fortunately, reading the base directly is efficient.		 */		task->thread.fsbase = rdfsbase();		task->thread.gsbase = __rdgsbase_inactive();	} else {		save_base_legacy(task, task->thread.fsindex, FS);		save_base_legacy(task, task->thread.gsindex, GS);	}} /* * While a process is running,current->thread.fsbase and current->thread.gsbase * may not match the corresponding CPU registers (see save_base_legacy()). */void current_save_fsgs(void){	unsigned long flags; 	/* Interrupts need to be off for FSGSBASE */	local_irq_save(flags);	save_fsgs(current);	local_irq_restore(flags);}EXPORT_SYMBOL_FOR_KVM(current_save_fsgs); static __always_inline void loadseg(enum which_selector which,				    unsigned short sel){	if (which == FS)		loadsegment(fs, sel);	else		load_gs_index(sel);} static __always_inline void load_seg_legacy(unsigned short prev_index,					    unsigned long prev_base,					    unsigned short next_index,					    unsigned long next_base,					    enum which_selector which){	if (likely(next_index <= 3)) {		/*		 * The next task is using 64-bit TLS, is not using this		 * segment at all, or is having fun with arcane CPU features.		 */		if (next_base == 0) {			/*			 * Nasty case: on AMD CPUs, we need to forcibly zero			 * the base.			 */			if (static_cpu_has_bug(X86_BUG_NULL_SEG)) {				loadseg(which, __USER_DS);				loadseg(which, next_index);			} else {				/*				 * We could try to exhaustively detect cases				 * under which we can skip the segment load,				 * but there's really only one case that matters				 * for performance: if both the previous and				 * next states are fully zeroed, we can skip				 * the load.				 *				 * (This assumes that prev_base == 0 has no				 * false positives.  This is the case on				 * Intel-style CPUs.)				 */				if (likely(prev_index | next_index | prev_base))					loadseg(which, next_index);			}		} else {			if (prev_index != next_index)				loadseg(which, next_index);			wrmsrq(which == FS ? MSR_FS_BASE : MSR_KERNEL_GS_BASE,			       next_base);		}	} else {		/*		 * The next task is using a real segment.  Loading the selector		 * is sufficient.		 */		loadseg(which, next_index);	}} /* * Store prev's PKRU value and load next's PKRU value if they differ. PKRU * is not XSTATE managed on context switch because that would require a * lookup in the task's FPU xsave buffer and require to keep that updated * in various places. */static __always_inline void x86_pkru_load(struct thread_struct *prev,					  struct thread_struct *next){	if (!cpu_feature_enabled(X86_FEATURE_OSPKE))		return; 	/* Stash the prev task's value: */	prev->pkru = rdpkru(); 	/*	 * PKRU writes are slightly expensive.  Avoid them when not	 * strictly necessary:	 */	if (prev->pkru != next->pkru)		wrpkru(next->pkru);} static __always_inline void x86_fsgsbase_load(struct thread_struct *prev,					      struct thread_struct *next){	if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) {		/* Update the FS and GS selectors if they could have changed. */		if (unlikely(prev->fsindex || next->fsindex))			loadseg(FS, next->fsindex);		if (unlikely(prev->gsindex || next->gsindex))			loadseg(GS, next->gsindex); 		/* Update the bases. */		wrfsbase(next->fsbase);		__wrgsbase_inactive(next->gsbase);	} else {		load_seg_legacy(prev->fsindex, prev->fsbase,				next->fsindex, next->fsbase, FS);		load_seg_legacy(prev->gsindex, prev->gsbase,				next->gsindex, next->gsbase, GS);	}} unsigned long x86_fsgsbase_read_task(struct task_struct *task,				     unsigned short selector){	unsigned short idx = selector >> 3;	unsigned long base; 	if (likely((selector & SEGMENT_TI_MASK) == 0)) {		if (unlikely(idx >= GDT_ENTRIES))			return 0; 		/*		 * There are no user segments in the GDT with nonzero bases		 * other than the TLS segments.		 */		if (idx < GDT_ENTRY_TLS_MIN || idx > GDT_ENTRY_TLS_MAX)			return 0; 		idx -= GDT_ENTRY_TLS_MIN;		base = get_desc_base(&task->thread.tls_array[idx]);	} else {#ifdef CONFIG_MODIFY_LDT_SYSCALL		struct ldt_struct *ldt; 		/*		 * If performance here mattered, we could protect the LDT		 * with RCU.  This is a slow path, though, so we can just		 * take the mutex.		 */		mutex_lock(&task->mm->context.lock);		ldt = task->mm->context.ldt;		if (unlikely(!ldt || idx >= ldt->nr_entries))			base = 0;		else			base = get_desc_base(ldt->entries + idx);		mutex_unlock(&task->mm->context.lock);#else		base = 0;#endif	} 	return base;} unsigned long x86_gsbase_read_cpu_inactive(void){	unsigned long gsbase; 	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {		unsigned long flags; 		local_irq_save(flags);		gsbase = __rdgsbase_inactive();		local_irq_restore(flags);	} else {		rdmsrq(MSR_KERNEL_GS_BASE, gsbase);	} 	return gsbase;} void x86_gsbase_write_cpu_inactive(unsigned long gsbase){	if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {		unsigned long flags; 		local_irq_save(flags);		__wrgsbase_inactive(gsbase);		local_irq_restore(flags);	} else {		wrmsrq(MSR_KERNEL_GS_BASE, gsbase);	}} unsigned long x86_fsbase_read_task(struct task_struct *task){	unsigned long fsbase; 	if (task == current)		fsbase = x86_fsbase_read_cpu();	else if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||		 (task->thread.fsindex == 0))		fsbase = task->thread.fsbase;	else		fsbase = x86_fsgsbase_read_task(task, task->thread.fsindex); 	return fsbase;} unsigned long x86_gsbase_read_task(struct task_struct *task)

Lines 1–500 of 982. Load more