1677 lines · 45.1 KB · C
RawDownload
/* *  Copyright (C) 1991, 1992  Linus Torvalds *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs * *  Pentium III FXSR, SSE support *	Gareth Hughes <gareth@valinux.com>, May 2000 */ /* * Handle hardware traps and faults. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/context_tracking.h>#include <linux/interrupt.h>#include <linux/kallsyms.h>#include <linux/kmsan.h>#include <linux/spinlock.h>#include <linux/kprobes.h>#include <linux/uaccess.h>#include <linux/kdebug.h>#include <linux/kgdb.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/ptrace.h>#include <linux/uprobes.h>#include <linux/string.h>#include <linux/delay.h>#include <linux/errno.h>#include <linux/kexec.h>#include <linux/sched.h>#include <linux/sched/task_stack.h>#include <linux/static_call.h>#include <linux/timer.h>#include <linux/init.h>#include <linux/bug.h>#include <linux/nmi.h>#include <linux/mm.h>#include <linux/smp.h>#include <linux/cpu.h>#include <linux/io.h>#include <linux/hardirq.h>#include <linux/atomic.h>#include <linux/iommu.h>#include <linux/ubsan.h> #include <asm/stacktrace.h>#include <asm/processor.h>#include <asm/debugreg.h>#include <asm/realmode.h>#include <asm/text-patching.h>#include <asm/ftrace.h>#include <asm/traps.h>#include <asm/desc.h>#include <asm/fred.h>#include <asm/fpu/api.h>#include <asm/cpu.h>#include <asm/cpu_entry_area.h>#include <asm/mce.h>#include <asm/fixmap.h>#include <asm/mach_traps.h>#include <asm/alternative.h>#include <asm/fpu/xstate.h>#include <asm/vm86.h>#include <asm/umip.h>#include <asm/insn.h>#include <asm/insn-eval.h>#include <asm/vdso.h>#include <asm/tdx.h>#include <asm/cfi.h>#include <asm/msr.h>#include <asm/vsyscall.h> #ifdef CONFIG_X86_64#include <asm/x86_init.h>#else#include <asm/processor-flags.h>#include <asm/setup.h>#endif #include <asm/proto.h> DECLARE_BITMAP(system_vectors, NR_VECTORS); __always_inline int is_valid_bugaddr(unsigned long addr){	if (addr < TASK_SIZE_MAX)		return 0; 	/*	 * We got #UD, if the text isn't readable we'd have gotten	 * a different exception.	 */	return *(unsigned short *)addr == INSN_UD2;} /* * Check for UD1 or UD2, accounting for Address Size Override Prefixes. * If it's a UD1, further decode to determine its use: * * FineIBT:      d6                      udb * FineIBT:      f0 75 f9                lock jne . - 6 * UBSan{0}:     67 0f b9 00             ud1    (%eax),%eax * UBSan{10}:    67 0f b9 40 10          ud1    0x10(%eax),%eax * static_call:  0f b9 cc                ud1    %esp,%ecx * __WARN_trap:  67 48 0f b9 3a          ud1    (%edx),%reg * * Notable, since __WARN_trap can use all registers, the distinction between * UD1 users is through R/M. */__always_inline int decode_bug(unsigned long addr, s32 *imm, int *len){	unsigned long start = addr;	u8 v, reg, rm, rex = 0;	int type = BUG_UD1;	bool lock = false; 	if (addr < TASK_SIZE_MAX)		return BUG_NONE; 	for (;;) {		v = *(u8 *)(addr++);		if (v == INSN_ASOP)			continue; 		if (v == INSN_LOCK) {			lock = true;			continue;		} 		if ((v & 0xf0) == 0x40) {			rex = v;			continue;		} 		break;	} 	switch (v) {	case 0x70 ... 0x7f: /* Jcc.d8 */		addr += 1; /* d8 */		*len = addr - start;		WARN_ON_ONCE(!lock);		return BUG_LOCK; 	case 0xd6:		*len = addr - start;		return BUG_UDB; 	case OPCODE_ESCAPE:		break; 	default:		return BUG_NONE;	} 	v = *(u8 *)(addr++);	if (v == SECOND_BYTE_OPCODE_UD2) {		*len = addr - start;		return BUG_UD2;	} 	if (v != SECOND_BYTE_OPCODE_UD1)		return BUG_NONE; 	*imm = 0;	v = *(u8 *)(addr++);		/* ModRM */ 	if (X86_MODRM_MOD(v) != 3 && X86_MODRM_RM(v) == 4)		addr++;			/* SIB */ 	reg = X86_MODRM_REG(v) + 8*!!X86_REX_R(rex);	rm  = X86_MODRM_RM(v)  + 8*!!X86_REX_B(rex); 	/* Decode immediate, if present */	switch (X86_MODRM_MOD(v)) {	case 0: if (X86_MODRM_RM(v) == 5)			addr += 4;	/* RIP + disp32 */ 		if (rm == 0)		/* (%eax) */			type = BUG_UD1_UBSAN; 		if (rm == 2) {		/* (%edx) */			*imm = reg;			type = BUG_UD1_WARN;		}		break; 	case 1: *imm = *(s8 *)addr;		addr += 1;		if (rm == 0)		/* (%eax) */			type = BUG_UD1_UBSAN;		break; 	case 2: *imm = *(s32 *)addr;		addr += 4;		if (rm == 0)		/* (%eax) */			type = BUG_UD1_UBSAN;		break; 	case 3: break;	} 	/* record instruction length */	*len = addr - start; 	return type;} static inline unsigned long pt_regs_val(struct pt_regs *regs, int nr){	int offset = pt_regs_offset(regs, nr);	if (WARN_ON_ONCE(offset < -0))		return 0;	return *((unsigned long *)((void *)regs + offset));} #ifdef HAVE_ARCH_BUG_FORMAT_ARGSDEFINE_STATIC_CALL(WARN_trap, __WARN_trap);EXPORT_STATIC_CALL_TRAMP(WARN_trap); /* * Create a va_list from an exception context. */void *__warn_args(struct arch_va_list *args, struct pt_regs *regs){	/*	 * Register save area; populate with function call argument registers	 */	args->regs[0] = regs->di;	args->regs[1] = regs->si;	args->regs[2] = regs->dx;	args->regs[3] = regs->cx;	args->regs[4] = regs->r8;	args->regs[5] = regs->r9; 	/*	 * From the ABI document:	 *	 * @gp_offset - the element holds the offset in bytes from	 * reg_save_area to the place where the next available general purpose	 * argument register is saved. In case all argument registers have	 * been exhausted, it is set to the value 48 (6*8).	 *	 * @fp_offset - the element holds the offset in bytes from	 * reg_save_area to the place where the next available floating point	 * argument is saved. In case all argument registers have been	 * exhausted, it is set to the value 176 (6*8 + 8*16)	 *	 * @overflow_arg_area - this pointer is used to fetch arguments passed	 * on the stack. It is initialized with the address of the first	 * argument passed on the stack, if any, and then always updated to	 * point to the start of the next argument on the stack.	 *	 * @reg_save_area - the element points to the start of the register	 * save area.	 *	 * Notably the vararg starts with the second argument and there are no	 * floating point arguments in the kernel.	 */	args->args.gp_offset = 1*8;	args->args.fp_offset = 6*8 + 8*16;	args->args.reg_save_area = &args->regs;	args->args.overflow_arg_area = (void *)regs->sp; 	/*	 * If the exception came from __WARN_trap, there is a return	 * address on the stack, skip that. This is why any __WARN_trap()	 * caller must inhibit tail-call optimization.	 */	if ((void *)regs->ip == &__WARN_trap)		args->args.overflow_arg_area += 8; 	return &args->args;}#endif /* HAVE_ARCH_BUG_FORMAT */ static nokprobe_inline intdo_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str,		  struct pt_regs *regs,	long error_code){	if (v8086_mode(regs)) {		/*		 * Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.		 * On nmi (interrupt 2), do_trap should not be called.		 */		if (trapnr < X86_TRAP_UD) {			if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,						error_code, trapnr))				return 0;		}	} else if (!user_mode(regs)) {		if (fixup_exception(regs, trapnr, error_code, 0))			return 0; 		tsk->thread.error_code = error_code;		tsk->thread.trap_nr = trapnr;		die(str, regs, error_code);	} else {		if (fixup_vdso_exception(regs, trapnr, error_code, 0))			return 0;	} 	/*	 * We want error_code and trap_nr set for userspace faults and	 * kernelspace faults which result in die(), but not	 * kernelspace faults which are fixed up.  die() gives the	 * process no chance to handle the signal and notice the	 * kernel fault information, so that won't result in polluting	 * the information about previously queued, but not yet	 * delivered, faults.  See also exc_general_protection below.	 */	tsk->thread.error_code = error_code;	tsk->thread.trap_nr = trapnr; 	return -1;} static void show_signal(struct task_struct *tsk, int signr,			const char *type, const char *desc,			struct pt_regs *regs, long error_code){	if (show_unhandled_signals && unhandled_signal(tsk, signr) &&	    printk_ratelimit()) {		pr_info("%s[%d] %s%s ip:%lx sp:%lx error:%lx",			tsk->comm, task_pid_nr(tsk), type, desc,			regs->ip, regs->sp, error_code);		print_vma_addr(KERN_CONT " in ", regs->ip);		pr_cont("\n");	}} static voiddo_trap(int trapnr, int signr, char *str, struct pt_regs *regs,	long error_code, int sicode, void __user *addr){	struct task_struct *tsk = current; 	if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))		return; 	show_signal(tsk, signr, "trap ", str, regs, error_code); 	if (!sicode)		force_sig(signr);	else		force_sig_fault(signr, sicode, addr);}NOKPROBE_SYMBOL(do_trap); static void do_error_trap(struct pt_regs *regs, long error_code, char *str,	unsigned long trapnr, int signr, int sicode, void __user *addr){	RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); 	if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=			NOTIFY_STOP) {		cond_local_irq_enable(regs);		do_trap(trapnr, signr, str, regs, error_code, sicode, addr);		cond_local_irq_disable(regs);	}} /* * Posix requires to provide the address of the faulting instruction for * SIGILL (#UD) and SIGFPE (#DE) in the si_addr member of siginfo_t. * * This address is usually regs->ip, but when an uprobe moved the code out * of line then regs->ip points to the XOL code which would confuse * anything which analyzes the fault address vs. the unmodified binary. If * a trap happened in XOL code then uprobe maps regs->ip back to the * original instruction address. */static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs){	return (void __user *)uprobe_get_trap_addr(regs);} DEFINE_IDTENTRY(exc_divide_error){	do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,		      FPE_INTDIV, error_get_trap_addr(regs));} DEFINE_IDTENTRY(exc_overflow){	do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);} #ifdef CONFIG_X86_F00F_BUGvoid handle_invalid_op(struct pt_regs *regs)#elsestatic inline void handle_invalid_op(struct pt_regs *regs)#endif{	do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL,		      ILL_ILLOPN, error_get_trap_addr(regs));} noinstr bool handle_bug(struct pt_regs *regs){	unsigned long addr = regs->ip;	bool handled = false;	int ud_type, ud_len;	s32 ud_imm; 	ud_type = decode_bug(addr, &ud_imm, &ud_len);	if (ud_type == BUG_NONE)		return handled; 	/*	 * All lies, just get the WARN/BUG out.	 */	instrumentation_begin();	/*	 * Normally @regs are unpoisoned by irqentry_enter(), but handle_bug()	 * is a rare case that uses @regs without passing them to	 * irqentry_enter().	 */	kmsan_unpoison_entry_regs(regs);	/*	 * Since we're emulating a CALL with exceptions, restore the interrupt	 * state to what it was at the exception site.	 */	if (regs->flags & X86_EFLAGS_IF)		raw_local_irq_enable(); 	switch (ud_type) {	case BUG_UD1_WARN:		if (report_bug_entry((void *)pt_regs_val(regs, ud_imm), regs) == BUG_TRAP_TYPE_WARN)			handled = true;		break; 	case BUG_UD2:		if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {			handled = true;			break;		}		fallthrough; 	case BUG_UDB:	case BUG_LOCK:		if (handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {			handled = true;			break;		}		break; 	case BUG_UD1_UBSAN:		if (IS_ENABLED(CONFIG_UBSAN_TRAP)) {			pr_crit("%s at %pS\n",				report_ubsan_failure(ud_imm),				(void *)regs->ip);		}		break; 	default:		break;	} 	/*	 * When continuing, and regs->ip hasn't changed, move it to the next	 * instruction. When not continuing execution, restore the instruction	 * pointer.	 */	if (handled) {		if (regs->ip == addr)			regs->ip += ud_len;	} else {		regs->ip = addr;	} 	if (regs->flags & X86_EFLAGS_IF)		raw_local_irq_disable();	instrumentation_end(); 	return handled;} DEFINE_IDTENTRY_RAW(exc_invalid_op){	irqentry_state_t state; 	/*	 * We use UD2 as a short encoding for 'CALL __WARN', as such	 * handle it before exception entry to avoid recursive WARN	 * in case exception entry is the one triggering WARNs.	 */	if (!user_mode(regs) && handle_bug(regs))		return; 	state = irqentry_enter(regs);	instrumentation_begin();	handle_invalid_op(regs);	instrumentation_end();	irqentry_exit(regs, state);} DEFINE_IDTENTRY(exc_coproc_segment_overrun)

Lines 1–500 of 1677.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t