/* * Low-level exception handling code * * Copyright (C) 2012 ARM Ltd. * Authors:	Catalin Marinas <catalin.marinas@arm.com> *		Will Deacon <will.deacon@arm.com> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program.  If not, see <http://www.gnu.org/licenses/>. */ #include <linux/init.h>#include <linux/linkage.h> #include <asm/alternative.h>#include <asm/assembler.h>#include <asm/asm-offsets.h>#include <asm/cpufeature.h>#include <asm/errno.h>#include <asm/esr.h>#include <asm/irq.h>#include <asm/memory.h>#include <asm/ptrace.h>#include <asm/thread_info.h>#include <asm/asm-uaccess.h>#include <asm/unistd.h> /* * Context tracking subsystem.  Used to instrument transitions * between user and kernel mode. */	.macro ct_user_exit, syscall = 0#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_exit	.if \syscall == 1	/*	 * Save/restore needed during syscalls.  Restore syscall arguments from	 * the values already saved on stack during kernel_entry.	 */	ldp	x0, x1, [sp]	ldp	x2, x3, [sp, #S_X2]	ldp	x4, x5, [sp, #S_X4]	ldp	x6, x7, [sp, #S_X6]	.endif#endif	.endm 	.macro ct_user_enter#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_enter#endif	.endm /* * Bad Abort numbers *----------------- */#define BAD_SYNC	0#define BAD_IRQ		1#define BAD_FIQ		2#define BAD_ERROR	3 	.macro	kernel_entry, el, regsize = 64	sub	sp, sp, #S_FRAME_SIZE	.if	\regsize == 32	mov	w0, w0				// zero upper 32 bits of x0	.endif	stp	x0, x1, [sp, #16 * 0]	stp	x2, x3, [sp, #16 * 1]	stp	x4, x5, [sp, #16 * 2]	stp	x6, x7, [sp, #16 * 3]	stp	x8, x9, [sp, #16 * 4]	stp	x10, x11, [sp, #16 * 5]	stp	x12, x13, [sp, #16 * 6]	stp	x14, x15, [sp, #16 * 7]	stp	x16, x17, [sp, #16 * 8]	stp	x18, x19, [sp, #16 * 9]	stp	x20, x21, [sp, #16 * 10]	stp	x22, x23, [sp, #16 * 11]	stp	x24, x25, [sp, #16 * 12]	stp	x26, x27, [sp, #16 * 13]	stp	x28, x29, [sp, #16 * 14] 	.if	\el == 0	mrs	x21, sp_el0	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug	disable_step_tsk x19, x20		// exceptions when scheduling. 	mov	x29, xzr			// fp pointed to user-space	.else	add	x21, sp, #S_FRAME_SIZE	get_thread_info tsk	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]	str	x20, [sp, #S_ORIG_ADDR_LIMIT]	mov	x20, #TASK_SIZE_64	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */	.endif /* \el == 0 */	mrs	x22, elr_el1	mrs	x23, spsr_el1	stp	lr, x21, [sp, #S_LR] #ifdef CONFIG_ARM64_SW_TTBR0_PAN	/*	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from	 * EL0, there is no need to check the state of TTBR0_EL1 since	 * accesses are always enabled.	 * Note that the meaning of this bit differs from the ARMv8.1 PAN	 * feature as all TTBR0_EL1 accesses are disabled, not just those to	 * user mappings.	 */alternative_if ARM64_HAS_PAN	b	1f				// skip TTBR0 PANalternative_else_nop_endif 	.if	\el != 0	mrs	x21, ttbr0_el1	tst	x21, #0xffff << 48		// Check for the reserved ASID	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR	b.eq	1f				// TTBR0 access already disabled	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR	.endif 	__uaccess_ttbr0_disable x211:#endif 	stp	x22, x23, [sp, #S_PC] 	/*	 * Set syscallno to -1 by default (overridden later if real syscall).	 */	.if	\el == 0	mvn	x21, xzr	str	x21, [sp, #S_SYSCALLNO]	.endif 	/*	 * Set sp_el0 to current thread_info.	 */	.if	\el == 0	msr	sp_el0, tsk	.endif 	/*	 * Registers that may be useful after this macro is invoked:	 *	 * x21 - aborted SP	 * x22 - aborted PC	 * x23 - aborted PSTATE	*/	.endm 	.macro	kernel_exit, el	.if	\el != 0	/* Restore the task's original addr_limit. */	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]	str	x20, [tsk, #TSK_TI_ADDR_LIMIT] 	/* No need to restore UAO, it will be restored from SPSR_EL1 */	.endif 	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR	.if	\el == 0	ct_user_enter	.endif #ifdef CONFIG_ARM64_SW_TTBR0_PAN	/*	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR	 * PAN bit checking.	 */alternative_if ARM64_HAS_PAN	b	2f				// skip TTBR0 PANalternative_else_nop_endif 	.if	\el != 0	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set	.endif 	__uaccess_ttbr0_enable x0 	.if	\el == 0	/*	 * Enable errata workarounds only if returning to user. The only	 * workaround currently required for TTBR0_EL1 changes are for the	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache	 * corruption).	 */	post_ttbr0_update_workaround	.endif1:	.if	\el != 0	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit	.endif2:#endif 	.if	\el == 0	ldr	x23, [sp, #S_SP]		// load return stack pointer	msr	sp_el0, x23#ifdef CONFIG_ARM64_ERRATUM_845719alternative_if ARM64_WORKAROUND_845719	tbz	x22, #4, 1f#ifdef CONFIG_PID_IN_CONTEXTIDR	mrs	x29, contextidr_el1	msr	contextidr_el1, x29#else	msr contextidr_el1, xzr#endif1:alternative_else_nop_endif#endif	.endif 	msr	elr_el1, x21			// set up the return data	msr	spsr_el1, x22	ldp	x0, x1, [sp, #16 * 0]	ldp	x2, x3, [sp, #16 * 1]	ldp	x4, x5, [sp, #16 * 2]	ldp	x6, x7, [sp, #16 * 3]	ldp	x8, x9, [sp, #16 * 4]	ldp	x10, x11, [sp, #16 * 5]	ldp	x12, x13, [sp, #16 * 6]	ldp	x14, x15, [sp, #16 * 7]	ldp	x16, x17, [sp, #16 * 8]	ldp	x18, x19, [sp, #16 * 9]	ldp	x20, x21, [sp, #16 * 10]	ldp	x22, x23, [sp, #16 * 11]	ldp	x24, x25, [sp, #16 * 12]	ldp	x26, x27, [sp, #16 * 13]	ldp	x28, x29, [sp, #16 * 14]	ldr	lr, [sp, #S_LR]	add	sp, sp, #S_FRAME_SIZE		// restore sp	eret					// return to kernel	.endm 	.macro	irq_stack_entry	mov	x19, sp			// preserve the original sp 	/*	 * Compare sp with the base of the task stack.	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,	 * and should switch to the irq stack.	 */	ldr	x25, [tsk, TSK_STACK]	eor	x25, x25, x19	and	x25, x25, #~(THREAD_SIZE - 1)	cbnz	x25, 9998f 	adr_this_cpu x25, irq_stack, x26	mov	x26, #IRQ_STACK_START_SP	add	x26, x25, x26 	/* switch to the irq stack */	mov	sp, x26 	/*	 * Add a dummy stack frame, this non-standard format is fixed up	 * by unwind_frame()	 */	stp     x29, x19, [sp, #-16]!	mov	x29, sp 9998:	.endm 	/*	 * x19 should be preserved between irq_stack_entry and	 * irq_stack_exit.	 */	.macro	irq_stack_exit	mov	sp, x19	.endm /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - x0 to x6. * * x7 is reserved for the system call number in 32-bit mode. */sc_nr	.req	x25		// number of system callsscno	.req	x26		// syscall numberstbl	.req	x27		// syscall table pointertsk	.req	x28		// current thread_info /* * Interrupt handling. */	.macro	irq_handler	ldr_l	x1, handle_arch_irq	mov	x0, sp	irq_stack_entry	blr	x1	irq_stack_exit	.endm 	.text /* * Exception vectors. */	.pushsection ".entry.text", "ax" 	.align	11ENTRY(vectors)	ventry	el1_sync_invalid		// Synchronous EL1t	ventry	el1_irq_invalid			// IRQ EL1t	ventry	el1_fiq_invalid			// FIQ EL1t	ventry	el1_error_invalid		// Error EL1t 	ventry	el1_sync			// Synchronous EL1h	ventry	el1_irq				// IRQ EL1h	ventry	el1_fiq_invalid			// FIQ EL1h	ventry	el1_error_invalid		// Error EL1h 	ventry	el0_sync			// Synchronous 64-bit EL0	ventry	el0_irq				// IRQ 64-bit EL0	ventry	el0_fiq_invalid			// FIQ 64-bit EL0	ventry	el0_error_invalid		// Error 64-bit EL0 #ifdef CONFIG_COMPAT	ventry	el0_sync_compat			// Synchronous 32-bit EL0	ventry	el0_irq_compat			// IRQ 32-bit EL0	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0	ventry	el0_error_invalid_compat	// Error 32-bit EL0#else	ventry	el0_sync_invalid		// Synchronous 32-bit EL0	ventry	el0_irq_invalid			// IRQ 32-bit EL0	ventry	el0_fiq_invalid			// FIQ 32-bit EL0	ventry	el0_error_invalid		// Error 32-bit EL0#endifEND(vectors) /* * Invalid mode handlers */	.macro	inv_entry, el, reason, regsize = 64	kernel_entry \el, \regsize	mov	x0, sp	mov	x1, #\reason	mrs	x2, esr_el1	b	bad_mode	.endm el0_sync_invalid:	inv_entry 0, BAD_SYNCENDPROC(el0_sync_invalid) el0_irq_invalid:	inv_entry 0, BAD_IRQENDPROC(el0_irq_invalid) el0_fiq_invalid:	inv_entry 0, BAD_FIQENDPROC(el0_fiq_invalid) el0_error_invalid:	inv_entry 0, BAD_ERRORENDPROC(el0_error_invalid) #ifdef CONFIG_COMPATel0_fiq_invalid_compat:	inv_entry 0, BAD_FIQ, 32ENDPROC(el0_fiq_invalid_compat) el0_error_invalid_compat:	inv_entry 0, BAD_ERROR, 32ENDPROC(el0_error_invalid_compat)#endif el1_sync_invalid:	inv_entry 1, BAD_SYNCENDPROC(el1_sync_invalid) el1_irq_invalid:	inv_entry 1, BAD_IRQENDPROC(el1_irq_invalid) el1_fiq_invalid:	inv_entry 1, BAD_FIQENDPROC(el1_fiq_invalid) el1_error_invalid:	inv_entry 1, BAD_ERRORENDPROC(el1_error_invalid) /* * EL1 mode handlers. */	.align	6el1_sync:	kernel_entry 1	mrs	x1, esr_el1			// read the syndrome register	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1	b.eq	el1_da	cmp	x24, #ESR_ELx_EC_IABT_CUR	// instruction abort in EL1	b.eq	el1_ia	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap	b.eq	el1_undef	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception	b.eq	el1_sp_pc	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception	b.eq	el1_sp_pc	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1	b.eq	el1_undef	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1	b.ge	el1_dbg	b	el1_inv el1_ia:	/*	 * Fall through to the Data abort case	 */el1_da:	/*	 * Data abort handling	 */	mrs	x3, far_el1	enable_dbg	// re-enable interrupts if they were enabled in the aborted context	tbnz	x23, #7, 1f			// PSR_I_BIT	enable_irq1:	clear_address_tag x0, x3	mov	x2, sp				// struct pt_regs	bl	do_mem_abort 	// disable interrupts before pulling preserved data off the stack	disable_irq	kernel_exit 1el1_sp_pc:	/*	 * Stack or PC alignment exception handling	 */	mrs	x0, far_el1	enable_dbg	mov	x2, sp	b	do_sp_pc_abortel1_undef:	/*	 * Undefined instruction	 */	enable_dbg	mov	x0, sp	b	do_undefinstrel1_dbg:	/*	 * Debug exception handling	 */	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64	cinc	x24, x24, eq			// set bit '0'	tbz	x24, #0, el1_inv		// EL1 only	mrs	x0, far_el1	mov	x2, sp				// struct pt_regs	bl	do_debug_exception	kernel_exit 1el1_inv:	// TODO: add support for undefined instructions in kernel mode	enable_dbg	mov	x0, sp	mov	x2, x1	mov	x1, #BAD_SYNC	b	bad_modeENDPROC(el1_sync) 	.align	6el1_irq:	kernel_entry 1	enable_dbg#ifdef CONFIG_TRACE_IRQFLAGS	bl	trace_hardirqs_off#endif 	irq_handler #ifdef CONFIG_PREEMPT	ldr	w24, [tsk, #TSK_TI_PREEMPT]	// get preempt count	cbnz	w24, 1f				// preempt count != 0	ldr	x0, [tsk, #TSK_TI_FLAGS]	// get flags	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?	bl	el1_preempt1:#endif#ifdef CONFIG_TRACE_IRQFLAGS	bl	trace_hardirqs_on#endif	kernel_exit 1

Lines 1–500 of 867. Load more