/* * Low-level exception handling code * * Copyright (C) 2012 ARM Ltd. * Authors:	Catalin Marinas <catalin.marinas@arm.com> *		Will Deacon <will.deacon@arm.com> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program.  If not, see <http://www.gnu.org/licenses/>. */ #include <linux/init.h>#include <linux/linkage.h> #include <asm/alternative.h>#include <asm/assembler.h>#include <asm/asm-offsets.h>#include <asm/cpufeature.h>#include <asm/errno.h>#include <asm/esr.h>#include <asm/thread_info.h>#include <asm/unistd.h> /* * Context tracking subsystem.  Used to instrument transitions * between user and kernel mode. */	.macro ct_user_exit, syscall = 0#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_exit	.if \syscall == 1	/*	 * Save/restore needed during syscalls.  Restore syscall arguments from	 * the values already saved on stack during kernel_entry.	 */	ldp	x0, x1, [sp]	ldp	x2, x3, [sp, #S_X2]	ldp	x4, x5, [sp, #S_X4]	ldp	x6, x7, [sp, #S_X6]	.endif#endif	.endm 	.macro ct_user_enter#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_enter#endif	.endm /* * Bad Abort numbers *----------------- */#define BAD_SYNC	0#define BAD_IRQ		1#define BAD_FIQ		2#define BAD_ERROR	3 	.macro	kernel_entry, el, regsize = 64	sub	sp, sp, #S_FRAME_SIZE	.if	\regsize == 32	mov	w0, w0				// zero upper 32 bits of x0	.endif	stp	x0, x1, [sp, #16 * 0]	stp	x2, x3, [sp, #16 * 1]	stp	x4, x5, [sp, #16 * 2]	stp	x6, x7, [sp, #16 * 3]	stp	x8, x9, [sp, #16 * 4]	stp	x10, x11, [sp, #16 * 5]	stp	x12, x13, [sp, #16 * 6]	stp	x14, x15, [sp, #16 * 7]	stp	x16, x17, [sp, #16 * 8]	stp	x18, x19, [sp, #16 * 9]	stp	x20, x21, [sp, #16 * 10]	stp	x22, x23, [sp, #16 * 11]	stp	x24, x25, [sp, #16 * 12]	stp	x26, x27, [sp, #16 * 13]	stp	x28, x29, [sp, #16 * 14] 	.if	\el == 0	mrs	x21, sp_el0	get_thread_info tsk			// Ensure MDSCR_EL1.SS is clear,	ldr	x19, [tsk, #TI_FLAGS]		// since we can unmask debug	disable_step_tsk x19, x20		// exceptions when scheduling.	.else	add	x21, sp, #S_FRAME_SIZE	.endif	mrs	x22, elr_el1	mrs	x23, spsr_el1	stp	lr, x21, [sp, #S_LR]	stp	x22, x23, [sp, #S_PC] 	/*	 * Set syscallno to -1 by default (overridden later if real syscall).	 */	.if	\el == 0	mvn	x21, xzr	str	x21, [sp, #S_SYSCALLNO]	.endif 	/*	 * Registers that may be useful after this macro is invoked:	 *	 * x21 - aborted SP	 * x22 - aborted PC	 * x23 - aborted PSTATE	*/	.endm 	.macro	kernel_exit, el, ret = 0	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR	.if	\el == 0	ct_user_enter	ldr	x23, [sp, #S_SP]		// load return stack pointer	msr	sp_el0, x23 #ifdef CONFIG_ARM64_ERRATUM_845719 #undef SEQUENCE_ORG#undef SEQUENCE_ALT #ifdef CONFIG_PID_IN_CONTEXTIDR #define SEQUENCE_ORG	"nop ; nop ; nop"#define SEQUENCE_ALT	"tbz x22, #4, 1f ; mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:" #else #define SEQUENCE_ORG	"nop ; nop"#define SEQUENCE_ALT	"tbz x22, #4, 1f ; msr contextidr_el1, xzr; 1:" #endif 	alternative_insn SEQUENCE_ORG, SEQUENCE_ALT, ARM64_WORKAROUND_845719 #endif	.endif	msr	elr_el1, x21			// set up the return data	msr	spsr_el1, x22	.if	\ret	ldr	x1, [sp, #S_X1]			// preserve x0 (syscall return)	.else	ldp	x0, x1, [sp, #16 * 0]	.endif	ldp	x2, x3, [sp, #16 * 1]	ldp	x4, x5, [sp, #16 * 2]	ldp	x6, x7, [sp, #16 * 3]	ldp	x8, x9, [sp, #16 * 4]	ldp	x10, x11, [sp, #16 * 5]	ldp	x12, x13, [sp, #16 * 6]	ldp	x14, x15, [sp, #16 * 7]	ldp	x16, x17, [sp, #16 * 8]	ldp	x18, x19, [sp, #16 * 9]	ldp	x20, x21, [sp, #16 * 10]	ldp	x22, x23, [sp, #16 * 11]	ldp	x24, x25, [sp, #16 * 12]	ldp	x26, x27, [sp, #16 * 13]	ldp	x28, x29, [sp, #16 * 14]	ldr	lr, [sp, #S_LR]	add	sp, sp, #S_FRAME_SIZE		// restore sp	eret					// return to kernel	.endm 	.macro	get_thread_info, rd	mov	\rd, sp	and	\rd, \rd, #~(THREAD_SIZE - 1)	// top of stack	.endm /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - x0 to x6. * * x7 is reserved for the system call number in 32-bit mode. */sc_nr	.req	x25		// number of system callsscno	.req	x26		// syscall numberstbl	.req	x27		// syscall table pointertsk	.req	x28		// current thread_info /* * Interrupt handling. */	.macro	irq_handler	adrp	x1, handle_arch_irq	ldr	x1, [x1, #:lo12:handle_arch_irq]	mov	x0, sp	blr	x1	.endm 	.text /* * Exception vectors. */ 	.align	11ENTRY(vectors)	ventry	el1_sync_invalid		// Synchronous EL1t	ventry	el1_irq_invalid			// IRQ EL1t	ventry	el1_fiq_invalid			// FIQ EL1t	ventry	el1_error_invalid		// Error EL1t 	ventry	el1_sync			// Synchronous EL1h	ventry	el1_irq				// IRQ EL1h	ventry	el1_fiq_invalid			// FIQ EL1h	ventry	el1_error_invalid		// Error EL1h 	ventry	el0_sync			// Synchronous 64-bit EL0	ventry	el0_irq				// IRQ 64-bit EL0	ventry	el0_fiq_invalid			// FIQ 64-bit EL0	ventry	el0_error_invalid		// Error 64-bit EL0 #ifdef CONFIG_COMPAT	ventry	el0_sync_compat			// Synchronous 32-bit EL0	ventry	el0_irq_compat			// IRQ 32-bit EL0	ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0	ventry	el0_error_invalid_compat	// Error 32-bit EL0#else	ventry	el0_sync_invalid		// Synchronous 32-bit EL0	ventry	el0_irq_invalid			// IRQ 32-bit EL0	ventry	el0_fiq_invalid			// FIQ 32-bit EL0	ventry	el0_error_invalid		// Error 32-bit EL0#endifEND(vectors) /* * Invalid mode handlers */	.macro	inv_entry, el, reason, regsize = 64	kernel_entry el, \regsize	mov	x0, sp	mov	x1, #\reason	mrs	x2, esr_el1	b	bad_mode	.endm el0_sync_invalid:	inv_entry 0, BAD_SYNCENDPROC(el0_sync_invalid) el0_irq_invalid:	inv_entry 0, BAD_IRQENDPROC(el0_irq_invalid) el0_fiq_invalid:	inv_entry 0, BAD_FIQENDPROC(el0_fiq_invalid) el0_error_invalid:	inv_entry 0, BAD_ERRORENDPROC(el0_error_invalid) #ifdef CONFIG_COMPATel0_fiq_invalid_compat:	inv_entry 0, BAD_FIQ, 32ENDPROC(el0_fiq_invalid_compat) el0_error_invalid_compat:	inv_entry 0, BAD_ERROR, 32ENDPROC(el0_error_invalid_compat)#endif el1_sync_invalid:	inv_entry 1, BAD_SYNCENDPROC(el1_sync_invalid) el1_irq_invalid:	inv_entry 1, BAD_IRQENDPROC(el1_irq_invalid) el1_fiq_invalid:	inv_entry 1, BAD_FIQENDPROC(el1_fiq_invalid) el1_error_invalid:	inv_entry 1, BAD_ERRORENDPROC(el1_error_invalid) /* * EL1 mode handlers. */	.align	6el1_sync:	kernel_entry 1	mrs	x1, esr_el1			// read the syndrome register	lsr	x24, x1, #ESR_ELx_EC_SHIFT	// exception class	cmp	x24, #ESR_ELx_EC_DABT_CUR	// data abort in EL1	b.eq	el1_da	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap	b.eq	el1_undef	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception	b.eq	el1_sp_pc	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception	b.eq	el1_sp_pc	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL1	b.eq	el1_undef	cmp	x24, #ESR_ELx_EC_BREAKPT_CUR	// debug exception in EL1	b.ge	el1_dbg	b	el1_invel1_da:	/*	 * Data abort handling	 */	mrs	x0, far_el1	enable_dbg	// re-enable interrupts if they were enabled in the aborted context	tbnz	x23, #7, 1f			// PSR_I_BIT	enable_irq1:	mov	x2, sp				// struct pt_regs	bl	do_mem_abort 	// disable interrupts before pulling preserved data off the stack	disable_irq	kernel_exit 1el1_sp_pc:	/*	 * Stack or PC alignment exception handling	 */	mrs	x0, far_el1	enable_dbg	mov	x2, sp	b	do_sp_pc_abortel1_undef:	/*	 * Undefined instruction	 */	enable_dbg	mov	x0, sp	b	do_undefinstrel1_dbg:	/*	 * Debug exception handling	 */	cmp	x24, #ESR_ELx_EC_BRK64		// if BRK64	cinc	x24, x24, eq			// set bit '0'	tbz	x24, #0, el1_inv		// EL1 only	mrs	x0, far_el1	mov	x2, sp				// struct pt_regs	bl	do_debug_exception	kernel_exit 1el1_inv:	// TODO: add support for undefined instructions in kernel mode	enable_dbg	mov	x0, sp	mov	x2, x1	mov	x1, #BAD_SYNC	b	bad_modeENDPROC(el1_sync) 	.align	6el1_irq:	kernel_entry 1	enable_dbg#ifdef CONFIG_TRACE_IRQFLAGS	bl	trace_hardirqs_off#endif 	irq_handler #ifdef CONFIG_PREEMPT	get_thread_info tsk	ldr	w24, [tsk, #TI_PREEMPT]		// get preempt count	cbnz	w24, 1f				// preempt count != 0	ldr	x0, [tsk, #TI_FLAGS]		// get flags	tbz	x0, #TIF_NEED_RESCHED, 1f	// needs rescheduling?	bl	el1_preempt1:#endif#ifdef CONFIG_TRACE_IRQFLAGS	bl	trace_hardirqs_on#endif	kernel_exit 1ENDPROC(el1_irq) #ifdef CONFIG_PREEMPTel1_preempt:	mov	x24, lr1:	bl	preempt_schedule_irq		// irq en/disable is done inside	ldr	x0, [tsk, #TI_FLAGS]		// get new tasks TI_FLAGS	tbnz	x0, #TIF_NEED_RESCHED, 1b	// needs rescheduling?	ret	x24#endif /* * EL0 mode handlers. */	.align	6el0_sync:	kernel_entry 0	mrs	x25, esr_el1			// read the syndrome register	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class	cmp	x24, #ESR_ELx_EC_SVC64		// SVC in 64-bit state	b.eq	el0_svc	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0	b.eq	el0_da	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0	b.eq	el0_ia	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access	b.eq	el0_fpsimd_acc	cmp	x24, #ESR_ELx_EC_FP_EXC64	// FP/ASIMD exception	b.eq	el0_fpsimd_exc	cmp	x24, #ESR_ELx_EC_SYS64		// configurable trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_SP_ALIGN	// stack alignment exception	b.eq	el0_sp_pc	cmp	x24, #ESR_ELx_EC_PC_ALIGN	// pc alignment exception	b.eq	el0_sp_pc	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0	b.ge	el0_dbg	b	el0_inv #ifdef CONFIG_COMPAT	.align	6el0_sync_compat:	kernel_entry 0, 32	mrs	x25, esr_el1			// read the syndrome register	lsr	x24, x25, #ESR_ELx_EC_SHIFT	// exception class	cmp	x24, #ESR_ELx_EC_SVC32		// SVC in 32-bit state	b.eq	el0_svc_compat	cmp	x24, #ESR_ELx_EC_DABT_LOW	// data abort in EL0	b.eq	el0_da	cmp	x24, #ESR_ELx_EC_IABT_LOW	// instruction abort in EL0	b.eq	el0_ia	cmp	x24, #ESR_ELx_EC_FP_ASIMD	// FP/ASIMD access	b.eq	el0_fpsimd_acc	cmp	x24, #ESR_ELx_EC_FP_EXC32	// FP/ASIMD exception	b.eq	el0_fpsimd_exc	cmp	x24, #ESR_ELx_EC_UNKNOWN	// unknown exception in EL0	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_CP15_32	// CP15 MRC/MCR trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_CP15_64	// CP15 MRRC/MCRR trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_CP14_MR	// CP14 MRC/MCR trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_CP14_LS	// CP14 LDC/STC trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_CP14_64	// CP14 MRRC/MCRR trap	b.eq	el0_undef	cmp	x24, #ESR_ELx_EC_BREAKPT_LOW	// debug exception in EL0	b.ge	el0_dbg	b	el0_invel0_svc_compat:	/*	 * AArch32 syscall handling	 */	adrp	stbl, compat_sys_call_table	// load compat syscall table pointer	uxtw	scno, w7			// syscall number in w7 (r7)	mov     sc_nr, #__NR_compat_syscalls	b	el0_svc_naked 	.align	6el0_irq_compat:	kernel_entry 0, 32	b	el0_irq_naked#endif el0_da:	/*	 * Data abort handling	 */	mrs	x26, far_el1	// enable interrupts before calling the main handler	enable_dbg_and_irq	ct_user_exit	bic	x0, x26, #(0xff << 56)	mov	x1, x25	mov	x2, sp	bl	do_mem_abort	b	ret_to_userel0_ia:	/*	 * Instruction abort handling	 */	mrs	x26, far_el1	// enable interrupts before calling the main handler	enable_dbg_and_irq	ct_user_exit	mov	x0, x26	orr	x1, x25, #1 << 24		// use reserved ISS bit for instruction aborts	mov	x2, sp	bl	do_mem_abort	b	ret_to_userel0_fpsimd_acc:	/*	 * Floating Point or Advanced SIMD access	 */	enable_dbg

Lines 1–500 of 740. Load more