/* SPDX-License-Identifier: GPL-2.0-only *//* * Low-level exception handling code * * Copyright (C) 2012 ARM Ltd. * Authors:	Catalin Marinas <catalin.marinas@arm.com> *		Will Deacon <will.deacon@arm.com> */ #include <linux/arm-smccc.h>#include <linux/init.h>#include <linux/linkage.h> #include <asm/alternative.h>#include <asm/assembler.h>#include <asm/asm-offsets.h>#include <asm/cpufeature.h>#include <asm/errno.h>#include <asm/esr.h>#include <asm/irq.h>#include <asm/memory.h>#include <asm/mmu.h>#include <asm/processor.h>#include <asm/ptrace.h>#include <asm/thread_info.h>#include <asm/asm-uaccess.h>#include <asm/unistd.h> /* * Context tracking subsystem.  Used to instrument transitions * between user and kernel mode. */	.macro ct_user_exit#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_exit#endif	.endm 	.macro ct_user_enter#ifdef CONFIG_CONTEXT_TRACKING	bl	context_tracking_user_enter#endif	.endm 	.macro	clear_gp_regs	.irp	n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29	mov	x\n, xzr	.endr	.endm /* * Bad Abort numbers *----------------- */#define BAD_SYNC	0#define BAD_IRQ		1#define BAD_FIQ		2#define BAD_ERROR	3 	.macro kernel_ventry, el, label, regsize = 64	.align 7#ifdef CONFIG_UNMAP_KERNEL_AT_EL0alternative_if ARM64_UNMAP_KERNEL_AT_EL0	.if	\el == 0	.if	\regsize == 64	mrs	x30, tpidrro_el0	msr	tpidrro_el0, xzr	.else	mov	x30, xzr	.endif	.endifalternative_else_nop_endif#endif 	sub	sp, sp, #S_FRAME_SIZE#ifdef CONFIG_VMAP_STACK	/*	 * Test whether the SP has overflowed, without corrupting a GPR.	 * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).	 */	add	sp, sp, x0			// sp' = sp + x0	sub	x0, sp, x0			// x0' = sp' - x0 = (sp + x0) - x0 = sp	tbnz	x0, #THREAD_SHIFT, 0f	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp	b	el\()\el\()_\label 0:	/*	 * Either we've just detected an overflow, or we've taken an exception	 * while on the overflow stack. Either way, we won't return to	 * userspace, and can clobber EL0 registers to free up GPRs.	 */ 	/* Stash the original SP (minus S_FRAME_SIZE) in tpidr_el0. */	msr	tpidr_el0, x0 	/* Recover the original x0 value and stash it in tpidrro_el0 */	sub	x0, sp, x0	msr	tpidrro_el0, x0 	/* Switch to the overflow stack */	adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0 	/*	 * Check whether we were already on the overflow stack. This may happen	 * after panic() re-enables interrupts.	 */	mrs	x0, tpidr_el0			// sp of interrupted context	sub	x0, sp, x0			// delta with top of overflow stack	tst	x0, #~(OVERFLOW_STACK_SIZE - 1)	// within range?	b.ne	__bad_stack			// no? -> bad stack pointer 	/* We were already on the overflow stack. Restore sp/x0 and carry on. */	sub	sp, sp, x0	mrs	x0, tpidrro_el0#endif	b	el\()\el\()_\label	.endm 	.macro tramp_alias, dst, sym	mov_q	\dst, TRAMP_VALIAS	add	\dst, \dst, #(\sym - .entry.tramp.text)	.endm 	// This macro corrupts x0-x3. It is the caller's duty	// to save/restore them if required.	.macro	apply_ssbd, state, tmp1, tmp2#ifdef CONFIG_ARM64_SSBDalternative_cb	arm64_enable_wa2_handling	b	.L__asm_ssbd_skip\@alternative_cb_end	ldr_this_cpu	\tmp2, arm64_ssbd_callback_required, \tmp1	cbz	\tmp2,	.L__asm_ssbd_skip\@	ldr	\tmp2, [tsk, #TSK_TI_FLAGS]	tbnz	\tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2	mov	w1, #\statealternative_cb	arm64_update_smccc_conduit	nop					// Patched to SMC/HVC #0alternative_cb_end.L__asm_ssbd_skip\@:#endif	.endm 	.macro	kernel_entry, el, regsize = 64	.if	\regsize == 32	mov	w0, w0				// zero upper 32 bits of x0	.endif	stp	x0, x1, [sp, #16 * 0]	stp	x2, x3, [sp, #16 * 1]	stp	x4, x5, [sp, #16 * 2]	stp	x6, x7, [sp, #16 * 3]	stp	x8, x9, [sp, #16 * 4]	stp	x10, x11, [sp, #16 * 5]	stp	x12, x13, [sp, #16 * 6]	stp	x14, x15, [sp, #16 * 7]	stp	x16, x17, [sp, #16 * 8]	stp	x18, x19, [sp, #16 * 9]	stp	x20, x21, [sp, #16 * 10]	stp	x22, x23, [sp, #16 * 11]	stp	x24, x25, [sp, #16 * 12]	stp	x26, x27, [sp, #16 * 13]	stp	x28, x29, [sp, #16 * 14] 	.if	\el == 0	clear_gp_regs	mrs	x21, sp_el0	ldr_this_cpu	tsk, __entry_task, x20	// Ensure MDSCR_EL1.SS is clear,	ldr	x19, [tsk, #TSK_TI_FLAGS]	// since we can unmask debug	disable_step_tsk x19, x20		// exceptions when scheduling. 	apply_ssbd 1, x22, x23 	.else	add	x21, sp, #S_FRAME_SIZE	get_current_task tsk	/* Save the task's original addr_limit and set USER_DS */	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]	str	x20, [sp, #S_ORIG_ADDR_LIMIT]	mov	x20, #USER_DS	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */	.endif /* \el == 0 */	mrs	x22, elr_el1	mrs	x23, spsr_el1	stp	lr, x21, [sp, #S_LR] 	/*	 * In order to be able to dump the contents of struct pt_regs at the	 * time the exception was taken (in case we attempt to walk the call	 * stack later), chain it together with the stack frames.	 */	.if \el == 0	stp	xzr, xzr, [sp, #S_STACKFRAME]	.else	stp	x29, x22, [sp, #S_STACKFRAME]	.endif	add	x29, sp, #S_STACKFRAME #ifdef CONFIG_ARM64_SW_TTBR0_PAN	/*	 * Set the TTBR0 PAN bit in SPSR. When the exception is taken from	 * EL0, there is no need to check the state of TTBR0_EL1 since	 * accesses are always enabled.	 * Note that the meaning of this bit differs from the ARMv8.1 PAN	 * feature as all TTBR0_EL1 accesses are disabled, not just those to	 * user mappings.	 */alternative_if ARM64_HAS_PAN	b	1f				// skip TTBR0 PANalternative_else_nop_endif 	.if	\el != 0	mrs	x21, ttbr0_el1	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR	b.eq	1f				// TTBR0 access already disabled	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR	.endif 	__uaccess_ttbr0_disable x211:#endif 	stp	x22, x23, [sp, #S_PC] 	/* Not in a syscall by default (el0_svc overwrites for real syscall) */	.if	\el == 0	mov	w21, #NO_SYSCALL	str	w21, [sp, #S_SYSCALLNO]	.endif 	/*	 * Set sp_el0 to current thread_info.	 */	.if	\el == 0	msr	sp_el0, tsk	.endif 	/* Save pmr */alternative_if ARM64_HAS_IRQ_PRIO_MASKING	mrs_s	x20, SYS_ICC_PMR_EL1	str	x20, [sp, #S_PMR_SAVE]alternative_else_nop_endif 	/*	 * Registers that may be useful after this macro is invoked:	 *	 * x21 - aborted SP	 * x22 - aborted PC	 * x23 - aborted PSTATE	*/	.endm 	.macro	kernel_exit, el	.if	\el != 0	disable_daif 	/* Restore the task's original addr_limit. */	ldr	x20, [sp, #S_ORIG_ADDR_LIMIT]	str	x20, [tsk, #TSK_TI_ADDR_LIMIT] 	/* No need to restore UAO, it will be restored from SPSR_EL1 */	.endif 	/* Restore pmr */alternative_if ARM64_HAS_IRQ_PRIO_MASKING	ldr	x20, [sp, #S_PMR_SAVE]	msr_s	SYS_ICC_PMR_EL1, x20	/* Ensure priority change is seen by redistributor */	dsb	syalternative_else_nop_endif 	ldp	x21, x22, [sp, #S_PC]		// load ELR, SPSR	.if	\el == 0	ct_user_enter	.endif #ifdef CONFIG_ARM64_SW_TTBR0_PAN	/*	 * Restore access to TTBR0_EL1. If returning to EL0, no need for SPSR	 * PAN bit checking.	 */alternative_if ARM64_HAS_PAN	b	2f				// skip TTBR0 PANalternative_else_nop_endif 	.if	\el != 0	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set	.endif 	__uaccess_ttbr0_enable x0, x1 	.if	\el == 0	/*	 * Enable errata workarounds only if returning to user. The only	 * workaround currently required for TTBR0_EL1 changes are for the	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache	 * corruption).	 */	bl	post_ttbr_update_workaround	.endif1:	.if	\el != 0	and	x22, x22, #~PSR_PAN_BIT		// ARMv8.0 CPUs do not understand this bit	.endif2:#endif 	.if	\el == 0	ldr	x23, [sp, #S_SP]		// load return stack pointer	msr	sp_el0, x23	tst	x22, #PSR_MODE32_BIT		// native task?	b.eq	3f #ifdef CONFIG_ARM64_ERRATUM_845719alternative_if ARM64_WORKAROUND_845719#ifdef CONFIG_PID_IN_CONTEXTIDR	mrs	x29, contextidr_el1	msr	contextidr_el1, x29#else	msr contextidr_el1, xzr#endifalternative_else_nop_endif#endif3:#ifdef CONFIG_ARM64_ERRATUM_1418040alternative_if_not ARM64_WORKAROUND_1418040	b	4falternative_else_nop_endif	/*	 * if (x22.mode32 == cntkctl_el1.el0vcten)	 *     cntkctl_el1.el0vcten = ~cntkctl_el1.el0vcten	 */	mrs	x1, cntkctl_el1	eon	x0, x1, x22, lsr #3	tbz	x0, #1, 4f	eor	x1, x1, #2	// ARCH_TIMER_USR_VCT_ACCESS_EN	msr	cntkctl_el1, x14:#endif	apply_ssbd 0, x0, x1	.endif 	msr	elr_el1, x21			// set up the return data	msr	spsr_el1, x22	ldp	x0, x1, [sp, #16 * 0]	ldp	x2, x3, [sp, #16 * 1]	ldp	x4, x5, [sp, #16 * 2]	ldp	x6, x7, [sp, #16 * 3]	ldp	x8, x9, [sp, #16 * 4]	ldp	x10, x11, [sp, #16 * 5]	ldp	x12, x13, [sp, #16 * 6]	ldp	x14, x15, [sp, #16 * 7]	ldp	x16, x17, [sp, #16 * 8]	ldp	x18, x19, [sp, #16 * 9]	ldp	x20, x21, [sp, #16 * 10]	ldp	x22, x23, [sp, #16 * 11]	ldp	x24, x25, [sp, #16 * 12]	ldp	x26, x27, [sp, #16 * 13]	ldp	x28, x29, [sp, #16 * 14]	ldr	lr, [sp, #S_LR]	add	sp, sp, #S_FRAME_SIZE		// restore sp 	.if	\el == 0alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0#ifdef CONFIG_UNMAP_KERNEL_AT_EL0	bne	5f	msr	far_el1, x30	tramp_alias	x30, tramp_exit_native	br	x305:	tramp_alias	x30, tramp_exit_compat	br	x30#endif	.else	eret	.endif	sb	.endm 	.macro	irq_stack_entry	mov	x19, sp			// preserve the original sp 	/*	 * Compare sp with the base of the task stack.	 * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,	 * and should switch to the irq stack.	 */	ldr	x25, [tsk, TSK_STACK]	eor	x25, x25, x19	and	x25, x25, #~(THREAD_SIZE - 1)	cbnz	x25, 9998f 	ldr_this_cpu x25, irq_stack_ptr, x26	mov	x26, #IRQ_STACK_SIZE	add	x26, x25, x26 	/* switch to the irq stack */	mov	sp, x269998:	.endm 	/*	 * x19 should be preserved between irq_stack_entry and	 * irq_stack_exit.	 */	.macro	irq_stack_exit	mov	sp, x19	.endm /* GPRs used by entry code */tsk	.req	x28		// current thread_info /* * Interrupt handling. */	.macro	irq_handler	ldr_l	x1, handle_arch_irq	mov	x0, sp	irq_stack_entry	blr	x1	irq_stack_exit	.endm 	.text /* * Exception vectors. */	.pushsection ".entry.text", "ax" 	.align	11ENTRY(vectors)	kernel_ventry	1, sync_invalid			// Synchronous EL1t	kernel_ventry	1, irq_invalid			// IRQ EL1t	kernel_ventry	1, fiq_invalid			// FIQ EL1t	kernel_ventry	1, error_invalid		// Error EL1t 	kernel_ventry	1, sync				// Synchronous EL1h	kernel_ventry	1, irq				// IRQ EL1h	kernel_ventry	1, fiq_invalid			// FIQ EL1h	kernel_ventry	1, error			// Error EL1h 	kernel_ventry	0, sync				// Synchronous 64-bit EL0	kernel_ventry	0, irq				// IRQ 64-bit EL0	kernel_ventry	0, fiq_invalid			// FIQ 64-bit EL0	kernel_ventry	0, error			// Error 64-bit EL0 #ifdef CONFIG_COMPAT	kernel_ventry	0, sync_compat, 32		// Synchronous 32-bit EL0	kernel_ventry	0, irq_compat, 32		// IRQ 32-bit EL0	kernel_ventry	0, fiq_invalid_compat, 32	// FIQ 32-bit EL0	kernel_ventry	0, error_compat, 32		// Error 32-bit EL0#else	kernel_ventry	0, sync_invalid, 32		// Synchronous 32-bit EL0	kernel_ventry	0, irq_invalid, 32		// IRQ 32-bit EL0	kernel_ventry	0, fiq_invalid, 32		// FIQ 32-bit EL0	kernel_ventry	0, error_invalid, 32		// Error 32-bit EL0#endifEND(vectors) #ifdef CONFIG_VMAP_STACK	/*	 * We detected an overflow in kernel_ventry, which switched to the	 * overflow stack. Stash the exception regs, and head to our overflow	 * handler.	 */__bad_stack:	/* Restore the original x0 value */	mrs	x0, tpidrro_el0 	/*	 * Store the original GPRs to the new stack. The orginal SP (minus	 * S_FRAME_SIZE) was stashed in tpidr_el0 by kernel_ventry.	 */	sub	sp, sp, #S_FRAME_SIZE	kernel_entry 1	mrs	x0, tpidr_el0	add	x0, x0, #S_FRAME_SIZE	str	x0, [sp, #S_SP] 	/* Stash the regs for handle_bad_stack */	mov	x0, sp 	/* Time to die */	bl	handle_bad_stack	ASM_BUG()#endif /* CONFIG_VMAP_STACK */ /* * Invalid mode handlers */	.macro	inv_entry, el, reason, regsize = 64	kernel_entry \el, \regsize	mov	x0, sp	mov	x1, #\reason	mrs	x2, esr_el1	bl	bad_mode

Lines 1–500 of 1291. Load more