/* SPDX-License-Identifier: GPL-2.0-or-later *//* *  PowerPC version *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) *  Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP *    Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com> *  Adapted for Power Macintosh by Paul Mackerras. *  Low-level exception handlers and MMU support *  rewritten by Paul Mackerras. *    Copyright (C) 1996 Paul Mackerras. *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net). * *  This file contains the system call entry code, context switch *  code, and exception/interrupt return code for PowerPC. */ #include <linux/errno.h>#include <linux/err.h>#include <linux/sys.h>#include <linux/threads.h>#include <linux/linkage.h> #include <asm/reg.h>#include <asm/page.h>#include <asm/mmu.h>#include <asm/cputable.h>#include <asm/thread_info.h>#include <asm/ppc_asm.h>#include <asm/asm-offsets.h>#include <asm/unistd.h>#include <asm/ptrace.h>#include <asm/feature-fixups.h>#include <asm/barrier.h>#include <asm/kup.h>#include <asm/bug.h>#include <asm/interrupt.h> #include "head_32.h" /* * powerpc relies on return from interrupt/syscall being context synchronising * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional * synchronisation instructions. */ /* * Align to 4k in order to ensure that all functions modyfing srr0/srr1 * fit into one page in order to not encounter a TLB miss between the * modification of srr0/srr1 and the associated rfi. */	.align	12 #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_E500)	.globl	prepare_transfer_to_handlerprepare_transfer_to_handler:	/* if from kernel, check interrupted DOZE/NAP mode */	lwz	r12,TI_LOCAL_FLAGS(r2)	mtcrf	0x01,r12	bt-	31-TLF_NAPPING,4f	bt-	31-TLF_SLEEPING,7f	blr 4:	rlwinm	r12,r12,0,~_TLF_NAPPING	stw	r12,TI_LOCAL_FLAGS(r2)	b	power_save_ppc32_restore 7:	rlwinm	r12,r12,0,~_TLF_SLEEPING	stw	r12,TI_LOCAL_FLAGS(r2)	lwz	r9,_MSR(r11)		/* if sleeping, clear MSR.EE */	rlwinm	r9,r9,0,~MSR_EE	lwz	r12,_LINK(r11)		/* and return to address in LR */	REST_GPR(2, r11)	b	fast_exception_return_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_PPC_E500 */ #if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)SYM_FUNC_START(__kuep_lock)	lwz	r9, THREAD+THSR0(r2)	update_user_segments_by_4 r9, r10, r11, r12	blrSYM_FUNC_END(__kuep_lock) SYM_FUNC_START_LOCAL(__kuep_unlock)	lwz	r9, THREAD+THSR0(r2)	rlwinm  r9,r9,0,~SR_NX	update_user_segments_by_4 r9, r10, r11, r12	blrSYM_FUNC_END(__kuep_unlock) .macro	kuep_lock	bl	__kuep_lock.endm.macro	kuep_unlock	bl	__kuep_unlock.endm#else.macro	kuep_lock.endm.macro	kuep_unlock.endm#endif 	.globl	transfer_to_syscalltransfer_to_syscall:	stw	r3, ORIG_GPR3(r1)	stw	r11, GPR1(r1)	stw	r11, 0(r1)	mflr	r12	stw	r12, _LINK(r1)#ifdef CONFIG_BOOKE	rlwinm	r9,r9,0,14,12		/* clear MSR_WE (necessary?) */#endif	lis	r12,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */	SAVE_GPR(2, r1)	addi	r12,r12,STACK_FRAME_REGS_MARKER@l	stw	r9,_MSR(r1)	li	r2, INTERRUPT_SYSCALL	stw	r12,STACK_INT_FRAME_MARKER(r1)	stw	r2,_TRAP(r1)	SAVE_GPR(0, r1)	SAVE_GPRS(3, 8, r1)	addi	r2,r10,-THREAD	SAVE_NVGPRS(r1)	kuep_lock 	/* Calling convention has r3 = regs, r4 = orig r0 */	addi	r3,r1,STACK_INT_FRAME_REGS	mr	r4,r0	bl	system_call_exception ret_from_syscall:	addi    r4,r1,STACK_INT_FRAME_REGS	li	r5,0	bl	syscall_exit_prepare#ifdef CONFIG_PPC_47x	lis	r4,icache_44x_need_flush@ha	lwz	r5,icache_44x_need_flush@l(r4)	cmplwi	cr0,r5,0	bne-	.L44x_icache_flush#endif /* CONFIG_PPC_47x */.L44x_icache_flush_return:	kuep_unlock	lwz	r4,_LINK(r1)	lwz	r5,_CCR(r1)	mtlr	r4	lwz	r7,_NIP(r1)	lwz	r8,_MSR(r1)	cmpwi	r3,0	REST_GPR(3, r1)syscall_exit_finish:	mtspr	SPRN_SRR0,r7	mtspr	SPRN_SRR1,r8 	bne	3f	mtcr	r5 1:	REST_GPR(2, r1)	REST_GPR(1, r1)	rfi 3:	mtcr	r5	lwz	r4,_CTR(r1)	lwz	r5,_XER(r1)	REST_NVGPRS(r1)	mtctr	r4	mtxer	r5	REST_GPR(0, r1)	REST_GPRS(3, 12, r1)	b	1b_ASM_NOKPROBE_SYMBOL(syscall_exit_finish) #ifdef CONFIG_44x.L44x_icache_flush:	li	r7,0	iccci	r0,r0	stw	r7,icache_44x_need_flush@l(r4)	b	.L44x_icache_flush_return#endif  /* CONFIG_44x */ 	.globl	ret_from_forkret_from_fork:	REST_NVGPRS(r1)	bl	schedule_tail	li	r3,0	/* fork() return value */	b	ret_from_syscall 	.globl	ret_from_kernel_user_threadret_from_kernel_user_thread:	bl	schedule_tail	mtctr	r14	mr	r3,r15	PPC440EP_ERR42	bctrl	li	r3,0	b	ret_from_syscall 	.globl	start_kernel_threadstart_kernel_thread:	bl	schedule_tail	mtctr	r14	mr	r3,r15	PPC440EP_ERR42	bctrl	/*	 * This must not return. We actually want to BUG here, not WARN,	 * because BUG will exit the process which is what the kernel thread	 * should have done, which may give some hope of continuing.	 */100:	trap	EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0 	.globl	fast_exception_returnfast_exception_return:#ifndef CONFIG_BOOKE	andi.	r10,r9,MSR_RI		/* check for recoverable interrupt */	beq	3f			/* if not, we've got problems */#endif 	lwz	r10,_CCR(r11)	REST_GPRS(1, 6, r11)	mtcr	r10	lwz	r10,_LINK(r11)	mtlr	r10	/* Clear the exception marker on the stack to avoid confusing stacktrace */	li	r10, 0	stw	r10, 8(r11)	mtspr	SPRN_SRR1,r9	mtspr	SPRN_SRR0,r12	REST_GPR(9, r11)	REST_GPR(10, r11)	REST_GPR(12, r11)	REST_GPR(11, r11)	rfi_ASM_NOKPROBE_SYMBOL(fast_exception_return) /* aargh, a nonrecoverable interrupt, panic *//* aargh, we don't know which trap this is */3:	li	r10,-1	stw	r10,_TRAP(r11)	prepare_transfer_to_handler	bl	unrecoverable_exception	trap	/* should not get here */ 	.globl interrupt_returninterrupt_return:	lwz	r4,_MSR(r1)	addi	r3,r1,STACK_INT_FRAME_REGS	andi.	r0,r4,MSR_PR	beq	.Lkernel_interrupt_return	bl	interrupt_exit_user_prepare	cmpwi	r3,0	kuep_unlock	bne-	.Lrestore_nvgprs .Lfast_user_interrupt_return:	lwz	r11,_NIP(r1)	lwz	r12,_MSR(r1)	mtspr	SPRN_SRR0,r11	mtspr	SPRN_SRR1,r12 BEGIN_FTR_SECTION	lwarx   r0,0,r1END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)	stwcx.	r0,0,r1		/* to clear the reservation */ 	lwz	r3,_CCR(r1)	lwz	r4,_LINK(r1)	lwz	r5,_CTR(r1)	lwz	r6,_XER(r1)	li	r0,0 	/*	 * Leaving a stale exception marker on the stack can confuse	 * the reliable stack unwinder later on. Clear it.	 */	stw	r0,8(r1)	REST_GPRS(7, 12, r1) 	mtcr	r3	mtlr	r4	mtctr	r5	mtspr	SPRN_XER,r6 	REST_GPRS(2, 6, r1)	REST_GPR(0, r1)	REST_GPR(1, r1)	rfi .Lrestore_nvgprs:	REST_NVGPRS(r1)	b	.Lfast_user_interrupt_return .Lkernel_interrupt_return:	bl	interrupt_exit_kernel_prepare .Lfast_kernel_interrupt_return:	cmpwi	cr1,r3,0	lwz	r11,_NIP(r1)	lwz	r12,_MSR(r1)	mtspr	SPRN_SRR0,r11	mtspr	SPRN_SRR1,r12 BEGIN_FTR_SECTION	lwarx   r0,0,r1END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)	stwcx.	r0,0,r1		/* to clear the reservation */ 	lwz	r3,_LINK(r1)	lwz	r4,_CTR(r1)	lwz	r5,_XER(r1)	lwz	r6,_CCR(r1)	li	r0,0 	REST_GPRS(7, 12, r1) 	mtlr	r3	mtctr	r4	mtspr	SPRN_XER,r5 	/*	 * Leaving a stale exception marker on the stack can confuse	 * the reliable stack unwinder later on. Clear it.	 */	stw	r0,8(r1) 	REST_GPRS(2, 5, r1) 	bne-	cr1,1f /* emulate stack store */	mtcr	r6	REST_GPR(6, r1)	REST_GPR(0, r1)	REST_GPR(1, r1)	rfi 1:	/*	 * Emulate stack store with update. New r1 value was already calculated	 * and updated in our interrupt regs by emulate_loadstore, but we can't	 * store the previous value of r1 to the stack before re-loading our	 * registers from it, otherwise they could be clobbered.  Use	 * SPRG Scratch0 as temporary storage to hold the store	 * data, as interrupts are disabled here so it won't be clobbered.	 */	mtcr	r6#ifdef CONFIG_BOOKE	mtspr	SPRN_SPRG_WSCRATCH0, r9#else	mtspr	SPRN_SPRG_SCRATCH0, r9#endif	addi	r9,r1,INT_FRAME_SIZE /* get original r1 */	REST_GPR(6, r1)	REST_GPR(0, r1)	REST_GPR(1, r1)	stw	r9,0(r1) /* perform store component of stwu */#ifdef CONFIG_BOOKE	mfspr	r9, SPRN_SPRG_RSCRATCH0#else	mfspr	r9, SPRN_SPRG_SCRATCH0#endif	rfi_ASM_NOKPROBE_SYMBOL(interrupt_return) #ifdef CONFIG_BOOKE /* * Returning from a critical interrupt in user mode doesn't need * to be any different from a normal exception.  For a critical * interrupt in the kernel, we just return (without checking for * preemption) since the interrupt may have happened at some crucial * place (e.g. inside the TLB miss handler), and because we will be * running with r1 pointing into critical_stack, not the current * process's kernel stack (and therefore current_thread_info() will * give the wrong answer). * We have to restore various SPRs that may have been in use at the * time of the critical interrupt. * */ #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi)	\	REST_NVGPRS(r1);						\	lwz	r3,_MSR(r1);						\	andi.	r3,r3,MSR_PR;						\	bne	interrupt_return;					\	REST_GPR(0, r1);						\	REST_GPRS(2, 8, r1);						\	lwz	r10,_XER(r1);						\	lwz	r11,_CTR(r1);						\	mtspr	SPRN_XER,r10;						\	mtctr	r11;							\	stwcx.	r0,0,r1;		/* to clear the reservation */	\	lwz	r11,_LINK(r1);						\	mtlr	r11;							\	lwz	r10,_CCR(r1);						\	mtcrf	0xff,r10;						\	lwz	r9,_DEAR(r1);						\	lwz	r10,_ESR(r1);						\	mtspr	SPRN_DEAR,r9;						\	mtspr	SPRN_ESR,r10;						\	lwz	r11,_NIP(r1);						\	lwz	r12,_MSR(r1);						\	mtspr	exc_lvl_srr0,r11;					\	mtspr	exc_lvl_srr1,r12;					\	REST_GPRS(9, 12, r1);						\	REST_GPR(1, r1);						\	exc_lvl_rfi;							\	b	.;		/* prevent prefetch past exc_lvl_rfi */ #define	RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1)			\	lwz	r9,_##exc_lvl_srr0(r1);					\	lwz	r10,_##exc_lvl_srr1(r1);				\	mtspr	SPRN_##exc_lvl_srr0,r9;					\	mtspr	SPRN_##exc_lvl_srr1,r10; #if defined(CONFIG_PPC_E500)#ifdef CONFIG_PHYS_64BIT#define	RESTORE_MAS7							\	lwz	r11,MAS7(r1);						\	mtspr	SPRN_MAS7,r11;#else#define	RESTORE_MAS7#endif /* CONFIG_PHYS_64BIT */#define RESTORE_MMU_REGS						\	lwz	r9,MAS0(r1);						\	lwz	r10,MAS1(r1);						\	lwz	r11,MAS2(r1);						\	mtspr	SPRN_MAS0,r9;						\	lwz	r9,MAS3(r1);						\	mtspr	SPRN_MAS1,r10;						\	lwz	r10,MAS6(r1);						\	mtspr	SPRN_MAS2,r11;						\	mtspr	SPRN_MAS3,r9;						\	mtspr	SPRN_MAS6,r10;						\	RESTORE_MAS7;#elif defined(CONFIG_44x)#define RESTORE_MMU_REGS						\	lwz	r9,MMUCR(r1);						\	mtspr	SPRN_MMUCR,r9;#else#define RESTORE_MMU_REGS#endif 	.globl	ret_from_crit_excret_from_crit_exc:	RESTORE_xSRR(SRR0,SRR1);	RESTORE_MMU_REGS;	RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)_ASM_NOKPROBE_SYMBOL(ret_from_crit_exc) 	.globl	ret_from_debug_excret_from_debug_exc:	RESTORE_xSRR(SRR0,SRR1);	RESTORE_xSRR(CSRR0,CSRR1);	RESTORE_MMU_REGS;	RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)_ASM_NOKPROBE_SYMBOL(ret_from_debug_exc) 	.globl	ret_from_mcheck_excret_from_mcheck_exc:	RESTORE_xSRR(SRR0,SRR1);	RESTORE_xSRR(CSRR0,CSRR1);	RESTORE_xSRR(DSRR0,DSRR1);	RESTORE_MMU_REGS;	RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)#endif /* CONFIG_BOOKE */