/* * Copyright 2011 Tilera Corporation. All Rights Reserved. * *   This program is free software; you can redistribute it and/or *   modify it under the terms of the GNU General Public License *   as published by the Free Software Foundation, version 2. * *   This program is distributed in the hope that it will be useful, but *   WITHOUT ANY WARRANTY; without even the implied warranty of *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or *   NON INFRINGEMENT.  See the GNU General Public License for *   more details. * * Linux interrupt vectors. */ #include <linux/linkage.h>#include <linux/errno.h>#include <linux/unistd.h>#include <linux/init.h>#include <asm/ptrace.h>#include <asm/thread_info.h>#include <asm/irqflags.h>#include <asm/asm-offsets.h>#include <asm/types.h>#include <asm/traps.h>#include <asm/signal.h>#include <hv/hypervisor.h>#include <arch/abi.h>#include <arch/interrupts.h>#include <arch/spr_def.h> #define PTREGS_PTR(reg, ptreg) addli reg, sp, C_ABI_SAVE_AREA_SIZE + (ptreg) #define PTREGS_OFFSET_SYSCALL PTREGS_OFFSET_REG(TREG_SYSCALL_NR) #if CONFIG_KERNEL_PL == 1 || CONFIG_KERNEL_PL == 2/* * Set "result" non-zero if ex1 holds the PL of the kernel * (with or without ICS being set).  Note this works only * because we never find the PL at level 3. */# define IS_KERNEL_EX1(result, ex1) andi result, ex1, CONFIG_KERNEL_PL#else# error Recode IS_KERNEL_EX1 for CONFIG_KERNEL_PL#endif 	.macro  push_reg reg, ptr=sp, delta=-8	{	 st     \ptr, \reg	 addli  \ptr, \ptr, \delta	}	.endm 	.macro  pop_reg reg, ptr=sp, delta=8	{	 ld     \reg, \ptr	 addli  \ptr, \ptr, \delta	}	.endm 	.macro  pop_reg_zero reg, zreg, ptr=sp, delta=8	{	 move   \zreg, zero	 ld     \reg, \ptr	 addi   \ptr, \ptr, \delta	}	.endm 	.macro  push_extra_callee_saves reg	PTREGS_PTR(\reg, PTREGS_OFFSET_REG(51))	push_reg r51, \reg	push_reg r50, \reg	push_reg r49, \reg	push_reg r48, \reg	push_reg r47, \reg	push_reg r46, \reg	push_reg r45, \reg	push_reg r44, \reg	push_reg r43, \reg	push_reg r42, \reg	push_reg r41, \reg	push_reg r40, \reg	push_reg r39, \reg	push_reg r38, \reg	push_reg r37, \reg	push_reg r36, \reg	push_reg r35, \reg	push_reg r34, \reg, PTREGS_OFFSET_BASE - PTREGS_OFFSET_REG(34)	.endm 	.macro  panic str	.pushsection .rodata, "a"1:	.asciz  "\str"	.popsection	{	 moveli r0, hw2_last(1b)	}	{	 shl16insli r0, r0, hw1(1b)	}	{	 shl16insli r0, r0, hw0(1b)	 jal    panic	}	.endm 	/*	 * Unalign data exception fast handling: In order to handle	 * unaligned data access, a fast JIT version is generated and stored	 * in a specific area in user space. We first need to do a quick poke	 * to see if the JIT is available. We use certain bits in the fault	 * PC (3 to 9 is used for 16KB page size) as index to address the JIT	 * code area. The first 64bit word is the fault PC, and the 2nd one is	 * the fault bundle itself. If these 2 words both match, then we	 * directly "iret" to JIT code. If not, a slow path is invoked to	 * generate new JIT code. Note: the current JIT code WILL be	 * overwritten if it existed. So, ideally we can handle 128 unalign	 * fixups via JIT. For lookup efficiency and to effectively support	 * tight loops with multiple unaligned reference, a simple	 * direct-mapped cache is used.	 *	 * SPR_EX_CONTEXT_K_0 is modified to return to JIT code.	 * SPR_EX_CONTEXT_K_1 has ICS set.	 * SPR_EX_CONTEXT_0_0 is setup to user program's next PC.	 * SPR_EX_CONTEXT_0_1 = 0.	 */	.macro int_hand_unalign_fast  vecnum, vecname	.org  (\vecnum << 8)intvec_\vecname:	/* Put r3 in SPR_SYSTEM_SAVE_K_1.  */	mtspr   SPR_SYSTEM_SAVE_K_1, r3 	mfspr   r3, SPR_EX_CONTEXT_K_1	/*	 * Examine if exception comes from user without ICS set.	 * If not, just go directly to the slow path.	 */	bnez    r3, hand_unalign_slow_nonuser 	mfspr   r3, SPR_SYSTEM_SAVE_K_0 	/* Get &thread_info->unalign_jit_tmp[0] in r3. */	bfexts  r3, r3, 0, CPU_SHIFT-1	mm      r3, zero, LOG2_THREAD_SIZE, 63	addli   r3, r3, THREAD_INFO_UNALIGN_JIT_TMP_OFFSET 	/*	 * Save r0, r1, r2 into thread_info array r3 points to	 * from low to high memory in order.	 */	st_add  r3, r0, 8	st_add  r3, r1, 8	{	 st_add r3, r2, 8	 andi   r2, sp, 7	} 	/* Save stored r3 value so we can revert it on a page fault. */	mfspr   r1, SPR_SYSTEM_SAVE_K_1	st      r3, r1 	{	 /* Generate a SIGBUS if sp is not 8-byte aligned. */	 bnez   r2, hand_unalign_slow_badsp	} 	/*	 * Get the thread_info in r0; load r1 with pc. Set the low bit of sp	 * as an indicator to the page fault code in case we fault.	 */	{	 ori    sp, sp, 1	 mfspr  r1, SPR_EX_CONTEXT_K_0	} 	/* Add the jit_info offset in thread_info; extract r1 [3:9] into r2. */	{	 addli  r0, r3, THREAD_INFO_UNALIGN_JIT_BASE_OFFSET - \	  (THREAD_INFO_UNALIGN_JIT_TMP_OFFSET + (3 * 8))	 bfextu r2, r1, 3, (2 + PAGE_SHIFT - UNALIGN_JIT_SHIFT)	} 	/* Load the jit_info; multiply r2 by 128. */	{	 ld     r0, r0	 shli   r2, r2, UNALIGN_JIT_SHIFT	} 	/*	 * If r0 is NULL, the JIT page is not mapped, so go to slow path;	 * add offset r2 to r0 at the same time.	 */	{	 beqz   r0, hand_unalign_slow	 add    r2, r0, r2	}         /*	 * We are loading from userspace (both the JIT info PC and	 * instruction word, and the instruction word we executed)	 * and since either could fault while holding the interrupt	 * critical section, we must tag this region and check it in	 * do_page_fault() to handle it properly.	 */ENTRY(__start_unalign_asm_code) 	/* Load first word of JIT in r0 and increment r2 by 8. */	ld_add  r0, r2, 8 	/*	 * Compare the PC with the 1st word in JIT; load the fault bundle	 * into r1.	 */	{	 cmpeq  r0, r0, r1	 ld     r1, r1	} 	/* Go to slow path if PC doesn't match. */	beqz    r0, hand_unalign_slow 	/*	 * Load the 2nd word of JIT, which is supposed to be the fault	 * bundle for a cache hit. Increment r2; after this bundle r2 will	 * point to the potential start of the JIT code we want to run.	 */	ld_add  r0, r2, 8 	/* No further accesses to userspace are done after this point. */ENTRY(__end_unalign_asm_code) 	/* Compare the real bundle with what is saved in the JIT area. */	{	 cmpeq  r0, r1, r0	 mtspr  SPR_EX_CONTEXT_0_1, zero	} 	/* Go to slow path if the fault bundle does not match. */	beqz    r0, hand_unalign_slow 	/*	 * A cache hit is found.	 * r2 points to start of JIT code (3rd word).	 * r0 is the fault pc.	 * r1 is the fault bundle.	 * Reset the low bit of sp.	 */	{	 mfspr  r0, SPR_EX_CONTEXT_K_0	 andi   sp, sp, ~1	} 	/* Write r2 into EX_CONTEXT_K_0 and increment PC. */	{	 mtspr  SPR_EX_CONTEXT_K_0, r2	 addi   r0, r0, 8	} 	/*	 * Set ICS on kernel EX_CONTEXT_K_1 in order to "iret" to	 * user with ICS set. This way, if the JIT fixup causes another	 * unalign exception (which shouldn't be possible) the user	 * process will be terminated with SIGBUS. Also, our fixup will	 * run without interleaving with external interrupts.	 * Each fixup is at most 14 bundles, so it won't hold ICS for long.	 */	{	 movei  r1, PL_ICS_EX1(USER_PL, 1)	 mtspr  SPR_EX_CONTEXT_0_0, r0	} 	{	 mtspr  SPR_EX_CONTEXT_K_1, r1	 addi   r3, r3, -(3 * 8)	} 	/* Restore r0..r3. */	ld_add  r0, r3, 8	ld_add  r1, r3, 8	ld_add  r2, r3, 8	ld      r3, r3 	iret	ENDPROC(intvec_\vecname)	.endm #ifdef __COLLECT_LINKER_FEEDBACK__	.pushsection .text.intvec_feedback,"ax"intvec_feedback:	.popsection#endif 	/*	 * Default interrupt handler.	 *	 * vecnum is where we'll put this code.	 * c_routine is the C routine we'll call.	 *	 * The C routine is passed two arguments:	 * - A pointer to the pt_regs state.	 * - The interrupt vector number.	 *	 * The "processing" argument specifies the code for processing	 * the interrupt. Defaults to "handle_interrupt".	 */	.macro __int_hand vecnum, vecname, c_routine,processing=handle_interruptintvec_\vecname:	/* Temporarily save a register so we have somewhere to work. */ 	mtspr   SPR_SYSTEM_SAVE_K_1, r0	mfspr   r0, SPR_EX_CONTEXT_K_1 	/*	 * The unalign data fastpath code sets the low bit in sp to	 * force us to reset it here on fault.	 */	{	 blbs   sp, 2f	 IS_KERNEL_EX1(r0, r0)	} 	.ifc    \vecnum, INT_DOUBLE_FAULT	/*	 * For double-faults from user-space, fall through to the normal	 * register save and stack setup path.  Otherwise, it's the	 * hypervisor giving us one last chance to dump diagnostics, and we	 * branch to the kernel_double_fault routine to do so.	 */	beqz    r0, 1f	j       _kernel_double_fault1:	.else	/*	 * If we're coming from user-space, then set sp to the top of	 * the kernel stack.  Otherwise, assume sp is already valid.	 */	{	 bnez   r0, 0f	 move   r0, sp	}	.endif 	.ifc    \c_routine, do_page_fault	/*	 * The page_fault handler may be downcalled directly by the	 * hypervisor even when Linux is running and has ICS set.	 *	 * In this case the contents of EX_CONTEXT_K_1 reflect the	 * previous fault and can't be relied on to choose whether or	 * not to reinitialize the stack pointer.  So we add a test	 * to see whether SYSTEM_SAVE_K_2 has the high bit set,	 * and if so we don't reinitialize sp, since we must be coming	 * from Linux.  (In fact the precise case is !(val & ~1),	 * but any Linux PC has to have the high bit set.)	 *	 * Note that the hypervisor *always* sets SYSTEM_SAVE_K_2 for	 * any path that turns into a downcall to one of our TLB handlers.	 *	 * FIXME: if we end up never using this path, perhaps we should	 * prevent the hypervisor from generating downcalls in this case.	 * The advantage of getting a downcall is we can panic in Linux.	 */	mfspr   r0, SPR_SYSTEM_SAVE_K_2	{	 bltz   r0, 0f    /* high bit in S_S_1_2 is for a PC to use */	 move   r0, sp	}	.endif 2:	/*	 * SYSTEM_SAVE_K_0 holds the cpu number in the high bits, and	 * the current stack top in the lower bits.  So we recover	 * our starting stack value by sign-extending the low bits, then	 * point sp at the top aligned address on the actual stack page.	 */	mfspr   r0, SPR_SYSTEM_SAVE_K_0	bfexts  r0, r0, 0, CPU_SHIFT-1 0:	/*	 * Align the stack mod 64 so we can properly predict what	 * cache lines we need to write-hint to reduce memory fetch	 * latency as we enter the kernel.  The layout of memory is	 * as follows, with cache line 0 at the lowest VA, and cache	 * line 8 just below the r0 value this "andi" computes.	 * Note that we never write to cache line 8, and we skip	 * cache lines 1-3 for syscalls.	 *	 *    cache line 8: ptregs padding (two words)	 *    cache line 7: sp, lr, pc, ex1, faultnum, orig_r0, flags, cmpexch	 *    cache line 6: r46...r53 (tp)	 *    cache line 5: r38...r45	 *    cache line 4: r30...r37	 *    cache line 3: r22...r29	 *    cache line 2: r14...r21	 *    cache line 1: r6...r13	 *    cache line 0: 2 x frame, r0..r5	 */#if STACK_TOP_DELTA != 64#error STACK_TOP_DELTA must be 64 for assumptions here and in task_pt_regs()#endif	andi    r0, r0, -64 	/*	 * Push the first four registers on the stack, so that we can set	 * them to vector-unique values before we jump to the common code.	 *	 * Registers are pushed on the stack as a struct pt_regs,	 * with the sp initially just above the struct, and when we're	 * done, sp points to the base of the struct, minus	 * C_ABI_SAVE_AREA_SIZE, so we can directly jal to C code.	 *	 * This routine saves just the first four registers, plus the	 * stack context so we can do proper backtracing right away,	 * and defers to handle_interrupt to save the rest.	 * The backtracer needs pc, ex1, lr, sp, r52, and faultnum,	 * and needs sp set to its final location at the bottom of	 * the stack frame.	 */	addli   r0, r0, PTREGS_OFFSET_LR - (PTREGS_SIZE + KSTK_PTREGS_GAP)	wh64    r0   /* cache line 7 */	{	 st     r0, lr	 addli  r0, r0, PTREGS_OFFSET_SP - PTREGS_OFFSET_LR	}	{	 st     r0, sp	 addli  sp, r0, PTREGS_OFFSET_REG(52) - PTREGS_OFFSET_SP	}	wh64    sp   /* cache line 6 */	{	 st     sp, r52	 addli  sp, sp, PTREGS_OFFSET_REG(1) - PTREGS_OFFSET_REG(52)	}	wh64    sp   /* cache line 0 */	{	 st     sp, r1	 addli  sp, sp, PTREGS_OFFSET_REG(2) - PTREGS_OFFSET_REG(1)	}	{	 st     sp, r2	 addli  sp, sp, PTREGS_OFFSET_REG(3) - PTREGS_OFFSET_REG(2)	}	{	 st     sp, r3	 addli  sp, sp, PTREGS_OFFSET_PC - PTREGS_OFFSET_REG(3)	}	mfspr   r0, SPR_EX_CONTEXT_K_0	.ifc \processing,handle_syscall	/*	 * Bump the saved PC by one bundle so that when we return, we won't	 * execute the same swint instruction again.  We need to do this while	 * we're in the critical section.	 */	addi    r0, r0, 8	.endif	{	 st     sp, r0	 addli  sp, sp, PTREGS_OFFSET_EX1 - PTREGS_OFFSET_PC	}	mfspr   r0, SPR_EX_CONTEXT_K_1	{	 st     sp, r0	 addi   sp, sp, PTREGS_OFFSET_FAULTNUM - PTREGS_OFFSET_EX1	/*	 * Use r0 for syscalls so it's a temporary; use r1 for interrupts	 * so that it gets passed through unchanged to the handler routine.	 * Note that the .if conditional confusingly spans bundles.	 */	 .ifc \processing,handle_syscall	 movei  r0, \vecnum	}	{	 st     sp, r0	 .else	 movei  r1, \vecnum	}	{	 st     sp, r1	 .endif	 addli  sp, sp, PTREGS_OFFSET_REG(0) - PTREGS_OFFSET_FAULTNUM	}	mfspr   r0, SPR_SYSTEM_SAVE_K_1    /* Original r0 */	{	 st     sp, r0	 addi   sp, sp, -PTREGS_OFFSET_REG(0) - 8	}	{	 st     sp, zero        /* write zero into "Next SP" frame pointer */	 addi   sp, sp, -8      /* leave SP pointing at bottom of frame */	}	.ifc \processing,handle_syscall	j       handle_syscall	.else	/* Capture per-interrupt SPR context to registers. */	.ifc \c_routine, do_page_fault	mfspr   r2, SPR_SYSTEM_SAVE_K_3   /* address of page fault */

Lines 1–500 of 1563. Load more