/* SPDX-License-Identifier: GPL-2.0-or-later *//* * Linux/PA-RISC Project (http://www.parisc-linux.org/) * * kernel entry points (interruptions, system call wrappers) * Copyright (C) 1999,2000 Philipp Rumpf * Copyright (C) 1999 SuSE GmbH Nuernberg * Copyright (C) 2000 Hewlett-Packard (John Marvin) * Copyright (C) 1999 Hewlett-Packard (Frank Rowand) */ #include <asm/asm-offsets.h> /* we have the following possibilities to act on an interruption: * - handle in assembly and use shadowed registers only * - save registers to kernel stack and handle in assembly or C */ #include <asm/psw.h>#include <asm/cache.h> /* for L1_CACHE_SHIFT */#include <asm/assembly.h> /* for LDREG/STREG defines */#include <asm/signal.h>#include <asm/unistd.h>#include <asm/ldcw.h>#include <asm/traps.h>#include <asm/thread_info.h>#include <asm/alternative.h> #include <linux/linkage.h>#include <linux/pgtable.h> #ifdef CONFIG_64BIT .level 2.0w#else .level 2.0#endif /* Get aligned page_table_lock address for this mm from cr28/tr4 */ .macro get_ptl reg mfctl %cr28,\reg .endm /* space_to_prot macro creates a prot id from a space id */ #if (SPACEID_SHIFT) == 0 .macro space_to_prot spc prot depd,z \spc,62,31,\prot .endm#else .macro space_to_prot spc prot extrd,u \spc,(64 - (SPACEID_SHIFT)),32,\prot .endm#endif /* Switch to virtual mapping, trashing only %r1 */ .macro virt_map /* pcxt_ssm_bug */ rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */ mtsp %r0, %sr4 mtsp %r0, %sr5 mtsp %r0, %sr6 tovirt_r1 %r29 load32 KERNEL_PSW, %r1 rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */ mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r1, %ipsw load32 4f, %r1 mtctl %r1, %cr18 /* Set IIAOQ tail */ ldo 4(%r1), %r1 mtctl %r1, %cr18 /* Set IIAOQ head */ rfir nop4: .endm /* * The "get_stack" macros are responsible for determining the * kernel stack value. * * If sr7 == 0 * Already using a kernel stack, so call the * get_stack_use_r30 macro to push a pt_regs structure * on the stack, and store registers there. * else * Need to set up a kernel stack, so call the * get_stack_use_cr30 macro to set up a pointer * to the pt_regs structure contained within the * task pointer pointed to by cr30. Set the stack * pointer to point to the end of the task structure. * * Note that we use shadowed registers for temps until * we can save %r26 and %r29. %r26 is used to preserve * %r8 (a shadowed register) which temporarily contained * either the fault type ("code") or the eirr. We need * to use a non-shadowed register to carry the value over * the rfir in virt_map. We use %r26 since this value winds * up being passed as the argument to either do_cpu_irq_mask * or handle_interruption. %r29 is used to hold a pointer * the register save area, and once again, it needs to * be a non-shadowed register so that it survives the rfir. * * N.B. TASK_SZ_ALGN and PT_SZ_ALGN include space for a stack frame. */ .macro get_stack_use_cr30 /* we save the registers in the task struct */ copy %r30, %r17 mfctl %cr30, %r1 ldo THREAD_SZ_ALGN(%r1), %r30 mtsp %r0,%sr7 mtsp %r16,%sr3 tophys %r1,%r9 LDREG TI_TASK(%r9), %r1 /* thread_info -> task_struct */ tophys %r1,%r9 ldo TASK_REGS(%r9),%r9 STREG %r17,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) STREG %r26,PT_GR26(%r9) STREG %r16,PT_SR7(%r9) copy %r9,%r29 .endm .macro get_stack_use_r30 /* we put a struct pt_regs on the stack and save the registers there */ tophys %r30,%r9 copy %r30,%r1 ldo PT_SZ_ALGN(%r30),%r30 STREG %r1,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) STREG %r26,PT_GR26(%r9) STREG %r16,PT_SR7(%r9) copy %r9,%r29 .endm .macro rest_stack LDREG PT_GR1(%r29), %r1 LDREG PT_GR30(%r29),%r30 LDREG PT_GR29(%r29),%r29 .endm /* default interruption handler * (calls traps.c:handle_interruption) */ .macro def code b intr_save ldi \code, %r8 .align 32 .endm /* Interrupt interruption handler * (calls irq.c:do_cpu_irq_mask) */ .macro extint code b intr_extint mfsp %sr7,%r16 .align 32 .endm .import os_hpmc, code /* HPMC handler */ .macro hpmc code nop /* must be a NOP, will be patched later */ load32 PA(os_hpmc), %r3 bv,n 0(%r3) nop .word 0 /* checksum (will be patched) */ .word 0 /* address of handler */ .word 0 /* length of handler */ .endm /* * Performance Note: Instructions will be moved up into * this part of the code later on, once we are sure * that the tlb miss handlers are close to final form. */ /* Register definitions for tlb miss handler macros */ va = r8 /* virtual address for which the trap occurred */ spc = r24 /* space for which the trap occurred */ #ifndef CONFIG_64BIT /* * itlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro itlb_11 code mfctl %pcsq, spc b itlb_miss_11 mfctl %pcoq, va .align 32 .endm#endif /* * itlb miss interruption handler (parisc 2.0) */ .macro itlb_20 code mfctl %pcsq, spc#ifdef CONFIG_64BIT b itlb_miss_20w#else b itlb_miss_20#endif mfctl %pcoq, va .align 32 .endm #ifndef CONFIG_64BIT /* * naitlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro naitlb_11 code mfctl %isr,spc b naitlb_miss_11 mfctl %ior,va .align 32 .endm#endif /* * naitlb miss interruption handler (parisc 2.0) */ .macro naitlb_20 code mfctl %isr,spc#ifdef CONFIG_64BIT b naitlb_miss_20w#else b naitlb_miss_20#endif mfctl %ior,va .align 32 .endm #ifndef CONFIG_64BIT /* * dtlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro dtlb_11 code mfctl %isr, spc b dtlb_miss_11 mfctl %ior, va .align 32 .endm#endif /* * dtlb miss interruption handler (parisc 2.0) */ .macro dtlb_20 code mfctl %isr, spc#ifdef CONFIG_64BIT b dtlb_miss_20w#else b dtlb_miss_20#endif mfctl %ior, va .align 32 .endm #ifndef CONFIG_64BIT /* nadtlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro nadtlb_11 code mfctl %isr,spc b nadtlb_miss_11 mfctl %ior,va .align 32 .endm#endif /* nadtlb miss interruption handler (parisc 2.0) */ .macro nadtlb_20 code mfctl %isr,spc#ifdef CONFIG_64BIT b nadtlb_miss_20w#else b nadtlb_miss_20#endif mfctl %ior,va .align 32 .endm #ifndef CONFIG_64BIT /* * dirty bit trap interruption handler (parisc 1.1 - 32 bit) */ .macro dbit_11 code mfctl %isr,spc b dbit_trap_11 mfctl %ior,va .align 32 .endm#endif /* * dirty bit trap interruption handler (parisc 2.0) */ .macro dbit_20 code mfctl %isr,spc#ifdef CONFIG_64BIT b dbit_trap_20w#else b dbit_trap_20#endif mfctl %ior,va .align 32 .endm /* In LP64, the space contains part of the upper 32 bits of the * fault. We have to extract this and place it in the va, * zeroing the corresponding bits in the space register */ .macro space_adjust spc,va,tmp#ifdef CONFIG_64BIT extrd,u \spc,63,SPACEID_SHIFT,\tmp depd %r0,63,SPACEID_SHIFT,\spc depd \tmp,31,SPACEID_SHIFT,\va#endif .endm .import swapper_pg_dir,code /* Get the pgd. For faults on space zero (kernel space), this * is simply swapper_pg_dir. For user space faults, the * pgd is stored in %cr25 */ .macro get_pgd spc,reg ldil L%PA(swapper_pg_dir),\reg ldo R%PA(swapper_pg_dir)(\reg),\reg or,COND(=) %r0,\spc,%r0 mfctl %cr25,\reg .endm /* space_check(spc,tmp,fault) spc - The space we saw the fault with. tmp - The place to store the current space. fault - Function to call on failure. Only allow faults on different spaces from the currently active one if we're the kernel */ .macro space_check spc,tmp,fault mfsp %sr7,\tmp /* check against %r0 which is same value as LINUX_GATEWAY_SPACE */ or,COND(<>) %r0,\spc,%r0 /* user may execute gateway page * as kernel, so defeat the space * check if it is */ copy \spc,\tmp or,COND(=) %r0,\tmp,%r0 /* nullify if executing as kernel */ cmpb,COND(<>),n \tmp,\spc,\fault .endm /* Look up a PTE in a 2-Level scheme (faulting at each * level if the entry isn't present * * NOTE: we use ldw even for LP64, since the short pointers * can address up to 1TB */ .macro L2_ptep pmd,pte,index,va,fault#if CONFIG_PGTABLE_LEVELS == 3 extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index#else# if defined(CONFIG_64BIT) extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index #else # if PAGE_SIZE > 4096 extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index # else extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index # endif# endif#endif dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */#if CONFIG_PGTABLE_LEVELS < 3 copy %r0,\pte#endif ldw,s \index(\pmd),\pmd bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */ SHLREG \pmd,PxD_VALUE_SHIFT,\pmd extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */ .endm /* Look up PTE in a 3-Level scheme. */ .macro L3_ptep pgd,pte,index,va,fault#if CONFIG_PGTABLE_LEVELS == 3 copy %r0,\pte extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index ldw,s \index(\pgd),\pgd bb,>=,n \pgd,_PxD_PRESENT_BIT,\fault shld \pgd,PxD_VALUE_SHIFT,\pgd#endif L2_ptep \pgd,\pte,\index,\va,\fault .endm /* Acquire page_table_lock and check page is present. */ .macro ptl_lock spc,ptp,pte,tmp,tmp1,fault#ifdef CONFIG_TLB_PTLOCK98: cmpib,COND(=),n 0,\spc,2f get_ptl \tmp1: LDCW 0(\tmp),\tmp1 cmpib,COND(=) 0,\tmp1,1b nop LDREG 0(\ptp),\pte bb,<,n \pte,_PAGE_PRESENT_BIT,3f b \fault stw \spc,0(\tmp)99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)#endif2: LDREG 0(\ptp),\pte bb,>=,n \pte,_PAGE_PRESENT_BIT,\fault3: .endm /* Release page_table_lock without reloading lock address. Note that the values in the register spc are limited to NR_SPACE_IDS (262144). Thus, the stw instruction always stores a nonzero value even when register spc is 64 bits. We use an ordered store to ensure all prior accesses are performed prior to releasing the lock. */ .macro ptl_unlock0 spc,tmp#ifdef CONFIG_TLB_PTLOCK98: or,COND(=) %r0,\spc,%r0 stw,ma \spc,0(\tmp)99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)#endif .endm /* Release page_table_lock. */ .macro ptl_unlock1 spc,tmp#ifdef CONFIG_TLB_PTLOCK98: get_ptl \tmp ptl_unlock0 \spc,\tmp99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP)#endif .endm /* Set the _PAGE_ACCESSED bit of the PTE. Be clever and * don't needlessly dirty the cache line if it was already set */ .macro update_accessed ptp,pte,tmp,tmp1 ldi _PAGE_ACCESSED,\tmp1 or \tmp1,\pte,\tmp and,COND(<>) \tmp1,\pte,%r0 STREG \tmp,0(\ptp) .endm /* Set the dirty bit (and accessed bit). No need to be * clever, this is only used from the dirty fault */ .macro update_dirty ptp,pte,tmp ldi _PAGE_ACCESSED|_PAGE_DIRTY,\tmp or \tmp,\pte,\pte STREG \pte,0(\ptp) .endm /* We have (depending on the page size): * - 38 to 52-bit Physical Page Number * - 12 to 26-bit page offset */ /* bitshift difference between a PFN (based on kernel's PAGE_SIZE) * to a CPU TLB 4k PFN (4k => 12 bits to shift) */ #define PAGE_ADD_SHIFT (PAGE_SHIFT-12) #define PAGE_ADD_HUGE_SHIFT (REAL_HPAGE_SHIFT-12) Lines 1–500 of 2416. Load more
/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Linux/PA-RISC Project (http://www.parisc-linux.org/) * * kernel entry points (interruptions, system call wrappers) * Copyright (C) 1999,2000 Philipp Rumpf * Copyright (C) 1999 SuSE GmbH Nuernberg * Copyright (C) 2000 Hewlett-Packard (John Marvin) * Copyright (C) 1999 Hewlett-Packard (Frank Rowand) */ #include <asm/asm-offsets.h> /* we have the following possibilities to act on an interruption: * - handle in assembly and use shadowed registers only * - save registers to kernel stack and handle in assembly or C */ #include <asm/psw.h> #include <asm/cache.h> /* for L1_CACHE_SHIFT */ #include <asm/assembly.h> /* for LDREG/STREG defines */ #include <asm/signal.h> #include <asm/unistd.h> #include <asm/ldcw.h> #include <asm/traps.h> #include <asm/thread_info.h> #include <asm/alternative.h> #include <linux/linkage.h> #include <linux/pgtable.h> #ifdef CONFIG_64BIT .level 2.0w #else .level 2.0 #endif /* Get aligned page_table_lock address for this mm from cr28/tr4 */ .macro get_ptl reg mfctl %cr28,\reg .endm /* space_to_prot macro creates a prot id from a space id */ #if (SPACEID_SHIFT) == 0 .macro space_to_prot spc prot depd,z \spc,62,31,\prot .endm #else .macro space_to_prot spc prot extrd,u \spc,(64 - (SPACEID_SHIFT)),32,\prot .endm #endif /* Switch to virtual mapping, trashing only %r1 */ .macro virt_map /* pcxt_ssm_bug */ rsm PSW_SM_I, %r0 /* barrier for "Relied upon Translation */ mtsp %r0, %sr4 mtsp %r0, %sr5 mtsp %r0, %sr6 tovirt_r1 %r29 load32 KERNEL_PSW, %r1 rsm PSW_SM_QUIET,%r0 /* second "heavy weight" ctl op */ mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r1, %ipsw load32 4f, %r1 mtctl %r1, %cr18 /* Set IIAOQ tail */ ldo 4(%r1), %r1 mtctl %r1, %cr18 /* Set IIAOQ head */ rfir nop 4: .endm /* * The "get_stack" macros are responsible for determining the * kernel stack value. * * If sr7 == 0 * Already using a kernel stack, so call the * get_stack_use_r30 macro to push a pt_regs structure * on the stack, and store registers there. * else * Need to set up a kernel stack, so call the * get_stack_use_cr30 macro to set up a pointer * to the pt_regs structure contained within the * task pointer pointed to by cr30. Set the stack * pointer to point to the end of the task structure. * * Note that we use shadowed registers for temps until * we can save %r26 and %r29. %r26 is used to preserve * %r8 (a shadowed register) which temporarily contained * either the fault type ("code") or the eirr. We need * to use a non-shadowed register to carry the value over * the rfir in virt_map. We use %r26 since this value winds * up being passed as the argument to either do_cpu_irq_mask * or handle_interruption. %r29 is used to hold a pointer * the register save area, and once again, it needs to * be a non-shadowed register so that it survives the rfir. * * N.B. TASK_SZ_ALGN and PT_SZ_ALGN include space for a stack frame. */ .macro get_stack_use_cr30 /* we save the registers in the task struct */ copy %r30, %r17 mfctl %cr30, %r1 ldo THREAD_SZ_ALGN(%r1), %r30 mtsp %r0,%sr7 mtsp %r16,%sr3 tophys %r1,%r9 LDREG TI_TASK(%r9), %r1 /* thread_info -> task_struct */ tophys %r1,%r9 ldo TASK_REGS(%r9),%r9 STREG %r17,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) STREG %r26,PT_GR26(%r9) STREG %r16,PT_SR7(%r9) copy %r9,%r29 .endm .macro get_stack_use_r30 /* we put a struct pt_regs on the stack and save the registers there */ tophys %r30,%r9 copy %r30,%r1 ldo PT_SZ_ALGN(%r30),%r30 STREG %r1,PT_GR30(%r9) STREG %r29,PT_GR29(%r9) STREG %r26,PT_GR26(%r9) STREG %r16,PT_SR7(%r9) copy %r9,%r29 .endm .macro rest_stack LDREG PT_GR1(%r29), %r1 LDREG PT_GR30(%r29),%r30 LDREG PT_GR29(%r29),%r29 .endm /* default interruption handler * (calls traps.c:handle_interruption) */ .macro def code b intr_save ldi \code, %r8 .align 32 .endm /* Interrupt interruption handler * (calls irq.c:do_cpu_irq_mask) */ .macro extint code b intr_extint mfsp %sr7,%r16 .align 32 .endm .import os_hpmc, code /* HPMC handler */ .macro hpmc code nop /* must be a NOP, will be patched later */ load32 PA(os_hpmc), %r3 bv,n 0(%r3) nop .word 0 /* checksum (will be patched) */ .word 0 /* address of handler */ .word 0 /* length of handler */ .endm /* * Performance Note: Instructions will be moved up into * this part of the code later on, once we are sure * that the tlb miss handlers are close to final form. */ /* Register definitions for tlb miss handler macros */ va = r8 /* virtual address for which the trap occurred */ spc = r24 /* space for which the trap occurred */ #ifndef CONFIG_64BIT /* * itlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro itlb_11 code mfctl %pcsq, spc b itlb_miss_11 mfctl %pcoq, va .align 32 .endm #endif /* * itlb miss interruption handler (parisc 2.0) */ .macro itlb_20 code mfctl %pcsq, spc #ifdef CONFIG_64BIT b itlb_miss_20w #else b itlb_miss_20 #endif mfctl %pcoq, va .align 32 .endm #ifndef CONFIG_64BIT /* * naitlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro naitlb_11 code mfctl %isr,spc b naitlb_miss_11 mfctl %ior,va .align 32 .endm #endif /* * naitlb miss interruption handler (parisc 2.0) */ .macro naitlb_20 code mfctl %isr,spc #ifdef CONFIG_64BIT b naitlb_miss_20w #else b naitlb_miss_20 #endif mfctl %ior,va .align 32 .endm #ifndef CONFIG_64BIT /* * dtlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro dtlb_11 code mfctl %isr, spc b dtlb_miss_11 mfctl %ior, va .align 32 .endm #endif /* * dtlb miss interruption handler (parisc 2.0) */ .macro dtlb_20 code mfctl %isr, spc #ifdef CONFIG_64BIT b dtlb_miss_20w #else b dtlb_miss_20 #endif mfctl %ior, va .align 32 .endm #ifndef CONFIG_64BIT /* nadtlb miss interruption handler (parisc 1.1 - 32 bit) */ .macro nadtlb_11 code mfctl %isr,spc b nadtlb_miss_11 mfctl %ior,va .align 32 .endm #endif /* nadtlb miss interruption handler (parisc 2.0) */ .macro nadtlb_20 code mfctl %isr,spc #ifdef CONFIG_64BIT b nadtlb_miss_20w #else b nadtlb_miss_20 #endif mfctl %ior,va .align 32 .endm #ifndef CONFIG_64BIT /* * dirty bit trap interruption handler (parisc 1.1 - 32 bit) */ .macro dbit_11 code mfctl %isr,spc b dbit_trap_11 mfctl %ior,va .align 32 .endm #endif /* * dirty bit trap interruption handler (parisc 2.0) */ .macro dbit_20 code mfctl %isr,spc #ifdef CONFIG_64BIT b dbit_trap_20w #else b dbit_trap_20 #endif mfctl %ior,va .align 32 .endm /* In LP64, the space contains part of the upper 32 bits of the * fault. We have to extract this and place it in the va, * zeroing the corresponding bits in the space register */ .macro space_adjust spc,va,tmp #ifdef CONFIG_64BIT extrd,u \spc,63,SPACEID_SHIFT,\tmp depd %r0,63,SPACEID_SHIFT,\spc depd \tmp,31,SPACEID_SHIFT,\va #endif .endm .import swapper_pg_dir,code /* Get the pgd. For faults on space zero (kernel space), this * is simply swapper_pg_dir. For user space faults, the * pgd is stored in %cr25 */ .macro get_pgd spc,reg ldil L%PA(swapper_pg_dir),\reg ldo R%PA(swapper_pg_dir)(\reg),\reg or,COND(=) %r0,\spc,%r0 mfctl %cr25,\reg .endm /* space_check(spc,tmp,fault) spc - The space we saw the fault with. tmp - The place to store the current space. fault - Function to call on failure. Only allow faults on different spaces from the currently active one if we're the kernel */ .macro space_check spc,tmp,fault mfsp %sr7,\tmp /* check against %r0 which is same value as LINUX_GATEWAY_SPACE */ or,COND(<>) %r0,\spc,%r0 /* user may execute gateway page * as kernel, so defeat the space * check if it is */ copy \spc,\tmp or,COND(=) %r0,\tmp,%r0 /* nullify if executing as kernel */ cmpb,COND(<>),n \tmp,\spc,\fault .endm /* Look up a PTE in a 2-Level scheme (faulting at each * level if the entry isn't present * * NOTE: we use ldw even for LP64, since the short pointers * can address up to 1TB */ .macro L2_ptep pmd,pte,index,va,fault #if CONFIG_PGTABLE_LEVELS == 3 extru \va,31-ASM_PMD_SHIFT,ASM_BITS_PER_PMD,\index #else # if defined(CONFIG_64BIT) extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index #else # if PAGE_SIZE > 4096 extru \va,31-ASM_PGDIR_SHIFT,32-ASM_PGDIR_SHIFT,\index # else extru \va,31-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index # endif # endif #endif dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ #if CONFIG_PGTABLE_LEVELS < 3 copy %r0,\pte #endif ldw,s \index(\pmd),\pmd bb,>=,n \pmd,_PxD_PRESENT_BIT,\fault dep %r0,31,PxD_FLAG_SHIFT,\pmd /* clear flags */ SHLREG \pmd,PxD_VALUE_SHIFT,\pmd extru \va,31-PAGE_SHIFT,ASM_BITS_PER_PTE,\index dep %r0,31,PAGE_SHIFT,\pmd /* clear offset */ shladd \index,BITS_PER_PTE_ENTRY,\pmd,\pmd /* pmd is now pte */ .endm /* Look up PTE in a 3-Level scheme. */ .macro L3_ptep pgd,pte,index,va,fault #if CONFIG_PGTABLE_LEVELS == 3 copy %r0,\pte extrd,u \va,63-ASM_PGDIR_SHIFT,ASM_BITS_PER_PGD,\index ldw,s \index(\pgd),\pgd bb,>=,n \pgd,_PxD_PRESENT_BIT,\fault shld \pgd,PxD_VALUE_SHIFT,\pgd #endif L2_ptep \pgd,\pte,\index,\va,\fault .endm /* Acquire page_table_lock and check page is present. */ .macro ptl_lock spc,ptp,pte,tmp,tmp1,fault #ifdef CONFIG_TLB_PTLOCK 98: cmpib,COND(=),n 0,\spc,2f get_ptl \tmp 1: LDCW 0(\tmp),\tmp1 cmpib,COND(=) 0,\tmp1,1b nop LDREG 0(\ptp),\pte bb,<,n \pte,_PAGE_PRESENT_BIT,3f b \fault stw \spc,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif 2: LDREG 0(\ptp),\pte bb,>=,n \pte,_PAGE_PRESENT_BIT,\fault 3: .endm /* Release page_table_lock without reloading lock address. Note that the values in the register spc are limited to NR_SPACE_IDS (262144). Thus, the stw instruction always stores a nonzero value even when register spc is 64 bits. We use an ordered store to ensure all prior accesses are performed prior to releasing the lock. */ .macro ptl_unlock0 spc,tmp #ifdef CONFIG_TLB_PTLOCK 98: or,COND(=) %r0,\spc,%r0 stw,ma \spc,0(\tmp) 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif .endm /* Release page_table_lock. */ .macro ptl_unlock1 spc,tmp #ifdef CONFIG_TLB_PTLOCK 98: get_ptl \tmp ptl_unlock0 \spc,\tmp 99: ALTERNATIVE(98b, 99b, ALT_COND_NO_SMP, INSN_NOP) #endif .endm /* Set the _PAGE_ACCESSED bit of the PTE. Be clever and * don't needlessly dirty the cache line if it was already set */ .macro update_accessed ptp,pte,tmp,tmp1 ldi _PAGE_ACCESSED,\tmp1 or \tmp1,\pte,\tmp and,COND(<>) \tmp1,\pte,%r0 STREG \tmp,0(\ptp) .endm /* Set the dirty bit (and accessed bit). No need to be * clever, this is only used from the dirty fault */ .macro update_dirty ptp,pte,tmp ldi _PAGE_ACCESSED|_PAGE_DIRTY,\tmp or \tmp,\pte,\pte STREG \pte,0(\ptp) .endm /* We have (depending on the page size): * - 38 to 52-bit Physical Page Number * - 12 to 26-bit page offset */ /* bitshift difference between a PFN (based on kernel's PAGE_SIZE) * to a CPU TLB 4k PFN (4k => 12 bits to shift) */ #define PAGE_ADD_SHIFT (PAGE_SHIFT-12) #define PAGE_ADD_HUGE_SHIFT (REAL_HPAGE_SHIFT-12)