Add memory management support for LoongArch, including: cache and tlb management, page fault handling and ioremap/mmap support.
Unified · Split
| @@ -0,0 +1,13 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_CACHE_H | ||
| 6 | #define _ASM_CACHE_H | ||
| 7 | |||
| 8 | #define L1_CACHE_SHIFT CONFIG_L1_CACHE_SHIFT | ||
| 9 | #define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) | ||
| 10 | |||
| 11 | #define __read_mostly __section(".data..read_mostly") | ||
| 12 | |||
| 13 | #endif /* _ASM_CACHE_H */ | ||
| @@ -0,0 +1,80 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_CACHEFLUSH_H | ||
| 6 | #define _ASM_CACHEFLUSH_H | ||
| 7 | |||
| 8 | #include <linux/mm.h> | ||
| 9 | #include <asm/cpu-features.h> | ||
| 10 | #include <asm/cacheops.h> | ||
| 11 | |||
| 12 | extern void local_flush_icache_range(unsigned long start, unsigned long end); | ||
| 13 | |||
| 14 | #define flush_icache_range local_flush_icache_range | ||
| 15 | #define flush_icache_user_range local_flush_icache_range | ||
| 16 | |||
| 17 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 | ||
| 18 | |||
| 19 | #define flush_cache_all() do { } while (0) | ||
| 20 | #define flush_cache_mm(mm) do { } while (0) | ||
| 21 | #define flush_cache_dup_mm(mm) do { } while (0) | ||
| 22 | #define flush_cache_range(vma, start, end) do { } while (0) | ||
| 23 | #define flush_cache_page(vma, vmaddr, pfn) do { } while (0) | ||
| 24 | #define flush_cache_vmap(start, end) do { } while (0) | ||
| 25 | #define flush_cache_vunmap(start, end) do { } while (0) | ||
| 26 | #define flush_icache_page(vma, page) do { } while (0) | ||
| 27 | #define flush_icache_user_page(vma, page, addr, len) do { } while (0) | ||
| 28 | #define flush_dcache_page(page) do { } while (0) | ||
| 29 | #define flush_dcache_mmap_lock(mapping) do { } while (0) | ||
| 30 | #define flush_dcache_mmap_unlock(mapping) do { } while (0) | ||
| 31 | |||
| 32 | #define cache_op(op, addr) \ | ||
| 33 | __asm__ __volatile__( \ | ||
| 34 | " cacop %0, %1 \n" \ | ||
| 35 | : \ | ||
| 36 | : "i" (op), "ZC" (*(unsigned char *)(addr))) | ||
| 37 | |||
| 38 | static inline void flush_icache_line_indexed(unsigned long addr) | ||
| 39 | { | ||
| 40 | cache_op(Index_Invalidate_I, addr); | ||
| 41 | } | ||
| 42 | |||
| 43 | static inline void flush_dcache_line_indexed(unsigned long addr) | ||
| 44 | { | ||
| 45 | cache_op(Index_Writeback_Inv_D, addr); | ||
| 46 | } | ||
| 47 | |||
| 48 | static inline void flush_vcache_line_indexed(unsigned long addr) | ||
| 49 | { | ||
| 50 | cache_op(Index_Writeback_Inv_V, addr); | ||
| 51 | } | ||
| 52 | |||
| 53 | static inline void flush_scache_line_indexed(unsigned long addr) | ||
| 54 | { | ||
| 55 | cache_op(Index_Writeback_Inv_S, addr); | ||
| 56 | } | ||
| 57 | |||
| 58 | static inline void flush_icache_line(unsigned long addr) | ||
| 59 | { | ||
| 60 | cache_op(Hit_Invalidate_I, addr); | ||
| 61 | } | ||
| 62 | |||
| 63 | static inline void flush_dcache_line(unsigned long addr) | ||
| 64 | { | ||
| 65 | cache_op(Hit_Writeback_Inv_D, addr); | ||
| 66 | } | ||
| 67 | |||
| 68 | static inline void flush_vcache_line(unsigned long addr) | ||
| 69 | { | ||
| 70 | cache_op(Hit_Writeback_Inv_V, addr); | ||
| 71 | } | ||
| 72 | |||
| 73 | static inline void flush_scache_line(unsigned long addr) | ||
| 74 | { | ||
| 75 | cache_op(Hit_Writeback_Inv_S, addr); | ||
| 76 | } | ||
| 77 | |||
| 78 | #include <asm-generic/cacheflush.h> | ||
| 79 | |||
| 80 | #endif /* _ASM_CACHEFLUSH_H */ | ||
| @@ -0,0 +1,37 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Cache operations for the cache instruction. | ||
| 4 | * | ||
| 5 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 6 | */ | ||
| 7 | #ifndef __ASM_CACHEOPS_H | ||
| 8 | #define __ASM_CACHEOPS_H | ||
| 9 | |||
| 10 | /* | ||
| 11 | * Most cache ops are split into a 2 bit field identifying the cache, and a 3 | ||
| 12 | * bit field identifying the cache operation. | ||
| 13 | */ | ||
| 14 | #define CacheOp_Cache 0x03 | ||
| 15 | #define CacheOp_Op 0x1c | ||
| 16 | |||
| 17 | #define Cache_I 0x00 | ||
| 18 | #define Cache_D 0x01 | ||
| 19 | #define Cache_V 0x02 | ||
| 20 | #define Cache_S 0x03 | ||
| 21 | |||
| 22 | #define Index_Invalidate 0x08 | ||
| 23 | #define Index_Writeback_Inv 0x08 | ||
| 24 | #define Hit_Invalidate 0x10 | ||
| 25 | #define Hit_Writeback_Inv 0x10 | ||
| 26 | #define CacheOp_User_Defined 0x18 | ||
| 27 | |||
| 28 | #define Index_Invalidate_I (Cache_I | Index_Invalidate) | ||
| 29 | #define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv) | ||
| 30 | #define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv) | ||
| 31 | #define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv) | ||
| 32 | #define Hit_Invalidate_I (Cache_I | Hit_Invalidate) | ||
| 33 | #define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv) | ||
| 34 | #define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv) | ||
| 35 | #define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv) | ||
| 36 | |||
| 37 | #endif /* __ASM_CACHEOPS_H */ | ||
| @@ -0,0 +1,13 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * fixmap.h: compile-time virtual memory allocation | ||
| 4 | * | ||
| 5 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 6 | */ | ||
| 7 | |||
| 8 | #ifndef _ASM_FIXMAP_H | ||
| 9 | #define _ASM_FIXMAP_H | ||
| 10 | |||
| 11 | #define NR_FIX_BTMAPS 64 | ||
| 12 | |||
| 13 | #endif | ||
| @@ -0,0 +1,83 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | |||
| 6 | #ifndef __ASM_HUGETLB_H | ||
| 7 | #define __ASM_HUGETLB_H | ||
| 8 | |||
| 9 | #include <asm/page.h> | ||
| 10 | |||
| 11 | uint64_t pmd_to_entrylo(unsigned long pmd_val); | ||
| 12 | |||
| 13 | #define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE | ||
| 14 | static inline int prepare_hugepage_range(struct file *file, | ||
| 15 | unsigned long addr, | ||
| 16 | unsigned long len) | ||
| 17 | { | ||
| 18 | unsigned long task_size = STACK_TOP; | ||
| 19 | struct hstate *h = hstate_file(file); | ||
| 20 | |||
| 21 | if (len & ~huge_page_mask(h)) | ||
| 22 | return -EINVAL; | ||
| 23 | if (addr & ~huge_page_mask(h)) | ||
| 24 | return -EINVAL; | ||
| 25 | if (len > task_size) | ||
| 26 | return -ENOMEM; | ||
| 27 | if (task_size - len < addr) | ||
| 28 | return -EINVAL; | ||
| 29 | return 0; | ||
| 30 | } | ||
| 31 | |||
| 32 | #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR | ||
| 33 | static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | ||
| 34 | unsigned long addr, pte_t *ptep) | ||
| 35 | { | ||
| 36 | pte_t clear; | ||
| 37 | pte_t pte = *ptep; | ||
| 38 | |||
| 39 | pte_val(clear) = (unsigned long)invalid_pte_table; | ||
| 40 | set_pte_at(mm, addr, ptep, clear); | ||
| 41 | return pte; | ||
| 42 | } | ||
| 43 | |||
| 44 | #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH | ||
| 45 | static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, | ||
| 46 | unsigned long addr, pte_t *ptep) | ||
| 47 | { | ||
| 48 | pte_t pte; | ||
| 49 | |||
| 50 | pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); | ||
| 51 | flush_tlb_page(vma, addr); | ||
| 52 | return pte; | ||
| 53 | } | ||
| 54 | |||
| 55 | #define __HAVE_ARCH_HUGE_PTE_NONE | ||
| 56 | static inline int huge_pte_none(pte_t pte) | ||
| 57 | { | ||
| 58 | unsigned long val = pte_val(pte) & ~_PAGE_GLOBAL; | ||
| 59 | return !val || (val == (unsigned long)invalid_pte_table); | ||
| 60 | } | ||
| 61 | |||
| 62 | #define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS | ||
| 63 | static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, | ||
| 64 | unsigned long addr, | ||
| 65 | pte_t *ptep, pte_t pte, | ||
| 66 | int dirty) | ||
| 67 | { | ||
| 68 | int changed = !pte_same(*ptep, pte); | ||
| 69 | |||
| 70 | if (changed) { | ||
| 71 | set_pte_at(vma->vm_mm, addr, ptep, pte); | ||
| 72 | /* | ||
| 73 | * There could be some standard sized pages in there, | ||
| 74 | * get them all. | ||
| 75 | */ | ||
| 76 | flush_tlb_range(vma, addr, addr + HPAGE_SIZE); | ||
| 77 | } | ||
| 78 | return changed; | ||
| 79 | } | ||
| 80 | |||
| 81 | #include <asm-generic/hugetlb.h> | ||
| 82 | |||
| 83 | #endif /* __ASM_HUGETLB_H */ | ||
| @@ -0,0 +1,115 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_PAGE_H | ||
| 6 | #define _ASM_PAGE_H | ||
| 7 | |||
| 8 | #include <linux/const.h> | ||
| 9 | |||
| 10 | /* | ||
| 11 | * PAGE_SHIFT determines the page size | ||
| 12 | */ | ||
| 13 | #ifdef CONFIG_PAGE_SIZE_4KB | ||
| 14 | #define PAGE_SHIFT 12 | ||
| 15 | #endif | ||
| 16 | #ifdef CONFIG_PAGE_SIZE_16KB | ||
| 17 | #define PAGE_SHIFT 14 | ||
| 18 | #endif | ||
| 19 | #ifdef CONFIG_PAGE_SIZE_64KB | ||
| 20 | #define PAGE_SHIFT 16 | ||
| 21 | #endif | ||
| 22 | #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | ||
| 23 | #define PAGE_MASK (~(PAGE_SIZE - 1)) | ||
| 24 | |||
| 25 | #define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) | ||
| 26 | #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) | ||
| 27 | #define HPAGE_MASK (~(HPAGE_SIZE - 1)) | ||
| 28 | #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) | ||
| 29 | |||
| 30 | #ifndef __ASSEMBLY__ | ||
| 31 | |||
| 32 | #include <linux/kernel.h> | ||
| 33 | #include <linux/pfn.h> | ||
| 34 | |||
| 35 | #define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) | ||
| 36 | |||
| 37 | /* | ||
| 38 | * It's normally defined only for FLATMEM config but it's | ||
| 39 | * used in our early mem init code for all memory models. | ||
| 40 | * So always define it. | ||
| 41 | */ | ||
| 42 | #define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET) | ||
| 43 | |||
| 44 | extern void clear_page(void *page); | ||
| 45 | extern void copy_page(void *to, void *from); | ||
| 46 | |||
| 47 | #define clear_user_page(page, vaddr, pg) clear_page(page) | ||
| 48 | #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) | ||
| 49 | |||
| 50 | extern unsigned long shm_align_mask; | ||
| 51 | |||
| 52 | struct page; | ||
| 53 | struct vm_area_struct; | ||
| 54 | void copy_user_highpage(struct page *to, struct page *from, | ||
| 55 | unsigned long vaddr, struct vm_area_struct *vma); | ||
| 56 | |||
| 57 | #define __HAVE_ARCH_COPY_USER_HIGHPAGE | ||
| 58 | |||
| 59 | typedef struct { unsigned long pte; } pte_t; | ||
| 60 | #define pte_val(x) ((x).pte) | ||
| 61 | #define __pte(x) ((pte_t) { (x) }) | ||
| 62 | typedef struct page *pgtable_t; | ||
| 63 | |||
| 64 | typedef struct { unsigned long pgd; } pgd_t; | ||
| 65 | #define pgd_val(x) ((x).pgd) | ||
| 66 | #define __pgd(x) ((pgd_t) { (x) }) | ||
| 67 | |||
| 68 | /* | ||
| 69 | * Manipulate page protection bits | ||
| 70 | */ | ||
| 71 | typedef struct { unsigned long pgprot; } pgprot_t; | ||
| 72 | #define pgprot_val(x) ((x).pgprot) | ||
| 73 | #define __pgprot(x) ((pgprot_t) { (x) }) | ||
| 74 | #define pte_pgprot(x) __pgprot(pte_val(x) & ~_PFN_MASK) | ||
| 75 | |||
| 76 | #define ptep_buddy(x) ((pte_t *)((unsigned long)(x) ^ sizeof(pte_t))) | ||
| 77 | |||
| 78 | /* | ||
| 79 | * __pa()/__va() should be used only during mem init. | ||
| 80 | */ | ||
| 81 | #define __pa(x) PHYSADDR(x) | ||
| 82 | #define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET)) | ||
| 83 | |||
| 84 | #define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) | ||
| 85 | |||
| 86 | #ifdef CONFIG_FLATMEM | ||
| 87 | |||
| 88 | static inline int pfn_valid(unsigned long pfn) | ||
| 89 | { | ||
| 90 | /* avoid <linux/mm.h> include hell */ | ||
| 91 | extern unsigned long max_mapnr; | ||
| 92 | unsigned long pfn_offset = ARCH_PFN_OFFSET; | ||
| 93 | |||
| 94 | return pfn >= pfn_offset && pfn < max_mapnr; | ||
| 95 | } | ||
| 96 | |||
| 97 | #endif | ||
| 98 | |||
| 99 | #define virt_to_pfn(kaddr) PFN_DOWN(virt_to_phys((void *)(kaddr))) | ||
| 100 | #define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) | ||
| 101 | |||
| 102 | extern int __virt_addr_valid(volatile void *kaddr); | ||
| 103 | #define virt_addr_valid(kaddr) __virt_addr_valid((volatile void *)(kaddr)) | ||
| 104 | |||
| 105 | #define VM_DATA_DEFAULT_FLAGS \ | ||
| 106 | (VM_READ | VM_WRITE | \ | ||
| 107 | ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \ | ||
| 108 | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) | ||
| 109 | |||
| 110 | #include <asm-generic/memory_model.h> | ||
| 111 | #include <asm-generic/getorder.h> | ||
| 112 | |||
| 113 | #endif /* !__ASSEMBLY__ */ | ||
| 114 | |||
| 115 | #endif /* _ASM_PAGE_H */ | ||
| @@ -0,0 +1,103 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_PGALLOC_H | ||
| 6 | #define _ASM_PGALLOC_H | ||
| 7 | |||
| 8 | #include <linux/mm.h> | ||
| 9 | #include <linux/sched.h> | ||
| 10 | |||
| 11 | #define __HAVE_ARCH_PMD_ALLOC_ONE | ||
| 12 | #define __HAVE_ARCH_PUD_ALLOC_ONE | ||
| 13 | #include <asm-generic/pgalloc.h> | ||
| 14 | |||
| 15 | static inline void pmd_populate_kernel(struct mm_struct *mm, | ||
| 16 | pmd_t *pmd, pte_t *pte) | ||
| 17 | { | ||
| 18 | set_pmd(pmd, __pmd((unsigned long)pte)); | ||
| 19 | } | ||
| 20 | |||
| 21 | static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) | ||
| 22 | { | ||
| 23 | set_pmd(pmd, __pmd((unsigned long)page_address(pte))); | ||
| 24 | } | ||
| 25 | |||
| 26 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 27 | |||
| 28 | static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) | ||
| 29 | { | ||
| 30 | set_pud(pud, __pud((unsigned long)pmd)); | ||
| 31 | } | ||
| 32 | #endif | ||
| 33 | |||
| 34 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 35 | |||
| 36 | static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) | ||
| 37 | { | ||
| 38 | set_p4d(p4d, __p4d((unsigned long)pud)); | ||
| 39 | } | ||
| 40 | |||
| 41 | #endif /* __PAGETABLE_PUD_FOLDED */ | ||
| 42 | |||
| 43 | extern void pagetable_init(void); | ||
| 44 | |||
| 45 | /* | ||
| 46 | * Initialize a new pmd table with invalid pointers. | ||
| 47 | */ | ||
| 48 | extern void pmd_init(unsigned long page, unsigned long pagetable); | ||
| 49 | |||
| 50 | /* | ||
| 51 | * Initialize a new pgd / pmd table with invalid pointers. | ||
| 52 | */ | ||
| 53 | extern void pgd_init(unsigned long page); | ||
| 54 | extern pgd_t *pgd_alloc(struct mm_struct *mm); | ||
| 55 | |||
| 56 | #define __pte_free_tlb(tlb, pte, address) \ | ||
| 57 | do { \ | ||
| 58 | pgtable_pte_page_dtor(pte); \ | ||
| 59 | tlb_remove_page((tlb), pte); \ | ||
| 60 | } while (0) | ||
| 61 | |||
| 62 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 63 | |||
| 64 | static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) | ||
| 65 | { | ||
| 66 | pmd_t *pmd; | ||
| 67 | struct page *pg; | ||
| 68 | |||
| 69 | pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_ORDER); | ||
| 70 | if (!pg) | ||
| 71 | return NULL; | ||
| 72 | |||
| 73 | if (!pgtable_pmd_page_ctor(pg)) { | ||
| 74 | __free_pages(pg, PMD_ORDER); | ||
| 75 | return NULL; | ||
| 76 | } | ||
| 77 | |||
| 78 | pmd = (pmd_t *)page_address(pg); | ||
| 79 | pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table); | ||
| 80 | return pmd; | ||
| 81 | } | ||
| 82 | |||
| 83 | #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) | ||
| 84 | |||
| 85 | #endif | ||
| 86 | |||
| 87 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 88 | |||
| 89 | static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) | ||
| 90 | { | ||
| 91 | pud_t *pud; | ||
| 92 | |||
| 93 | pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER); | ||
| 94 | if (pud) | ||
| 95 | pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table); | ||
| 96 | return pud; | ||
| 97 | } | ||
| 98 | |||
| 99 | #define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) | ||
| 100 | |||
| 101 | #endif /* __PAGETABLE_PUD_FOLDED */ | ||
| 102 | |||
| 103 | #endif /* _ASM_PGALLOC_H */ | ||
| @@ -0,0 +1,131 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_PGTABLE_BITS_H | ||
| 6 | #define _ASM_PGTABLE_BITS_H | ||
| 7 | |||
| 8 | /* Page table bits */ | ||
| 9 | #define _PAGE_VALID_SHIFT 0 | ||
| 10 | #define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ | ||
| 11 | #define _PAGE_DIRTY_SHIFT 1 | ||
| 12 | #define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */ | ||
| 13 | #define _CACHE_SHIFT 4 /* 4~5, two bits */ | ||
| 14 | #define _PAGE_GLOBAL_SHIFT 6 | ||
| 15 | #define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */ | ||
| 16 | #define _PAGE_PRESENT_SHIFT 7 | ||
| 17 | #define _PAGE_WRITE_SHIFT 8 | ||
| 18 | #define _PAGE_MODIFIED_SHIFT 9 | ||
| 19 | #define _PAGE_PROTNONE_SHIFT 10 | ||
| 20 | #define _PAGE_SPECIAL_SHIFT 11 | ||
| 21 | #define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ | ||
| 22 | #define _PAGE_PFN_SHIFT 12 | ||
| 23 | #define _PAGE_PFN_END_SHIFT 48 | ||
| 24 | #define _PAGE_NO_READ_SHIFT 61 | ||
| 25 | #define _PAGE_NO_EXEC_SHIFT 62 | ||
| 26 | #define _PAGE_RPLV_SHIFT 63 | ||
| 27 | |||
| 28 | /* Used by software */ | ||
| 29 | #define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT) | ||
| 30 | #define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT) | ||
| 31 | #define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT) | ||
| 32 | #define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT) | ||
| 33 | #define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT) | ||
| 34 | #define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT) | ||
| 35 | |||
| 36 | /* Used by TLB hardware (placed in EntryLo*) */ | ||
| 37 | #define _PAGE_VALID (_ULCAST_(1) << _PAGE_VALID_SHIFT) | ||
| 38 | #define _PAGE_DIRTY (_ULCAST_(1) << _PAGE_DIRTY_SHIFT) | ||
| 39 | #define _PAGE_PLV (_ULCAST_(3) << _PAGE_PLV_SHIFT) | ||
| 40 | #define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT) | ||
| 41 | #define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT) | ||
| 42 | #define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT) | ||
| 43 | #define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT) | ||
| 44 | #define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT) | ||
| 45 | #define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT) | ||
| 46 | #define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT) | ||
| 47 | #define _PFN_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT) | ||
| 48 | |||
| 49 | #define _PAGE_USER (PLV_USER << _PAGE_PLV_SHIFT) | ||
| 50 | #define _PAGE_KERN (PLV_KERN << _PAGE_PLV_SHIFT) | ||
| 51 | |||
| 52 | #define _PFN_MASK (~((_ULCAST_(1) << (_PFN_SHIFT)) - 1) & \ | ||
| 53 | ((_ULCAST_(1) << (_PAGE_PFN_END_SHIFT)) - 1)) | ||
| 54 | |||
| 55 | /* | ||
| 56 | * Cache attributes | ||
| 57 | */ | ||
| 58 | #ifndef _CACHE_SUC | ||
| 59 | #define _CACHE_SUC (0<<_CACHE_SHIFT) /* Strong-ordered UnCached */ | ||
| 60 | #endif | ||
| 61 | #ifndef _CACHE_CC | ||
| 62 | #define _CACHE_CC (1<<_CACHE_SHIFT) /* Coherent Cached */ | ||
| 63 | #endif | ||
| 64 | #ifndef _CACHE_WUC | ||
| 65 | #define _CACHE_WUC (2<<_CACHE_SHIFT) /* Weak-ordered UnCached */ | ||
| 66 | #endif | ||
| 67 | |||
| 68 | #define __READABLE (_PAGE_VALID) | ||
| 69 | #define __WRITEABLE (_PAGE_DIRTY | _PAGE_WRITE) | ||
| 70 | |||
| 71 | #define _PAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV) | ||
| 72 | #define _HPAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV | _PAGE_HUGE) | ||
| 73 | |||
| 74 | #define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_NO_READ | \ | ||
| 75 | _PAGE_USER | _CACHE_CC) | ||
| 76 | #define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \ | ||
| 77 | _PAGE_USER | _CACHE_CC) | ||
| 78 | #define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _CACHE_CC) | ||
| 79 | |||
| 80 | #define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | ||
| 81 | _PAGE_GLOBAL | _PAGE_KERN | _CACHE_CC) | ||
| 82 | #define PAGE_KERNEL_SUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | ||
| 83 | _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC) | ||
| 84 | #define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | ||
| 85 | _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC) | ||
| 86 | |||
| 87 | #define __P000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) | ||
| 88 | #define __P001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | ||
| 89 | #define __P010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | ||
| 90 | #define __P011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | ||
| 91 | #define __P100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 92 | #define __P101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 93 | #define __P110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 94 | #define __P111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 95 | |||
| 96 | #define __S000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) | ||
| 97 | #define __S001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | ||
| 98 | #define __S010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) | ||
| 99 | #define __S011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) | ||
| 100 | #define __S100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 101 | #define __S101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | ||
| 102 | #define __S110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) | ||
| 103 | #define __S111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) | ||
| 104 | |||
| 105 | #ifndef __ASSEMBLY__ | ||
| 106 | |||
| 107 | #define pgprot_noncached pgprot_noncached | ||
| 108 | |||
| 109 | static inline pgprot_t pgprot_noncached(pgprot_t _prot) | ||
| 110 | { | ||
| 111 | unsigned long prot = pgprot_val(_prot); | ||
| 112 | |||
| 113 | prot = (prot & ~_CACHE_MASK) | _CACHE_SUC; | ||
| 114 | |||
| 115 | return __pgprot(prot); | ||
| 116 | } | ||
| 117 | |||
| 118 | #define pgprot_writecombine pgprot_writecombine | ||
| 119 | |||
| 120 | static inline pgprot_t pgprot_writecombine(pgprot_t _prot) | ||
| 121 | { | ||
| 122 | unsigned long prot = pgprot_val(_prot); | ||
| 123 | |||
| 124 | prot = (prot & ~_CACHE_MASK) | _CACHE_WUC; | ||
| 125 | |||
| 126 | return __pgprot(prot); | ||
| 127 | } | ||
| 128 | |||
| 129 | #endif /* !__ASSEMBLY__ */ | ||
| 130 | |||
| 131 | #endif /* _ASM_PGTABLE_BITS_H */ | ||
| @@ -0,0 +1,532 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | * | ||
| 5 | * Derived from MIPS: | ||
| 6 | * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle | ||
| 7 | * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc. | ||
| 8 | */ | ||
| 9 | #ifndef _ASM_PGTABLE_H | ||
| 10 | #define _ASM_PGTABLE_H | ||
| 11 | |||
| 12 | #include <linux/compiler.h> | ||
| 13 | #include <asm/addrspace.h> | ||
| 14 | #include <asm/pgtable-bits.h> | ||
| 15 | |||
| 16 | #if CONFIG_PGTABLE_LEVELS == 2 | ||
| 17 | #include <asm-generic/pgtable-nopmd.h> | ||
| 18 | #elif CONFIG_PGTABLE_LEVELS == 3 | ||
| 19 | #include <asm-generic/pgtable-nopud.h> | ||
| 20 | #else | ||
| 21 | #include <asm-generic/pgtable-nop4d.h> | ||
| 22 | #endif | ||
| 23 | |||
| 24 | #define PGD_ORDER 0 | ||
| 25 | #define PUD_ORDER 0 | ||
| 26 | #define PMD_ORDER 0 | ||
| 27 | #define PTE_ORDER 0 | ||
| 28 | |||
| 29 | #if CONFIG_PGTABLE_LEVELS == 2 | ||
| 30 | #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | ||
| 31 | #elif CONFIG_PGTABLE_LEVELS == 3 | ||
| 32 | #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | ||
| 33 | #define PMD_SIZE (1UL << PMD_SHIFT) | ||
| 34 | #define PMD_MASK (~(PMD_SIZE-1)) | ||
| 35 | #define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) | ||
| 36 | #elif CONFIG_PGTABLE_LEVELS == 4 | ||
| 37 | #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | ||
| 38 | #define PMD_SIZE (1UL << PMD_SHIFT) | ||
| 39 | #define PMD_MASK (~(PMD_SIZE-1)) | ||
| 40 | #define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) | ||
| 41 | #define PUD_SIZE (1UL << PUD_SHIFT) | ||
| 42 | #define PUD_MASK (~(PUD_SIZE-1)) | ||
| 43 | #define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3)) | ||
| 44 | #endif | ||
| 45 | |||
| 46 | #define PGDIR_SIZE (1UL << PGDIR_SHIFT) | ||
| 47 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) | ||
| 48 | |||
| 49 | #define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT + PGD_ORDER - 3)) | ||
| 50 | |||
| 51 | #define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) >> 3) | ||
| 52 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 53 | #define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) >> 3) | ||
| 54 | #endif | ||
| 55 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 56 | #define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) >> 3) | ||
| 57 | #endif | ||
| 58 | #define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) >> 3) | ||
| 59 | |||
| 60 | #define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) | ||
| 61 | |||
| 62 | #ifndef __ASSEMBLY__ | ||
| 63 | |||
| 64 | #include <linux/mm_types.h> | ||
| 65 | #include <linux/mmzone.h> | ||
| 66 | #include <asm/fixmap.h> | ||
| 67 | #include <asm/io.h> | ||
| 68 | |||
| 69 | struct mm_struct; | ||
| 70 | struct vm_area_struct; | ||
| 71 | |||
| 72 | /* | ||
| 73 | * ZERO_PAGE is a global shared page that is always zero; used | ||
| 74 | * for zero-mapped memory areas etc.. | ||
| 75 | */ | ||
| 76 | |||
| 77 | extern unsigned long empty_zero_page; | ||
| 78 | extern unsigned long zero_page_mask; | ||
| 79 | |||
| 80 | #define ZERO_PAGE(vaddr) \ | ||
| 81 | (virt_to_page((void *)(empty_zero_page + (((unsigned long)(vaddr)) & zero_page_mask)))) | ||
| 82 | #define __HAVE_COLOR_ZERO_PAGE | ||
| 83 | |||
| 84 | /* | ||
| 85 | * TLB refill handlers may also map the vmalloc area into xkvrange. | ||
| 86 | * Avoid the first couple of pages so NULL pointer dereferences will | ||
| 87 | * still reliably trap. | ||
| 88 | */ | ||
| 89 | #define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) | ||
| 90 | #define MODULES_END (MODULES_VADDR + SZ_256M) | ||
| 91 | |||
| 92 | #define VMALLOC_START MODULES_END | ||
| 93 | #define VMALLOC_END \ | ||
| 94 | (vm_map_base + \ | ||
| 95 | min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE) | ||
| 96 | |||
| 97 | #define pte_ERROR(e) \ | ||
| 98 | pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) | ||
| 99 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 100 | #define pmd_ERROR(e) \ | ||
| 101 | pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) | ||
| 102 | #endif | ||
| 103 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 104 | #define pud_ERROR(e) \ | ||
| 105 | pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) | ||
| 106 | #endif | ||
| 107 | #define pgd_ERROR(e) \ | ||
| 108 | pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) | ||
| 109 | |||
| 110 | extern pte_t invalid_pte_table[PTRS_PER_PTE]; | ||
| 111 | |||
| 112 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 113 | |||
| 114 | typedef struct { unsigned long pud; } pud_t; | ||
| 115 | #define pud_val(x) ((x).pud) | ||
| 116 | #define __pud(x) ((pud_t) { (x) }) | ||
| 117 | |||
| 118 | extern pud_t invalid_pud_table[PTRS_PER_PUD]; | ||
| 119 | |||
| 120 | /* | ||
| 121 | * Empty pgd/p4d entries point to the invalid_pud_table. | ||
| 122 | */ | ||
| 123 | static inline int p4d_none(p4d_t p4d) | ||
| 124 | { | ||
| 125 | return p4d_val(p4d) == (unsigned long)invalid_pud_table; | ||
| 126 | } | ||
| 127 | |||
| 128 | static inline int p4d_bad(p4d_t p4d) | ||
| 129 | { | ||
| 130 | return p4d_val(p4d) & ~PAGE_MASK; | ||
| 131 | } | ||
| 132 | |||
| 133 | static inline int p4d_present(p4d_t p4d) | ||
| 134 | { | ||
| 135 | return p4d_val(p4d) != (unsigned long)invalid_pud_table; | ||
| 136 | } | ||
| 137 | |||
| 138 | static inline void p4d_clear(p4d_t *p4dp) | ||
| 139 | { | ||
| 140 | p4d_val(*p4dp) = (unsigned long)invalid_pud_table; | ||
| 141 | } | ||
| 142 | |||
| 143 | static inline pud_t *p4d_pgtable(p4d_t p4d) | ||
| 144 | { | ||
| 145 | return (pud_t *)p4d_val(p4d); | ||
| 146 | } | ||
| 147 | |||
| 148 | static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) | ||
| 149 | { | ||
| 150 | *p4d = p4dval; | ||
| 151 | } | ||
| 152 | |||
| 153 | #define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) | ||
| 154 | #define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) | ||
| 155 | |||
| 156 | #endif | ||
| 157 | |||
| 158 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 159 | |||
| 160 | typedef struct { unsigned long pmd; } pmd_t; | ||
| 161 | #define pmd_val(x) ((x).pmd) | ||
| 162 | #define __pmd(x) ((pmd_t) { (x) }) | ||
| 163 | |||
| 164 | extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; | ||
| 165 | |||
| 166 | /* | ||
| 167 | * Empty pud entries point to the invalid_pmd_table. | ||
| 168 | */ | ||
| 169 | static inline int pud_none(pud_t pud) | ||
| 170 | { | ||
| 171 | return pud_val(pud) == (unsigned long)invalid_pmd_table; | ||
| 172 | } | ||
| 173 | |||
| 174 | static inline int pud_bad(pud_t pud) | ||
| 175 | { | ||
| 176 | return pud_val(pud) & ~PAGE_MASK; | ||
| 177 | } | ||
| 178 | |||
| 179 | static inline int pud_present(pud_t pud) | ||
| 180 | { | ||
| 181 | return pud_val(pud) != (unsigned long)invalid_pmd_table; | ||
| 182 | } | ||
| 183 | |||
| 184 | static inline void pud_clear(pud_t *pudp) | ||
| 185 | { | ||
| 186 | pud_val(*pudp) = ((unsigned long)invalid_pmd_table); | ||
| 187 | } | ||
| 188 | |||
| 189 | static inline pmd_t *pud_pgtable(pud_t pud) | ||
| 190 | { | ||
| 191 | return (pmd_t *)pud_val(pud); | ||
| 192 | } | ||
| 193 | |||
| 194 | #define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0) | ||
| 195 | |||
| 196 | #define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) | ||
| 197 | #define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) | ||
| 198 | |||
| 199 | #endif | ||
| 200 | |||
| 201 | /* | ||
| 202 | * Empty pmd entries point to the invalid_pte_table. | ||
| 203 | */ | ||
| 204 | static inline int pmd_none(pmd_t pmd) | ||
| 205 | { | ||
| 206 | return pmd_val(pmd) == (unsigned long)invalid_pte_table; | ||
| 207 | } | ||
| 208 | |||
| 209 | static inline int pmd_bad(pmd_t pmd) | ||
| 210 | { | ||
| 211 | return (pmd_val(pmd) & ~PAGE_MASK); | ||
| 212 | } | ||
| 213 | |||
| 214 | static inline int pmd_present(pmd_t pmd) | ||
| 215 | { | ||
| 216 | if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) | ||
| 217 | return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE)); | ||
| 218 | |||
| 219 | return pmd_val(pmd) != (unsigned long)invalid_pte_table; | ||
| 220 | } | ||
| 221 | |||
| 222 | static inline void pmd_clear(pmd_t *pmdp) | ||
| 223 | { | ||
| 224 | pmd_val(*pmdp) = ((unsigned long)invalid_pte_table); | ||
| 225 | } | ||
| 226 | |||
| 227 | #define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0) | ||
| 228 | |||
| 229 | #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) | ||
| 230 | |||
| 231 | #ifndef CONFIG_TRANSPARENT_HUGEPAGE | ||
| 232 | #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) | ||
| 233 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | ||
| 234 | |||
| 235 | #define pmd_page_vaddr(pmd) pmd_val(pmd) | ||
| 236 | |||
| 237 | extern pmd_t mk_pmd(struct page *page, pgprot_t prot); | ||
| 238 | extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd); | ||
| 239 | |||
| 240 | #define pte_page(x) pfn_to_page(pte_pfn(x)) | ||
| 241 | #define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> _PFN_SHIFT)) | ||
| 242 | #define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) | ||
| 243 | #define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) | ||
| 244 | |||
| 245 | /* | ||
| 246 | * Initialize a new pgd / pmd table with invalid pointers. | ||
| 247 | */ | ||
| 248 | extern void pgd_init(unsigned long page); | ||
| 249 | extern void pud_init(unsigned long page, unsigned long pagetable); | ||
| 250 | extern void pmd_init(unsigned long page, unsigned long pagetable); | ||
| 251 | |||
| 252 | /* | ||
| 253 | * Non-present pages: high 40 bits are offset, next 8 bits type, | ||
| 254 | * low 16 bits zero. | ||
| 255 | */ | ||
| 256 | static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) | ||
| 257 | { pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; } | ||
| 258 | |||
| 259 | #define __swp_type(x) (((x).val >> 16) & 0xff) | ||
| 260 | #define __swp_offset(x) ((x).val >> 24) | ||
| 261 | #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) | ||
| 262 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) | ||
| 263 | #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) | ||
| 264 | #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) | ||
| 265 | #define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE }) | ||
| 266 | |||
| 267 | extern void paging_init(void); | ||
| 268 | |||
| 269 | #define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) | ||
| 270 | #define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE)) | ||
| 271 | #define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC) | ||
| 272 | |||
| 273 | static inline void set_pte(pte_t *ptep, pte_t pteval) | ||
| 274 | { | ||
| 275 | *ptep = pteval; | ||
| 276 | if (pte_val(pteval) & _PAGE_GLOBAL) { | ||
| 277 | pte_t *buddy = ptep_buddy(ptep); | ||
| 278 | /* | ||
| 279 | * Make sure the buddy is global too (if it's !none, | ||
| 280 | * it better already be global) | ||
| 281 | */ | ||
| 282 | if (pte_none(*buddy)) | ||
| 283 | pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL; | ||
| 284 | } | ||
| 285 | } | ||
| 286 | |||
| 287 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, | ||
| 288 | pte_t *ptep, pte_t pteval) | ||
| 289 | { | ||
| 290 | set_pte(ptep, pteval); | ||
| 291 | } | ||
| 292 | |||
| 293 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | ||
| 294 | { | ||
| 295 | /* Preserve global status for the pair */ | ||
| 296 | if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL) | ||
| 297 | set_pte_at(mm, addr, ptep, __pte(_PAGE_GLOBAL)); | ||
| 298 | else | ||
| 299 | set_pte_at(mm, addr, ptep, __pte(0)); | ||
| 300 | } | ||
| 301 | |||
| 302 | #define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1) | ||
| 303 | #define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1) | ||
| 304 | #define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1) | ||
| 305 | |||
| 306 | extern pgd_t swapper_pg_dir[]; | ||
| 307 | extern pgd_t invalid_pg_dir[]; | ||
| 308 | |||
| 309 | /* | ||
| 310 | * The following only work if pte_present() is true. | ||
| 311 | * Undefined behaviour if not.. | ||
| 312 | */ | ||
| 313 | static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } | ||
| 314 | static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } | ||
| 315 | static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; } | ||
| 316 | |||
| 317 | static inline pte_t pte_mkold(pte_t pte) | ||
| 318 | { | ||
| 319 | pte_val(pte) &= ~_PAGE_ACCESSED; | ||
| 320 | return pte; | ||
| 321 | } | ||
| 322 | |||
| 323 | static inline pte_t pte_mkyoung(pte_t pte) | ||
| 324 | { | ||
| 325 | pte_val(pte) |= _PAGE_ACCESSED; | ||
| 326 | return pte; | ||
| 327 | } | ||
| 328 | |||
| 329 | static inline pte_t pte_mkclean(pte_t pte) | ||
| 330 | { | ||
| 331 | pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); | ||
| 332 | return pte; | ||
| 333 | } | ||
| 334 | |||
| 335 | static inline pte_t pte_mkdirty(pte_t pte) | ||
| 336 | { | ||
| 337 | pte_val(pte) |= (_PAGE_DIRTY | _PAGE_MODIFIED); | ||
| 338 | return pte; | ||
| 339 | } | ||
| 340 | |||
| 341 | static inline pte_t pte_mkwrite(pte_t pte) | ||
| 342 | { | ||
| 343 | pte_val(pte) |= (_PAGE_WRITE | _PAGE_DIRTY); | ||
| 344 | return pte; | ||
| 345 | } | ||
| 346 | |||
| 347 | static inline pte_t pte_wrprotect(pte_t pte) | ||
| 348 | { | ||
| 349 | pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY); | ||
| 350 | return pte; | ||
| 351 | } | ||
| 352 | |||
| 353 | static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; } | ||
| 354 | |||
| 355 | static inline pte_t pte_mkhuge(pte_t pte) | ||
| 356 | { | ||
| 357 | pte_val(pte) |= _PAGE_HUGE; | ||
| 358 | return pte; | ||
| 359 | } | ||
| 360 | |||
| 361 | #if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) | ||
| 362 | static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } | ||
| 363 | static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; } | ||
| 364 | #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ | ||
| 365 | |||
| 366 | #define pte_accessible pte_accessible | ||
| 367 | static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) | ||
| 368 | { | ||
| 369 | if (pte_val(a) & _PAGE_PRESENT) | ||
| 370 | return true; | ||
| 371 | |||
| 372 | if ((pte_val(a) & _PAGE_PROTNONE) && | ||
| 373 | atomic_read(&mm->tlb_flush_pending)) | ||
| 374 | return true; | ||
| 375 | |||
| 376 | return false; | ||
| 377 | } | ||
| 378 | |||
| 379 | /* | ||
| 380 | * Conversion functions: convert a page and protection to a page entry, | ||
| 381 | * and a page entry and page directory to the page they refer to. | ||
| 382 | */ | ||
| 383 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | ||
| 384 | |||
| 385 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) | ||
| 386 | { | ||
| 387 | return __pte((pte_val(pte) & _PAGE_CHG_MASK) | | ||
| 388 | (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); | ||
| 389 | } | ||
| 390 | |||
| 391 | extern void __update_tlb(struct vm_area_struct *vma, | ||
| 392 | unsigned long address, pte_t *ptep); | ||
| 393 | |||
| 394 | static inline void update_mmu_cache(struct vm_area_struct *vma, | ||
| 395 | unsigned long address, pte_t *ptep) | ||
| 396 | { | ||
| 397 | __update_tlb(vma, address, ptep); | ||
| 398 | } | ||
| 399 | |||
| 400 | static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, | ||
| 401 | unsigned long address, pmd_t *pmdp) | ||
| 402 | { | ||
| 403 | __update_tlb(vma, address, (pte_t *)pmdp); | ||
| 404 | } | ||
| 405 | |||
| 406 | #define kern_addr_valid(addr) (1) | ||
| 407 | |||
| 408 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
| 409 | |||
| 410 | /* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/ | ||
| 411 | #define pmdp_establish generic_pmdp_establish | ||
| 412 | |||
| 413 | static inline int pmd_trans_huge(pmd_t pmd) | ||
| 414 | { | ||
| 415 | return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd); | ||
| 416 | } | ||
| 417 | |||
| 418 | static inline pmd_t pmd_mkhuge(pmd_t pmd) | ||
| 419 | { | ||
| 420 | pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) | | ||
| 421 | ((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); | ||
| 422 | pmd_val(pmd) |= _PAGE_HUGE; | ||
| 423 | |||
| 424 | return pmd; | ||
| 425 | } | ||
| 426 | |||
| 427 | #define pmd_write pmd_write | ||
| 428 | static inline int pmd_write(pmd_t pmd) | ||
| 429 | { | ||
| 430 | return !!(pmd_val(pmd) & _PAGE_WRITE); | ||
| 431 | } | ||
| 432 | |||
| 433 | static inline pmd_t pmd_mkwrite(pmd_t pmd) | ||
| 434 | { | ||
| 435 | pmd_val(pmd) |= (_PAGE_WRITE | _PAGE_DIRTY); | ||
| 436 | return pmd; | ||
| 437 | } | ||
| 438 | |||
| 439 | static inline pmd_t pmd_wrprotect(pmd_t pmd) | ||
| 440 | { | ||
| 441 | pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY); | ||
| 442 | return pmd; | ||
| 443 | } | ||
| 444 | |||
| 445 | static inline int pmd_dirty(pmd_t pmd) | ||
| 446 | { | ||
| 447 | return !!(pmd_val(pmd) & _PAGE_MODIFIED); | ||
| 448 | } | ||
| 449 | |||
| 450 | static inline pmd_t pmd_mkclean(pmd_t pmd) | ||
| 451 | { | ||
| 452 | pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); | ||
| 453 | return pmd; | ||
| 454 | } | ||
| 455 | |||
| 456 | static inline pmd_t pmd_mkdirty(pmd_t pmd) | ||
| 457 | { | ||
| 458 | pmd_val(pmd) |= (_PAGE_DIRTY | _PAGE_MODIFIED); | ||
| 459 | return pmd; | ||
| 460 | } | ||
| 461 | |||
| 462 | static inline int pmd_young(pmd_t pmd) | ||
| 463 | { | ||
| 464 | return !!(pmd_val(pmd) & _PAGE_ACCESSED); | ||
| 465 | } | ||
| 466 | |||
| 467 | static inline pmd_t pmd_mkold(pmd_t pmd) | ||
| 468 | { | ||
| 469 | pmd_val(pmd) &= ~_PAGE_ACCESSED; | ||
| 470 | return pmd; | ||
| 471 | } | ||
| 472 | |||
| 473 | static inline pmd_t pmd_mkyoung(pmd_t pmd) | ||
| 474 | { | ||
| 475 | pmd_val(pmd) |= _PAGE_ACCESSED; | ||
| 476 | return pmd; | ||
| 477 | } | ||
| 478 | |||
| 479 | static inline unsigned long pmd_pfn(pmd_t pmd) | ||
| 480 | { | ||
| 481 | return (pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT; | ||
| 482 | } | ||
| 483 | |||
| 484 | static inline struct page *pmd_page(pmd_t pmd) | ||
| 485 | { | ||
| 486 | if (pmd_trans_huge(pmd)) | ||
| 487 | return pfn_to_page(pmd_pfn(pmd)); | ||
| 488 | |||
| 489 | return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); | ||
| 490 | } | ||
| 491 | |||
| 492 | static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) | ||
| 493 | { | ||
| 494 | pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | | ||
| 495 | (pgprot_val(newprot) & ~_HPAGE_CHG_MASK); | ||
| 496 | return pmd; | ||
| 497 | } | ||
| 498 | |||
| 499 | static inline pmd_t pmd_mkinvalid(pmd_t pmd) | ||
| 500 | { | ||
| 501 | pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE); | ||
| 502 | |||
| 503 | return pmd; | ||
| 504 | } | ||
| 505 | |||
| 506 | /* | ||
| 507 | * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a | ||
| 508 | * different prototype. | ||
| 509 | */ | ||
| 510 | #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR | ||
| 511 | static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, | ||
| 512 | unsigned long address, pmd_t *pmdp) | ||
| 513 | { | ||
| 514 | pmd_t old = *pmdp; | ||
| 515 | |||
| 516 | pmd_clear(pmdp); | ||
| 517 | |||
| 518 | return old; | ||
| 519 | } | ||
| 520 | |||
| 521 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | ||
| 522 | |||
| 523 | /* | ||
| 524 | * We provide our own get_unmapped area to cope with the virtual aliasing | ||
| 525 | * constraints placed on us by the cache architecture. | ||
| 526 | */ | ||
| 527 | #define HAVE_ARCH_UNMAPPED_AREA | ||
| 528 | #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN | ||
| 529 | |||
| 530 | #endif /* !__ASSEMBLY__ */ | ||
| 531 | |||
| 532 | #endif /* _ASM_PGTABLE_H */ | ||
| @@ -0,0 +1,12 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef _ASM_SHMPARAM_H | ||
| 6 | #define _ASM_SHMPARAM_H | ||
| 7 | |||
| 8 | #define __ARCH_FORCE_SHMLBA 1 | ||
| 9 | |||
| 10 | #define SHMLBA SZ_64K /* attach addr a multiple of this */ | ||
| 11 | |||
| 12 | #endif /* _ASM_SHMPARAM_H */ | ||
| @@ -0,0 +1,23 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | #ifndef _LOONGARCH_SPARSEMEM_H | ||
| 3 | #define _LOONGARCH_SPARSEMEM_H | ||
| 4 | |||
| 5 | #ifdef CONFIG_SPARSEMEM | ||
| 6 | |||
| 7 | /* | ||
| 8 | * SECTION_SIZE_BITS 2^N: how big each section will be | ||
| 9 | * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space | ||
| 10 | */ | ||
| 11 | #define SECTION_SIZE_BITS 29 /* 2^29 = Largest Huge Page Size */ | ||
| 12 | #define MAX_PHYSMEM_BITS 48 | ||
| 13 | |||
| 14 | #endif /* CONFIG_SPARSEMEM */ | ||
| 15 | |||
| 16 | #ifdef CONFIG_MEMORY_HOTPLUG | ||
| 17 | int memory_add_physaddr_to_nid(u64 addr); | ||
| 18 | #define memory_add_physaddr_to_nid memory_add_physaddr_to_nid | ||
| 19 | #endif | ||
| 20 | |||
| 21 | #define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS) | ||
| 22 | |||
| 23 | #endif /* _LOONGARCH_SPARSEMEM_H */ | ||
| @@ -0,0 +1,180 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef __ASM_TLB_H | ||
| 6 | #define __ASM_TLB_H | ||
| 7 | |||
| 8 | #include <linux/mm_types.h> | ||
| 9 | #include <asm/cpu-features.h> | ||
| 10 | #include <asm/loongarch.h> | ||
| 11 | |||
| 12 | /* | ||
| 13 | * TLB Invalidate Flush | ||
| 14 | */ | ||
| 15 | static inline void tlbclr(void) | ||
| 16 | { | ||
| 17 | __asm__ __volatile__("tlbclr"); | ||
| 18 | } | ||
| 19 | |||
| 20 | static inline void tlbflush(void) | ||
| 21 | { | ||
| 22 | __asm__ __volatile__("tlbflush"); | ||
| 23 | } | ||
| 24 | |||
| 25 | /* | ||
| 26 | * TLB R/W operations. | ||
| 27 | */ | ||
| 28 | static inline void tlb_probe(void) | ||
| 29 | { | ||
| 30 | __asm__ __volatile__("tlbsrch"); | ||
| 31 | } | ||
| 32 | |||
| 33 | static inline void tlb_read(void) | ||
| 34 | { | ||
| 35 | __asm__ __volatile__("tlbrd"); | ||
| 36 | } | ||
| 37 | |||
| 38 | static inline void tlb_write_indexed(void) | ||
| 39 | { | ||
| 40 | __asm__ __volatile__("tlbwr"); | ||
| 41 | } | ||
| 42 | |||
| 43 | static inline void tlb_write_random(void) | ||
| 44 | { | ||
| 45 | __asm__ __volatile__("tlbfill"); | ||
| 46 | } | ||
| 47 | |||
| 48 | enum invtlb_ops { | ||
| 49 | /* Invalid all tlb */ | ||
| 50 | INVTLB_ALL = 0x0, | ||
| 51 | /* Invalid current tlb */ | ||
| 52 | INVTLB_CURRENT_ALL = 0x1, | ||
| 53 | /* Invalid all global=1 lines in current tlb */ | ||
| 54 | INVTLB_CURRENT_GTRUE = 0x2, | ||
| 55 | /* Invalid all global=0 lines in current tlb */ | ||
| 56 | INVTLB_CURRENT_GFALSE = 0x3, | ||
| 57 | /* Invalid global=0 and matched asid lines in current tlb */ | ||
| 58 | INVTLB_GFALSE_AND_ASID = 0x4, | ||
| 59 | /* Invalid addr with global=0 and matched asid in current tlb */ | ||
| 60 | INVTLB_ADDR_GFALSE_AND_ASID = 0x5, | ||
| 61 | /* Invalid addr with global=1 or matched asid in current tlb */ | ||
| 62 | INVTLB_ADDR_GTRUE_OR_ASID = 0x6, | ||
| 63 | /* Invalid matched gid in guest tlb */ | ||
| 64 | INVGTLB_GID = 0x9, | ||
| 65 | /* Invalid global=1, matched gid in guest tlb */ | ||
| 66 | INVGTLB_GID_GTRUE = 0xa, | ||
| 67 | /* Invalid global=0, matched gid in guest tlb */ | ||
| 68 | INVGTLB_GID_GFALSE = 0xb, | ||
| 69 | /* Invalid global=0, matched gid and asid in guest tlb */ | ||
| 70 | INVGTLB_GID_GFALSE_ASID = 0xc, | ||
| 71 | /* Invalid global=0 , matched gid, asid and addr in guest tlb */ | ||
| 72 | INVGTLB_GID_GFALSE_ASID_ADDR = 0xd, | ||
| 73 | /* Invalid global=1 , matched gid, asid and addr in guest tlb */ | ||
| 74 | INVGTLB_GID_GTRUE_ASID_ADDR = 0xe, | ||
| 75 | /* Invalid all gid gva-->gpa guest tlb */ | ||
| 76 | INVGTLB_ALLGID_GVA_TO_GPA = 0x10, | ||
| 77 | /* Invalid all gid gpa-->hpa tlb */ | ||
| 78 | INVTLB_ALLGID_GPA_TO_HPA = 0x11, | ||
| 79 | /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */ | ||
| 80 | INVTLB_ALLGID = 0x12, | ||
| 81 | /* Invalid matched gid gva-->gpa guest tlb */ | ||
| 82 | INVGTLB_GID_GVA_TO_GPA = 0x13, | ||
| 83 | /* Invalid matched gid gpa-->hpa tlb */ | ||
| 84 | INVTLB_GID_GPA_TO_HPA = 0x14, | ||
| 85 | /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */ | ||
| 86 | INVTLB_GID_ALL = 0x15, | ||
| 87 | /* Invalid matched gid and addr gpa-->hpa tlb */ | ||
| 88 | INVTLB_GID_ADDR = 0x16, | ||
| 89 | }; | ||
| 90 | |||
| 91 | /* | ||
| 92 | * invtlb op info addr | ||
| 93 | * (0x1 << 26) | (0x24 << 20) | (0x13 << 15) | | ||
| 94 | * (addr << 10) | (info << 5) | op | ||
| 95 | */ | ||
| 96 | static inline void invtlb(u32 op, u32 info, u64 addr) | ||
| 97 | { | ||
| 98 | __asm__ __volatile__( | ||
| 99 | "parse_r addr,%0\n\t" | ||
| 100 | "parse_r info,%1\n\t" | ||
| 101 | ".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t" | ||
| 102 | : | ||
| 103 | : "r"(addr), "r"(info), "i"(op) | ||
| 104 | : | ||
| 105 | ); | ||
| 106 | } | ||
| 107 | |||
| 108 | static inline void invtlb_addr(u32 op, u32 info, u64 addr) | ||
| 109 | { | ||
| 110 | __asm__ __volatile__( | ||
| 111 | "parse_r addr,%0\n\t" | ||
| 112 | ".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t" | ||
| 113 | : | ||
| 114 | : "r"(addr), "i"(op) | ||
| 115 | : | ||
| 116 | ); | ||
| 117 | } | ||
| 118 | |||
| 119 | static inline void invtlb_info(u32 op, u32 info, u64 addr) | ||
| 120 | { | ||
| 121 | __asm__ __volatile__( | ||
| 122 | "parse_r info,%0\n\t" | ||
| 123 | ".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t" | ||
| 124 | : | ||
| 125 | : "r"(info), "i"(op) | ||
| 126 | : | ||
| 127 | ); | ||
| 128 | } | ||
| 129 | |||
| 130 | static inline void invtlb_all(u32 op, u32 info, u64 addr) | ||
| 131 | { | ||
| 132 | __asm__ __volatile__( | ||
| 133 | ".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t" | ||
| 134 | : | ||
| 135 | : "i"(op) | ||
| 136 | : | ||
| 137 | ); | ||
| 138 | } | ||
| 139 | |||
| 140 | /* | ||
| 141 | * LoongArch doesn't need any special per-pte or per-vma handling, except | ||
| 142 | * we need to flush cache for area to be unmapped. | ||
| 143 | */ | ||
| 144 | #define tlb_start_vma(tlb, vma) \ | ||
| 145 | do { \ | ||
| 146 | if (!(tlb)->fullmm) \ | ||
| 147 | flush_cache_range(vma, vma->vm_start, vma->vm_end); \ | ||
| 148 | } while (0) | ||
| 149 | #define tlb_end_vma(tlb, vma) do { } while (0) | ||
| 150 | #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) | ||
| 151 | |||
| 152 | static void tlb_flush(struct mmu_gather *tlb); | ||
| 153 | |||
| 154 | #define tlb_flush tlb_flush | ||
| 155 | #include <asm-generic/tlb.h> | ||
| 156 | |||
| 157 | static inline void tlb_flush(struct mmu_gather *tlb) | ||
| 158 | { | ||
| 159 | struct vm_area_struct vma; | ||
| 160 | |||
| 161 | vma.vm_mm = tlb->mm; | ||
| 162 | vma.vm_flags = 0; | ||
| 163 | if (tlb->fullmm) { | ||
| 164 | flush_tlb_mm(tlb->mm); | ||
| 165 | return; | ||
| 166 | } | ||
| 167 | |||
| 168 | flush_tlb_range(&vma, tlb->start, tlb->end); | ||
| 169 | } | ||
| 170 | |||
| 171 | extern void handle_tlb_load(void); | ||
| 172 | extern void handle_tlb_store(void); | ||
| 173 | extern void handle_tlb_modify(void); | ||
| 174 | extern void handle_tlb_refill(void); | ||
| 175 | extern void handle_tlb_protect(void); | ||
| 176 | |||
| 177 | extern void dump_tlb_all(void); | ||
| 178 | extern void dump_tlb_regs(void); | ||
| 179 | |||
| 180 | #endif /* __ASM_TLB_H */ | ||
| @@ -0,0 +1,35 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #ifndef __ASM_TLBFLUSH_H | ||
| 6 | #define __ASM_TLBFLUSH_H | ||
| 7 | |||
| 8 | #include <linux/mm.h> | ||
| 9 | |||
| 10 | /* | ||
| 11 | * TLB flushing: | ||
| 12 | * | ||
| 13 | * - flush_tlb_all() flushes all processes TLB entries | ||
| 14 | * - flush_tlb_mm(mm) flushes the specified mm context TLB entries | ||
| 15 | * - flush_tlb_page(vma, vmaddr) flushes one page | ||
| 16 | * - flush_tlb_range(vma, start, end) flushes a range of pages | ||
| 17 | * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages | ||
| 18 | */ | ||
| 19 | extern void local_flush_tlb_all(void); | ||
| 20 | extern void local_flush_tlb_user(void); | ||
| 21 | extern void local_flush_tlb_kernel(void); | ||
| 22 | extern void local_flush_tlb_mm(struct mm_struct *mm); | ||
| 23 | extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); | ||
| 24 | extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); | ||
| 25 | extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); | ||
| 26 | extern void local_flush_tlb_one(unsigned long vaddr); | ||
| 27 | |||
| 28 | #define flush_tlb_all() local_flush_tlb_all() | ||
| 29 | #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) | ||
| 30 | #define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end) | ||
| 31 | #define flush_tlb_kernel_range(vmaddr, end) local_flush_tlb_kernel_range(vmaddr, end) | ||
| 32 | #define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page) | ||
| 33 | #define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr) | ||
| 34 | |||
| 35 | #endif /* __ASM_TLBFLUSH_H */ | ||
| @@ -0,0 +1,4 @@ | |||
| 1 | #ifndef _ASM_LOONGARCH_VMALLOC_H | ||
| 2 | #define _ASM_LOONGARCH_VMALLOC_H | ||
| 3 | |||
| 4 | #endif /* _ASM_LOONGARCH_VMALLOC_H */ | ||
| @@ -0,0 +1,141 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | * | ||
| 5 | * Derived from MIPS: | ||
| 6 | * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) | ||
| 7 | * Copyright (C) 2007 MIPS Technologies, Inc. | ||
| 8 | */ | ||
| 9 | #include <linux/export.h> | ||
| 10 | #include <linux/fcntl.h> | ||
| 11 | #include <linux/fs.h> | ||
| 12 | #include <linux/highmem.h> | ||
| 13 | #include <linux/kernel.h> | ||
| 14 | #include <linux/linkage.h> | ||
| 15 | #include <linux/mm.h> | ||
| 16 | #include <linux/sched.h> | ||
| 17 | #include <linux/syscalls.h> | ||
| 18 | |||
| 19 | #include <asm/cacheflush.h> | ||
| 20 | #include <asm/cpu.h> | ||
| 21 | #include <asm/cpu-features.h> | ||
| 22 | #include <asm/dma.h> | ||
| 23 | #include <asm/loongarch.h> | ||
| 24 | #include <asm/processor.h> | ||
| 25 | #include <asm/setup.h> | ||
| 26 | |||
| 27 | /* | ||
| 28 | * LoongArch maintains ICache/DCache coherency by hardware, | ||
| 29 | * we just need "ibar" to avoid instruction hazard here. | ||
| 30 | */ | ||
| 31 | void local_flush_icache_range(unsigned long start, unsigned long end) | ||
| 32 | { | ||
| 33 | asm volatile ("\tibar 0\n"::); | ||
| 34 | } | ||
| 35 | EXPORT_SYMBOL(local_flush_icache_range); | ||
| 36 | |||
| 37 | void cache_error_setup(void) | ||
| 38 | { | ||
| 39 | extern char __weak except_vec_cex; | ||
| 40 | set_merr_handler(0x0, &except_vec_cex, 0x80); | ||
| 41 | } | ||
| 42 | |||
| 43 | static unsigned long icache_size __read_mostly; | ||
| 44 | static unsigned long dcache_size __read_mostly; | ||
| 45 | static unsigned long vcache_size __read_mostly; | ||
| 46 | static unsigned long scache_size __read_mostly; | ||
| 47 | |||
| 48 | static char *way_string[] = { NULL, "direct mapped", "2-way", | ||
| 49 | "3-way", "4-way", "5-way", "6-way", "7-way", "8-way", | ||
| 50 | "9-way", "10-way", "11-way", "12-way", | ||
| 51 | "13-way", "14-way", "15-way", "16-way", | ||
| 52 | }; | ||
| 53 | |||
| 54 | static void probe_pcache(void) | ||
| 55 | { | ||
| 56 | struct cpuinfo_loongarch *c = ¤t_cpu_data; | ||
| 57 | unsigned int lsize, sets, ways; | ||
| 58 | unsigned int config; | ||
| 59 | |||
| 60 | config = read_cpucfg(LOONGARCH_CPUCFG17); | ||
| 61 | lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE); | ||
| 62 | sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS); | ||
| 63 | ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1; | ||
| 64 | |||
| 65 | c->icache.linesz = lsize; | ||
| 66 | c->icache.sets = sets; | ||
| 67 | c->icache.ways = ways; | ||
| 68 | icache_size = sets * ways * lsize; | ||
| 69 | c->icache.waysize = icache_size / c->icache.ways; | ||
| 70 | |||
| 71 | config = read_cpucfg(LOONGARCH_CPUCFG18); | ||
| 72 | lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE); | ||
| 73 | sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS); | ||
| 74 | ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1; | ||
| 75 | |||
| 76 | c->dcache.linesz = lsize; | ||
| 77 | c->dcache.sets = sets; | ||
| 78 | c->dcache.ways = ways; | ||
| 79 | dcache_size = sets * ways * lsize; | ||
| 80 | c->dcache.waysize = dcache_size / c->dcache.ways; | ||
| 81 | |||
| 82 | c->options |= LOONGARCH_CPU_PREFETCH; | ||
| 83 | |||
| 84 | pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", | ||
| 85 | icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz); | ||
| 86 | |||
| 87 | pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", | ||
| 88 | dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz); | ||
| 89 | } | ||
| 90 | |||
| 91 | static void probe_vcache(void) | ||
| 92 | { | ||
| 93 | struct cpuinfo_loongarch *c = ¤t_cpu_data; | ||
| 94 | unsigned int lsize, sets, ways; | ||
| 95 | unsigned int config; | ||
| 96 | |||
| 97 | config = read_cpucfg(LOONGARCH_CPUCFG19); | ||
| 98 | lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE); | ||
| 99 | sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS); | ||
| 100 | ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1; | ||
| 101 | |||
| 102 | c->vcache.linesz = lsize; | ||
| 103 | c->vcache.sets = sets; | ||
| 104 | c->vcache.ways = ways; | ||
| 105 | vcache_size = lsize * sets * ways; | ||
| 106 | c->vcache.waysize = vcache_size / c->vcache.ways; | ||
| 107 | |||
| 108 | pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n", | ||
| 109 | vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz); | ||
| 110 | } | ||
| 111 | |||
| 112 | static void probe_scache(void) | ||
| 113 | { | ||
| 114 | struct cpuinfo_loongarch *c = ¤t_cpu_data; | ||
| 115 | unsigned int lsize, sets, ways; | ||
| 116 | unsigned int config; | ||
| 117 | |||
| 118 | config = read_cpucfg(LOONGARCH_CPUCFG20); | ||
| 119 | lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE); | ||
| 120 | sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS); | ||
| 121 | ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1; | ||
| 122 | |||
| 123 | c->scache.linesz = lsize; | ||
| 124 | c->scache.sets = sets; | ||
| 125 | c->scache.ways = ways; | ||
| 126 | /* 4 cores. scaches are shared */ | ||
| 127 | scache_size = lsize * sets * ways; | ||
| 128 | c->scache.waysize = scache_size / c->scache.ways; | ||
| 129 | |||
| 130 | pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", | ||
| 131 | scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); | ||
| 132 | } | ||
| 133 | |||
| 134 | void cpu_cache_init(void) | ||
| 135 | { | ||
| 136 | probe_pcache(); | ||
| 137 | probe_vcache(); | ||
| 138 | probe_scache(); | ||
| 139 | |||
| 140 | shm_align_mask = PAGE_SIZE - 1; | ||
| 141 | } | ||
| @@ -0,0 +1,22 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/extable.h> | ||
| 6 | #include <linux/spinlock.h> | ||
| 7 | #include <asm/branch.h> | ||
| 8 | #include <linux/uaccess.h> | ||
| 9 | |||
| 10 | int fixup_exception(struct pt_regs *regs) | ||
| 11 | { | ||
| 12 | const struct exception_table_entry *fixup; | ||
| 13 | |||
| 14 | fixup = search_exception_tables(exception_era(regs)); | ||
| 15 | if (fixup) { | ||
| 16 | regs->csr_era = fixup->fixup; | ||
| 17 | |||
| 18 | return 1; | ||
| 19 | } | ||
| 20 | |||
| 21 | return 0; | ||
| 22 | } | ||
| @@ -0,0 +1,261 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | * | ||
| 5 | * Derived from MIPS: | ||
| 6 | * Copyright (C) 1995 - 2000 by Ralf Baechle | ||
| 7 | */ | ||
| 8 | #include <linux/context_tracking.h> | ||
| 9 | #include <linux/signal.h> | ||
| 10 | #include <linux/sched.h> | ||
| 11 | #include <linux/interrupt.h> | ||
| 12 | #include <linux/kernel.h> | ||
| 13 | #include <linux/entry-common.h> | ||
| 14 | #include <linux/errno.h> | ||
| 15 | #include <linux/string.h> | ||
| 16 | #include <linux/types.h> | ||
| 17 | #include <linux/ptrace.h> | ||
| 18 | #include <linux/ratelimit.h> | ||
| 19 | #include <linux/mman.h> | ||
| 20 | #include <linux/mm.h> | ||
| 21 | #include <linux/smp.h> | ||
| 22 | #include <linux/kdebug.h> | ||
| 23 | #include <linux/kprobes.h> | ||
| 24 | #include <linux/perf_event.h> | ||
| 25 | #include <linux/uaccess.h> | ||
| 26 | |||
| 27 | #include <asm/branch.h> | ||
| 28 | #include <asm/mmu_context.h> | ||
| 29 | #include <asm/ptrace.h> | ||
| 30 | |||
| 31 | int show_unhandled_signals = 1; | ||
| 32 | |||
| 33 | static void __kprobes no_context(struct pt_regs *regs, unsigned long address) | ||
| 34 | { | ||
| 35 | const int field = sizeof(unsigned long) * 2; | ||
| 36 | |||
| 37 | /* Are we prepared to handle this kernel fault? */ | ||
| 38 | if (fixup_exception(regs)) | ||
| 39 | return; | ||
| 40 | |||
| 41 | /* | ||
| 42 | * Oops. The kernel tried to access some bad page. We'll have to | ||
| 43 | * terminate things with extreme prejudice. | ||
| 44 | */ | ||
| 45 | bust_spinlocks(1); | ||
| 46 | |||
| 47 | pr_alert("CPU %d Unable to handle kernel paging request at " | ||
| 48 | "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", | ||
| 49 | raw_smp_processor_id(), field, address, field, regs->csr_era, | ||
| 50 | field, regs->regs[1]); | ||
| 51 | die("Oops", regs); | ||
| 52 | } | ||
| 53 | |||
| 54 | static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address) | ||
| 55 | { | ||
| 56 | /* | ||
| 57 | * We ran out of memory, call the OOM killer, and return the userspace | ||
| 58 | * (which will retry the fault, or kill us if we got oom-killed). | ||
| 59 | */ | ||
| 60 | if (!user_mode(regs)) { | ||
| 61 | no_context(regs, address); | ||
| 62 | return; | ||
| 63 | } | ||
| 64 | pagefault_out_of_memory(); | ||
| 65 | } | ||
| 66 | |||
| 67 | static void __kprobes do_sigbus(struct pt_regs *regs, | ||
| 68 | unsigned long write, unsigned long address, int si_code) | ||
| 69 | { | ||
| 70 | /* Kernel mode? Handle exceptions or die */ | ||
| 71 | if (!user_mode(regs)) { | ||
| 72 | no_context(regs, address); | ||
| 73 | return; | ||
| 74 | } | ||
| 75 | |||
| 76 | /* | ||
| 77 | * Send a sigbus, regardless of whether we were in kernel | ||
| 78 | * or user mode. | ||
| 79 | */ | ||
| 80 | current->thread.csr_badvaddr = address; | ||
| 81 | current->thread.trap_nr = read_csr_excode(); | ||
| 82 | force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); | ||
| 83 | } | ||
| 84 | |||
| 85 | static void __kprobes do_sigsegv(struct pt_regs *regs, | ||
| 86 | unsigned long write, unsigned long address, int si_code) | ||
| 87 | { | ||
| 88 | const int field = sizeof(unsigned long) * 2; | ||
| 89 | static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); | ||
| 90 | |||
| 91 | /* Kernel mode? Handle exceptions or die */ | ||
| 92 | if (!user_mode(regs)) { | ||
| 93 | no_context(regs, address); | ||
| 94 | return; | ||
| 95 | } | ||
| 96 | |||
| 97 | /* User mode accesses just cause a SIGSEGV */ | ||
| 98 | current->thread.csr_badvaddr = address; | ||
| 99 | if (!write) | ||
| 100 | current->thread.error_code = 1; | ||
| 101 | else | ||
| 102 | current->thread.error_code = 2; | ||
| 103 | current->thread.trap_nr = read_csr_excode(); | ||
| 104 | |||
| 105 | if (show_unhandled_signals && | ||
| 106 | unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { | ||
| 107 | pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", | ||
| 108 | current->comm, | ||
| 109 | write ? "write access to" : "read access from", | ||
| 110 | field, address); | ||
| 111 | pr_info("era = %0*lx in", field, | ||
| 112 | (unsigned long) regs->csr_era); | ||
| 113 | print_vma_addr(KERN_CONT " ", regs->csr_era); | ||
| 114 | pr_cont("\n"); | ||
| 115 | pr_info("ra = %0*lx in", field, | ||
| 116 | (unsigned long) regs->regs[1]); | ||
| 117 | print_vma_addr(KERN_CONT " ", regs->regs[1]); | ||
| 118 | pr_cont("\n"); | ||
| 119 | } | ||
| 120 | force_sig_fault(SIGSEGV, si_code, (void __user *)address); | ||
| 121 | } | ||
| 122 | |||
| 123 | /* | ||
| 124 | * This routine handles page faults. It determines the address, | ||
| 125 | * and the problem, and then passes it off to one of the appropriate | ||
| 126 | * routines. | ||
| 127 | */ | ||
| 128 | static void __kprobes __do_page_fault(struct pt_regs *regs, | ||
| 129 | unsigned long write, unsigned long address) | ||
| 130 | { | ||
| 131 | int si_code = SEGV_MAPERR; | ||
| 132 | unsigned int flags = FAULT_FLAG_DEFAULT; | ||
| 133 | struct task_struct *tsk = current; | ||
| 134 | struct mm_struct *mm = tsk->mm; | ||
| 135 | struct vm_area_struct *vma = NULL; | ||
| 136 | vm_fault_t fault; | ||
| 137 | |||
| 138 | /* | ||
| 139 | * We fault-in kernel-space virtual memory on-demand. The | ||
| 140 | * 'reference' page table is init_mm.pgd. | ||
| 141 | * | ||
| 142 | * NOTE! We MUST NOT take any locks for this case. We may | ||
| 143 | * be in an interrupt or a critical region, and should | ||
| 144 | * only copy the information from the master page table, | ||
| 145 | * nothing more. | ||
| 146 | */ | ||
| 147 | if (address & __UA_LIMIT) { | ||
| 148 | if (!user_mode(regs)) | ||
| 149 | no_context(regs, address); | ||
| 150 | else | ||
| 151 | do_sigsegv(regs, write, address, si_code); | ||
| 152 | return; | ||
| 153 | } | ||
| 154 | |||
| 155 | /* | ||
| 156 | * If we're in an interrupt or have no user | ||
| 157 | * context, we must not take the fault.. | ||
| 158 | */ | ||
| 159 | if (faulthandler_disabled() || !mm) { | ||
| 160 | do_sigsegv(regs, write, address, si_code); | ||
| 161 | return; | ||
| 162 | } | ||
| 163 | |||
| 164 | if (user_mode(regs)) | ||
| 165 | flags |= FAULT_FLAG_USER; | ||
| 166 | |||
| 167 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); | ||
| 168 | retry: | ||
| 169 | mmap_read_lock(mm); | ||
| 170 | vma = find_vma(mm, address); | ||
| 171 | if (!vma) | ||
| 172 | goto bad_area; | ||
| 173 | if (vma->vm_start <= address) | ||
| 174 | goto good_area; | ||
| 175 | if (!(vma->vm_flags & VM_GROWSDOWN)) | ||
| 176 | goto bad_area; | ||
| 177 | if (!expand_stack(vma, address)) | ||
| 178 | goto good_area; | ||
| 179 | /* | ||
| 180 | * Something tried to access memory that isn't in our memory map.. | ||
| 181 | * Fix it, but check if it's kernel or user first.. | ||
| 182 | */ | ||
| 183 | bad_area: | ||
| 184 | mmap_read_unlock(mm); | ||
| 185 | do_sigsegv(regs, write, address, si_code); | ||
| 186 | return; | ||
| 187 | |||
| 188 | /* | ||
| 189 | * Ok, we have a good vm_area for this memory access, so | ||
| 190 | * we can handle it.. | ||
| 191 | */ | ||
| 192 | good_area: | ||
| 193 | si_code = SEGV_ACCERR; | ||
| 194 | |||
| 195 | if (write) { | ||
| 196 | flags |= FAULT_FLAG_WRITE; | ||
| 197 | if (!(vma->vm_flags & VM_WRITE)) | ||
| 198 | goto bad_area; | ||
| 199 | } else { | ||
| 200 | if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) | ||
| 201 | goto bad_area; | ||
| 202 | if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) | ||
| 203 | goto bad_area; | ||
| 204 | } | ||
| 205 | |||
| 206 | /* | ||
| 207 | * If for any reason at all we couldn't handle the fault, | ||
| 208 | * make sure we exit gracefully rather than endlessly redo | ||
| 209 | * the fault. | ||
| 210 | */ | ||
| 211 | fault = handle_mm_fault(vma, address, flags, regs); | ||
| 212 | |||
| 213 | if (fault_signal_pending(fault, regs)) { | ||
| 214 | if (!user_mode(regs)) | ||
| 215 | no_context(regs, address); | ||
| 216 | return; | ||
| 217 | } | ||
| 218 | |||
| 219 | if (unlikely(fault & VM_FAULT_RETRY)) { | ||
| 220 | flags |= FAULT_FLAG_TRIED; | ||
| 221 | |||
| 222 | /* | ||
| 223 | * No need to mmap_read_unlock(mm) as we would | ||
| 224 | * have already released it in __lock_page_or_retry | ||
| 225 | * in mm/filemap.c. | ||
| 226 | */ | ||
| 227 | goto retry; | ||
| 228 | } | ||
| 229 | if (unlikely(fault & VM_FAULT_ERROR)) { | ||
| 230 | mmap_read_unlock(mm); | ||
| 231 | if (fault & VM_FAULT_OOM) { | ||
| 232 | do_out_of_memory(regs, address); | ||
| 233 | return; | ||
| 234 | } else if (fault & VM_FAULT_SIGSEGV) { | ||
| 235 | do_sigsegv(regs, write, address, si_code); | ||
| 236 | return; | ||
| 237 | } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { | ||
| 238 | do_sigbus(regs, write, address, si_code); | ||
| 239 | return; | ||
| 240 | } | ||
| 241 | BUG(); | ||
| 242 | } | ||
| 243 | |||
| 244 | mmap_read_unlock(mm); | ||
| 245 | } | ||
| 246 | |||
| 247 | asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, | ||
| 248 | unsigned long write, unsigned long address) | ||
| 249 | { | ||
| 250 | irqentry_state_t state = irqentry_enter(regs); | ||
| 251 | |||
| 252 | /* Enable interrupt if enabled in parent context */ | ||
| 253 | if (likely(regs->csr_prmd & CSR_PRMD_PIE)) | ||
| 254 | local_irq_enable(); | ||
| 255 | |||
| 256 | __do_page_fault(regs, write, address); | ||
| 257 | |||
| 258 | local_irq_disable(); | ||
| 259 | |||
| 260 | irqentry_exit(regs, state); | ||
| 261 | } | ||
| @@ -0,0 +1,87 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | |||
| 6 | #include <linux/fs.h> | ||
| 7 | #include <linux/mm.h> | ||
| 8 | #include <linux/hugetlb.h> | ||
| 9 | #include <linux/pagemap.h> | ||
| 10 | #include <linux/err.h> | ||
| 11 | #include <linux/sysctl.h> | ||
| 12 | #include <asm/mman.h> | ||
| 13 | #include <asm/tlb.h> | ||
| 14 | #include <asm/tlbflush.h> | ||
| 15 | |||
| 16 | pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, | ||
| 17 | unsigned long addr, unsigned long sz) | ||
| 18 | { | ||
| 19 | pgd_t *pgd; | ||
| 20 | p4d_t *p4d; | ||
| 21 | pud_t *pud; | ||
| 22 | pte_t *pte = NULL; | ||
| 23 | |||
| 24 | pgd = pgd_offset(mm, addr); | ||
| 25 | p4d = p4d_alloc(mm, pgd, addr); | ||
| 26 | pud = pud_alloc(mm, p4d, addr); | ||
| 27 | if (pud) | ||
| 28 | pte = (pte_t *)pmd_alloc(mm, pud, addr); | ||
| 29 | |||
| 30 | return pte; | ||
| 31 | } | ||
| 32 | |||
| 33 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, | ||
| 34 | unsigned long sz) | ||
| 35 | { | ||
| 36 | pgd_t *pgd; | ||
| 37 | p4d_t *p4d; | ||
| 38 | pud_t *pud; | ||
| 39 | pmd_t *pmd = NULL; | ||
| 40 | |||
| 41 | pgd = pgd_offset(mm, addr); | ||
| 42 | if (pgd_present(*pgd)) { | ||
| 43 | p4d = p4d_offset(pgd, addr); | ||
| 44 | if (p4d_present(*p4d)) { | ||
| 45 | pud = pud_offset(p4d, addr); | ||
| 46 | if (pud_present(*pud)) | ||
| 47 | pmd = pmd_offset(pud, addr); | ||
| 48 | } | ||
| 49 | } | ||
| 50 | return (pte_t *) pmd; | ||
| 51 | } | ||
| 52 | |||
| 53 | /* | ||
| 54 | * This function checks for proper alignment of input addr and len parameters. | ||
| 55 | */ | ||
| 56 | int is_aligned_hugepage_range(unsigned long addr, unsigned long len) | ||
| 57 | { | ||
| 58 | if (len & ~HPAGE_MASK) | ||
| 59 | return -EINVAL; | ||
| 60 | if (addr & ~HPAGE_MASK) | ||
| 61 | return -EINVAL; | ||
| 62 | return 0; | ||
| 63 | } | ||
| 64 | |||
| 65 | int pmd_huge(pmd_t pmd) | ||
| 66 | { | ||
| 67 | return (pmd_val(pmd) & _PAGE_HUGE) != 0; | ||
| 68 | } | ||
| 69 | |||
| 70 | int pud_huge(pud_t pud) | ||
| 71 | { | ||
| 72 | return (pud_val(pud) & _PAGE_HUGE) != 0; | ||
| 73 | } | ||
| 74 | |||
| 75 | uint64_t pmd_to_entrylo(unsigned long pmd_val) | ||
| 76 | { | ||
| 77 | uint64_t val; | ||
| 78 | /* PMD as PTE. Must be huge page */ | ||
| 79 | if (!pmd_huge(__pmd(pmd_val))) | ||
| 80 | panic("%s", __func__); | ||
| 81 | |||
| 82 | val = pmd_val ^ _PAGE_HUGE; | ||
| 83 | val |= ((val & _PAGE_HGLOBAL) >> | ||
| 84 | (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); | ||
| 85 | |||
| 86 | return val; | ||
| 87 | } | ||
| @@ -0,0 +1,165 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/init.h> | ||
| 6 | #include <linux/export.h> | ||
| 7 | #include <linux/signal.h> | ||
| 8 | #include <linux/sched.h> | ||
| 9 | #include <linux/smp.h> | ||
| 10 | #include <linux/kernel.h> | ||
| 11 | #include <linux/errno.h> | ||
| 12 | #include <linux/string.h> | ||
| 13 | #include <linux/types.h> | ||
| 14 | #include <linux/pagemap.h> | ||
| 15 | #include <linux/memblock.h> | ||
| 16 | #include <linux/memremap.h> | ||
| 17 | #include <linux/mm.h> | ||
| 18 | #include <linux/mman.h> | ||
| 19 | #include <linux/highmem.h> | ||
| 20 | #include <linux/swap.h> | ||
| 21 | #include <linux/proc_fs.h> | ||
| 22 | #include <linux/pfn.h> | ||
| 23 | #include <linux/hardirq.h> | ||
| 24 | #include <linux/gfp.h> | ||
| 25 | #include <linux/initrd.h> | ||
| 26 | #include <linux/mmzone.h> | ||
| 27 | |||
| 28 | #include <asm/asm-offsets.h> | ||
| 29 | #include <asm/bootinfo.h> | ||
| 30 | #include <asm/cpu.h> | ||
| 31 | #include <asm/dma.h> | ||
| 32 | #include <asm/mmu_context.h> | ||
| 33 | #include <asm/sections.h> | ||
| 34 | #include <asm/pgtable.h> | ||
| 35 | #include <asm/pgalloc.h> | ||
| 36 | #include <asm/tlb.h> | ||
| 37 | |||
| 38 | /* | ||
| 39 | * We have up to 8 empty zeroed pages so we can map one of the right colour | ||
| 40 | * when needed. Since page is never written to after the initialization we | ||
| 41 | * don't have to care about aliases on other CPUs. | ||
| 42 | */ | ||
| 43 | unsigned long empty_zero_page, zero_page_mask; | ||
| 44 | EXPORT_SYMBOL_GPL(empty_zero_page); | ||
| 45 | EXPORT_SYMBOL(zero_page_mask); | ||
| 46 | |||
| 47 | void setup_zero_pages(void) | ||
| 48 | { | ||
| 49 | unsigned int order, i; | ||
| 50 | struct page *page; | ||
| 51 | |||
| 52 | order = 0; | ||
| 53 | |||
| 54 | empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); | ||
| 55 | if (!empty_zero_page) | ||
| 56 | panic("Oh boy, that early out of memory?"); | ||
| 57 | |||
| 58 | page = virt_to_page((void *)empty_zero_page); | ||
| 59 | split_page(page, order); | ||
| 60 | for (i = 0; i < (1 << order); i++, page++) | ||
| 61 | mark_page_reserved(page); | ||
| 62 | |||
| 63 | zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; | ||
| 64 | } | ||
| 65 | |||
| 66 | void copy_user_highpage(struct page *to, struct page *from, | ||
| 67 | unsigned long vaddr, struct vm_area_struct *vma) | ||
| 68 | { | ||
| 69 | void *vfrom, *vto; | ||
| 70 | |||
| 71 | vto = kmap_atomic(to); | ||
| 72 | vfrom = kmap_atomic(from); | ||
| 73 | copy_page(vto, vfrom); | ||
| 74 | kunmap_atomic(vfrom); | ||
| 75 | kunmap_atomic(vto); | ||
| 76 | /* Make sure this page is cleared on other CPU's too before using it */ | ||
| 77 | smp_wmb(); | ||
| 78 | } | ||
| 79 | |||
| 80 | int __ref page_is_ram(unsigned long pfn) | ||
| 81 | { | ||
| 82 | unsigned long addr = PFN_PHYS(pfn); | ||
| 83 | |||
| 84 | return memblock_is_memory(addr) && !memblock_is_reserved(addr); | ||
| 85 | } | ||
| 86 | |||
| 87 | void __init paging_init(void) | ||
| 88 | { | ||
| 89 | unsigned long max_zone_pfns[MAX_NR_ZONES]; | ||
| 90 | |||
| 91 | #ifdef CONFIG_ZONE_DMA | ||
| 92 | max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; | ||
| 93 | #endif | ||
| 94 | #ifdef CONFIG_ZONE_DMA32 | ||
| 95 | max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; | ||
| 96 | #endif | ||
| 97 | max_zone_pfns[ZONE_NORMAL] = max_low_pfn; | ||
| 98 | |||
| 99 | free_area_init(max_zone_pfns); | ||
| 100 | } | ||
| 101 | |||
| 102 | void __init mem_init(void) | ||
| 103 | { | ||
| 104 | max_mapnr = max_low_pfn; | ||
| 105 | high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); | ||
| 106 | |||
| 107 | memblock_free_all(); | ||
| 108 | setup_zero_pages(); /* Setup zeroed pages. */ | ||
| 109 | } | ||
| 110 | |||
| 111 | void __ref free_initmem(void) | ||
| 112 | { | ||
| 113 | free_initmem_default(POISON_FREE_INITMEM); | ||
| 114 | } | ||
| 115 | |||
| 116 | #ifdef CONFIG_MEMORY_HOTPLUG | ||
| 117 | int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) | ||
| 118 | { | ||
| 119 | unsigned long start_pfn = start >> PAGE_SHIFT; | ||
| 120 | unsigned long nr_pages = size >> PAGE_SHIFT; | ||
| 121 | int ret; | ||
| 122 | |||
| 123 | ret = __add_pages(nid, start_pfn, nr_pages, params); | ||
| 124 | |||
| 125 | if (ret) | ||
| 126 | pr_warn("%s: Problem encountered in __add_pages() as ret=%d\n", | ||
| 127 | __func__, ret); | ||
| 128 | |||
| 129 | return ret; | ||
| 130 | } | ||
| 131 | |||
| 132 | #ifdef CONFIG_MEMORY_HOTREMOVE | ||
| 133 | void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) | ||
| 134 | { | ||
| 135 | unsigned long start_pfn = start >> PAGE_SHIFT; | ||
| 136 | unsigned long nr_pages = size >> PAGE_SHIFT; | ||
| 137 | struct page *page = pfn_to_page(start_pfn); | ||
| 138 | |||
| 139 | /* With altmap the first mapped page is offset from @start */ | ||
| 140 | if (altmap) | ||
| 141 | page += vmem_altmap_offset(altmap); | ||
| 142 | __remove_pages(start_pfn, nr_pages, altmap); | ||
| 143 | } | ||
| 144 | #endif | ||
| 145 | #endif | ||
| 146 | |||
| 147 | /* | ||
| 148 | * Align swapper_pg_dir in to 64K, allows its address to be loaded | ||
| 149 | * with a single LUI instruction in the TLB handlers. If we used | ||
| 150 | * __aligned(64K), its size would get rounded up to the alignment | ||
| 151 | * size, and waste space. So we place it in its own section and align | ||
| 152 | * it in the linker script. | ||
| 153 | */ | ||
| 154 | pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); | ||
| 155 | |||
| 156 | pgd_t invalid_pg_dir[_PTRS_PER_PGD] __page_aligned_bss; | ||
| 157 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 158 | pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss; | ||
| 159 | #endif | ||
| 160 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 161 | pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss; | ||
| 162 | EXPORT_SYMBOL_GPL(invalid_pmd_table); | ||
| 163 | #endif | ||
| 164 | pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; | ||
| 165 | EXPORT_SYMBOL(invalid_pte_table); | ||
| @@ -0,0 +1,27 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | |||
| 6 | #include <asm/io.h> | ||
| 7 | |||
| 8 | void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size) | ||
| 9 | { | ||
| 10 | return ((void __iomem *)TO_CACHE(phys_addr)); | ||
| 11 | } | ||
| 12 | |||
| 13 | void __init early_iounmap(void __iomem *addr, unsigned long size) | ||
| 14 | { | ||
| 15 | |||
| 16 | } | ||
| 17 | |||
| 18 | void *early_memremap_ro(resource_size_t phys_addr, unsigned long size) | ||
| 19 | { | ||
| 20 | return early_memremap(phys_addr, size); | ||
| 21 | } | ||
| 22 | |||
| 23 | void *early_memremap_prot(resource_size_t phys_addr, unsigned long size, | ||
| 24 | unsigned long prot_val) | ||
| 25 | { | ||
| 26 | return early_memremap(phys_addr, size); | ||
| 27 | } | ||
| @@ -0,0 +1,10 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0-only | ||
| 2 | |||
| 3 | #include <linux/uaccess.h> | ||
| 4 | #include <linux/kernel.h> | ||
| 5 | |||
| 6 | bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) | ||
| 7 | { | ||
| 8 | /* highest bit set means kernel space */ | ||
| 9 | return (unsigned long)unsafe_src >> (BITS_PER_LONG - 1); | ||
| 10 | } | ||
| @@ -0,0 +1,125 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/compiler.h> | ||
| 6 | #include <linux/elf-randomize.h> | ||
| 7 | #include <linux/errno.h> | ||
| 8 | #include <linux/mm.h> | ||
| 9 | #include <linux/mman.h> | ||
| 10 | #include <linux/export.h> | ||
| 11 | #include <linux/personality.h> | ||
| 12 | #include <linux/random.h> | ||
| 13 | #include <linux/sched/signal.h> | ||
| 14 | #include <linux/sched/mm.h> | ||
| 15 | |||
| 16 | unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */ | ||
| 17 | EXPORT_SYMBOL(shm_align_mask); | ||
| 18 | |||
| 19 | #define COLOUR_ALIGN(addr, pgoff) \ | ||
| 20 | ((((addr) + shm_align_mask) & ~shm_align_mask) + \ | ||
| 21 | (((pgoff) << PAGE_SHIFT) & shm_align_mask)) | ||
| 22 | |||
| 23 | enum mmap_allocation_direction {UP, DOWN}; | ||
| 24 | |||
| 25 | static unsigned long arch_get_unmapped_area_common(struct file *filp, | ||
| 26 | unsigned long addr0, unsigned long len, unsigned long pgoff, | ||
| 27 | unsigned long flags, enum mmap_allocation_direction dir) | ||
| 28 | { | ||
| 29 | struct mm_struct *mm = current->mm; | ||
| 30 | struct vm_area_struct *vma; | ||
| 31 | unsigned long addr = addr0; | ||
| 32 | int do_color_align; | ||
| 33 | struct vm_unmapped_area_info info; | ||
| 34 | |||
| 35 | if (unlikely(len > TASK_SIZE)) | ||
| 36 | return -ENOMEM; | ||
| 37 | |||
| 38 | if (flags & MAP_FIXED) { | ||
| 39 | /* Even MAP_FIXED mappings must reside within TASK_SIZE */ | ||
| 40 | if (TASK_SIZE - len < addr) | ||
| 41 | return -EINVAL; | ||
| 42 | |||
| 43 | /* | ||
| 44 | * We do not accept a shared mapping if it would violate | ||
| 45 | * cache aliasing constraints. | ||
| 46 | */ | ||
| 47 | if ((flags & MAP_SHARED) && | ||
| 48 | ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask)) | ||
| 49 | return -EINVAL; | ||
| 50 | return addr; | ||
| 51 | } | ||
| 52 | |||
| 53 | do_color_align = 0; | ||
| 54 | if (filp || (flags & MAP_SHARED)) | ||
| 55 | do_color_align = 1; | ||
| 56 | |||
| 57 | /* requesting a specific address */ | ||
| 58 | if (addr) { | ||
| 59 | if (do_color_align) | ||
| 60 | addr = COLOUR_ALIGN(addr, pgoff); | ||
| 61 | else | ||
| 62 | addr = PAGE_ALIGN(addr); | ||
| 63 | |||
| 64 | vma = find_vma(mm, addr); | ||
| 65 | if (TASK_SIZE - len >= addr && | ||
| 66 | (!vma || addr + len <= vm_start_gap(vma))) | ||
| 67 | return addr; | ||
| 68 | } | ||
| 69 | |||
| 70 | info.length = len; | ||
| 71 | info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0; | ||
| 72 | info.align_offset = pgoff << PAGE_SHIFT; | ||
| 73 | |||
| 74 | if (dir == DOWN) { | ||
| 75 | info.flags = VM_UNMAPPED_AREA_TOPDOWN; | ||
| 76 | info.low_limit = PAGE_SIZE; | ||
| 77 | info.high_limit = mm->mmap_base; | ||
| 78 | addr = vm_unmapped_area(&info); | ||
| 79 | |||
| 80 | if (!(addr & ~PAGE_MASK)) | ||
| 81 | return addr; | ||
| 82 | |||
| 83 | /* | ||
| 84 | * A failed mmap() very likely causes application failure, | ||
| 85 | * so fall back to the bottom-up function here. This scenario | ||
| 86 | * can happen with large stack limits and large mmap() | ||
| 87 | * allocations. | ||
| 88 | */ | ||
| 89 | } | ||
| 90 | |||
| 91 | info.flags = 0; | ||
| 92 | info.low_limit = mm->mmap_base; | ||
| 93 | info.high_limit = TASK_SIZE; | ||
| 94 | return vm_unmapped_area(&info); | ||
| 95 | } | ||
| 96 | |||
| 97 | unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0, | ||
| 98 | unsigned long len, unsigned long pgoff, unsigned long flags) | ||
| 99 | { | ||
| 100 | return arch_get_unmapped_area_common(filp, | ||
| 101 | addr0, len, pgoff, flags, UP); | ||
| 102 | } | ||
| 103 | |||
| 104 | /* | ||
| 105 | * There is no need to export this but sched.h declares the function as | ||
| 106 | * extern so making it static here results in an error. | ||
| 107 | */ | ||
| 108 | unsigned long arch_get_unmapped_area_topdown(struct file *filp, | ||
| 109 | unsigned long addr0, unsigned long len, unsigned long pgoff, | ||
| 110 | unsigned long flags) | ||
| 111 | { | ||
| 112 | return arch_get_unmapped_area_common(filp, | ||
| 113 | addr0, len, pgoff, flags, DOWN); | ||
| 114 | } | ||
| 115 | |||
| 116 | int __virt_addr_valid(volatile void *kaddr) | ||
| 117 | { | ||
| 118 | unsigned long vaddr = (unsigned long)kaddr; | ||
| 119 | |||
| 120 | if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base)) | ||
| 121 | return 0; | ||
| 122 | |||
| 123 | return pfn_valid(PFN_DOWN(virt_to_phys(kaddr))); | ||
| 124 | } | ||
| 125 | EXPORT_SYMBOL_GPL(__virt_addr_valid); | ||
| @@ -0,0 +1,84 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/linkage.h> | ||
| 6 | #include <asm/asm.h> | ||
| 7 | #include <asm/export.h> | ||
| 8 | #include <asm/page.h> | ||
| 9 | #include <asm/regdef.h> | ||
| 10 | |||
| 11 | .align 5 | ||
| 12 | SYM_FUNC_START(clear_page) | ||
| 13 | lu12i.w t0, 1 << (PAGE_SHIFT - 12) | ||
| 14 | add.d t0, t0, a0 | ||
| 15 | 1: | ||
| 16 | st.d zero, a0, 0 | ||
| 17 | st.d zero, a0, 8 | ||
| 18 | st.d zero, a0, 16 | ||
| 19 | st.d zero, a0, 24 | ||
| 20 | st.d zero, a0, 32 | ||
| 21 | st.d zero, a0, 40 | ||
| 22 | st.d zero, a0, 48 | ||
| 23 | st.d zero, a0, 56 | ||
| 24 | addi.d a0, a0, 128 | ||
| 25 | st.d zero, a0, -64 | ||
| 26 | st.d zero, a0, -56 | ||
| 27 | st.d zero, a0, -48 | ||
| 28 | st.d zero, a0, -40 | ||
| 29 | st.d zero, a0, -32 | ||
| 30 | st.d zero, a0, -24 | ||
| 31 | st.d zero, a0, -16 | ||
| 32 | st.d zero, a0, -8 | ||
| 33 | bne t0, a0, 1b | ||
| 34 | |||
| 35 | jirl $r0, ra, 0 | ||
| 36 | SYM_FUNC_END(clear_page) | ||
| 37 | EXPORT_SYMBOL(clear_page) | ||
| 38 | |||
| 39 | .align 5 | ||
| 40 | SYM_FUNC_START(copy_page) | ||
| 41 | lu12i.w t8, 1 << (PAGE_SHIFT - 12) | ||
| 42 | add.d t8, t8, a0 | ||
| 43 | 1: | ||
| 44 | ld.d t0, a1, 0 | ||
| 45 | ld.d t1, a1, 8 | ||
| 46 | ld.d t2, a1, 16 | ||
| 47 | ld.d t3, a1, 24 | ||
| 48 | ld.d t4, a1, 32 | ||
| 49 | ld.d t5, a1, 40 | ||
| 50 | ld.d t6, a1, 48 | ||
| 51 | ld.d t7, a1, 56 | ||
| 52 | |||
| 53 | st.d t0, a0, 0 | ||
| 54 | st.d t1, a0, 8 | ||
| 55 | ld.d t0, a1, 64 | ||
| 56 | ld.d t1, a1, 72 | ||
| 57 | st.d t2, a0, 16 | ||
| 58 | st.d t3, a0, 24 | ||
| 59 | ld.d t2, a1, 80 | ||
| 60 | ld.d t3, a1, 88 | ||
| 61 | st.d t4, a0, 32 | ||
| 62 | st.d t5, a0, 40 | ||
| 63 | ld.d t4, a1, 96 | ||
| 64 | ld.d t5, a1, 104 | ||
| 65 | st.d t6, a0, 48 | ||
| 66 | st.d t7, a0, 56 | ||
| 67 | ld.d t6, a1, 112 | ||
| 68 | ld.d t7, a1, 120 | ||
| 69 | addi.d a0, a0, 128 | ||
| 70 | addi.d a1, a1, 128 | ||
| 71 | |||
| 72 | st.d t0, a0, -64 | ||
| 73 | st.d t1, a0, -56 | ||
| 74 | st.d t2, a0, -48 | ||
| 75 | st.d t3, a0, -40 | ||
| 76 | st.d t4, a0, -32 | ||
| 77 | st.d t5, a0, -24 | ||
| 78 | st.d t6, a0, -16 | ||
| 79 | st.d t7, a0, -8 | ||
| 80 | |||
| 81 | bne t8, a0, 1b | ||
| 82 | jirl $r0, ra, 0 | ||
| 83 | SYM_FUNC_END(copy_page) | ||
| 84 | EXPORT_SYMBOL(copy_page) | ||
| @@ -0,0 +1,130 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/init.h> | ||
| 6 | #include <linux/export.h> | ||
| 7 | #include <linux/mm.h> | ||
| 8 | #include <asm/pgalloc.h> | ||
| 9 | #include <asm/pgtable.h> | ||
| 10 | #include <asm/tlbflush.h> | ||
| 11 | |||
| 12 | pgd_t *pgd_alloc(struct mm_struct *mm) | ||
| 13 | { | ||
| 14 | pgd_t *ret, *init; | ||
| 15 | |||
| 16 | ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER); | ||
| 17 | if (ret) { | ||
| 18 | init = pgd_offset(&init_mm, 0UL); | ||
| 19 | pgd_init((unsigned long)ret); | ||
| 20 | memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, | ||
| 21 | (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); | ||
| 22 | } | ||
| 23 | |||
| 24 | return ret; | ||
| 25 | } | ||
| 26 | EXPORT_SYMBOL_GPL(pgd_alloc); | ||
| 27 | |||
| 28 | void pgd_init(unsigned long page) | ||
| 29 | { | ||
| 30 | unsigned long *p, *end; | ||
| 31 | unsigned long entry; | ||
| 32 | |||
| 33 | #if !defined(__PAGETABLE_PUD_FOLDED) | ||
| 34 | entry = (unsigned long)invalid_pud_table; | ||
| 35 | #elif !defined(__PAGETABLE_PMD_FOLDED) | ||
| 36 | entry = (unsigned long)invalid_pmd_table; | ||
| 37 | #else | ||
| 38 | entry = (unsigned long)invalid_pte_table; | ||
| 39 | #endif | ||
| 40 | |||
| 41 | p = (unsigned long *) page; | ||
| 42 | end = p + PTRS_PER_PGD; | ||
| 43 | |||
| 44 | do { | ||
| 45 | p[0] = entry; | ||
| 46 | p[1] = entry; | ||
| 47 | p[2] = entry; | ||
| 48 | p[3] = entry; | ||
| 49 | p[4] = entry; | ||
| 50 | p += 8; | ||
| 51 | p[-3] = entry; | ||
| 52 | p[-2] = entry; | ||
| 53 | p[-1] = entry; | ||
| 54 | } while (p != end); | ||
| 55 | } | ||
| 56 | EXPORT_SYMBOL_GPL(pgd_init); | ||
| 57 | |||
| 58 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 59 | void pmd_init(unsigned long addr, unsigned long pagetable) | ||
| 60 | { | ||
| 61 | unsigned long *p, *end; | ||
| 62 | |||
| 63 | p = (unsigned long *) addr; | ||
| 64 | end = p + PTRS_PER_PMD; | ||
| 65 | |||
| 66 | do { | ||
| 67 | p[0] = pagetable; | ||
| 68 | p[1] = pagetable; | ||
| 69 | p[2] = pagetable; | ||
| 70 | p[3] = pagetable; | ||
| 71 | p[4] = pagetable; | ||
| 72 | p += 8; | ||
| 73 | p[-3] = pagetable; | ||
| 74 | p[-2] = pagetable; | ||
| 75 | p[-1] = pagetable; | ||
| 76 | } while (p != end); | ||
| 77 | } | ||
| 78 | EXPORT_SYMBOL_GPL(pmd_init); | ||
| 79 | #endif | ||
| 80 | |||
| 81 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 82 | void pud_init(unsigned long addr, unsigned long pagetable) | ||
| 83 | { | ||
| 84 | unsigned long *p, *end; | ||
| 85 | |||
| 86 | p = (unsigned long *)addr; | ||
| 87 | end = p + PTRS_PER_PUD; | ||
| 88 | |||
| 89 | do { | ||
| 90 | p[0] = pagetable; | ||
| 91 | p[1] = pagetable; | ||
| 92 | p[2] = pagetable; | ||
| 93 | p[3] = pagetable; | ||
| 94 | p[4] = pagetable; | ||
| 95 | p += 8; | ||
| 96 | p[-3] = pagetable; | ||
| 97 | p[-2] = pagetable; | ||
| 98 | p[-1] = pagetable; | ||
| 99 | } while (p != end); | ||
| 100 | } | ||
| 101 | #endif | ||
| 102 | |||
| 103 | pmd_t mk_pmd(struct page *page, pgprot_t prot) | ||
| 104 | { | ||
| 105 | pmd_t pmd; | ||
| 106 | |||
| 107 | pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); | ||
| 108 | |||
| 109 | return pmd; | ||
| 110 | } | ||
| 111 | |||
| 112 | void set_pmd_at(struct mm_struct *mm, unsigned long addr, | ||
| 113 | pmd_t *pmdp, pmd_t pmd) | ||
| 114 | { | ||
| 115 | *pmdp = pmd; | ||
| 116 | flush_tlb_all(); | ||
| 117 | } | ||
| 118 | |||
| 119 | void __init pagetable_init(void) | ||
| 120 | { | ||
| 121 | /* Initialize the entire pgd. */ | ||
| 122 | pgd_init((unsigned long)swapper_pg_dir); | ||
| 123 | pgd_init((unsigned long)invalid_pg_dir); | ||
| 124 | #ifndef __PAGETABLE_PUD_FOLDED | ||
| 125 | pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table); | ||
| 126 | #endif | ||
| 127 | #ifndef __PAGETABLE_PMD_FOLDED | ||
| 128 | pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table); | ||
| 129 | #endif | ||
| 130 | } | ||
| @@ -0,0 +1,282 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <linux/init.h> | ||
| 6 | #include <linux/sched.h> | ||
| 7 | #include <linux/smp.h> | ||
| 8 | #include <linux/mm.h> | ||
| 9 | #include <linux/hugetlb.h> | ||
| 10 | #include <linux/export.h> | ||
| 11 | |||
| 12 | #include <asm/cpu.h> | ||
| 13 | #include <asm/bootinfo.h> | ||
| 14 | #include <asm/mmu_context.h> | ||
| 15 | #include <asm/pgtable.h> | ||
| 16 | #include <asm/tlb.h> | ||
| 17 | |||
| 18 | void local_flush_tlb_all(void) | ||
| 19 | { | ||
| 20 | invtlb_all(INVTLB_CURRENT_ALL, 0, 0); | ||
| 21 | } | ||
| 22 | EXPORT_SYMBOL(local_flush_tlb_all); | ||
| 23 | |||
| 24 | void local_flush_tlb_user(void) | ||
| 25 | { | ||
| 26 | invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0); | ||
| 27 | } | ||
| 28 | EXPORT_SYMBOL(local_flush_tlb_user); | ||
| 29 | |||
| 30 | void local_flush_tlb_kernel(void) | ||
| 31 | { | ||
| 32 | invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0); | ||
| 33 | } | ||
| 34 | EXPORT_SYMBOL(local_flush_tlb_kernel); | ||
| 35 | |||
| 36 | /* | ||
| 37 | * All entries common to a mm share an asid. To effectively flush | ||
| 38 | * these entries, we just bump the asid. | ||
| 39 | */ | ||
| 40 | void local_flush_tlb_mm(struct mm_struct *mm) | ||
| 41 | { | ||
| 42 | int cpu; | ||
| 43 | |||
| 44 | preempt_disable(); | ||
| 45 | |||
| 46 | cpu = smp_processor_id(); | ||
| 47 | |||
| 48 | if (asid_valid(mm, cpu)) | ||
| 49 | drop_mmu_context(mm, cpu); | ||
| 50 | else | ||
| 51 | cpumask_clear_cpu(cpu, mm_cpumask(mm)); | ||
| 52 | |||
| 53 | preempt_enable(); | ||
| 54 | } | ||
| 55 | |||
| 56 | void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, | ||
| 57 | unsigned long end) | ||
| 58 | { | ||
| 59 | struct mm_struct *mm = vma->vm_mm; | ||
| 60 | int cpu = smp_processor_id(); | ||
| 61 | |||
| 62 | if (asid_valid(mm, cpu)) { | ||
| 63 | unsigned long size, flags; | ||
| 64 | |||
| 65 | local_irq_save(flags); | ||
| 66 | start = round_down(start, PAGE_SIZE << 1); | ||
| 67 | end = round_up(end, PAGE_SIZE << 1); | ||
| 68 | size = (end - start) >> (PAGE_SHIFT + 1); | ||
| 69 | if (size <= (current_cpu_data.tlbsizestlbsets ? | ||
| 70 | current_cpu_data.tlbsize / 8 : | ||
| 71 | current_cpu_data.tlbsize / 2)) { | ||
| 72 | int asid = cpu_asid(cpu, mm); | ||
| 73 | |||
| 74 | while (start < end) { | ||
| 75 | invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start); | ||
| 76 | start += (PAGE_SIZE << 1); | ||
| 77 | } | ||
| 78 | } else { | ||
| 79 | drop_mmu_context(mm, cpu); | ||
| 80 | } | ||
| 81 | local_irq_restore(flags); | ||
| 82 | } else { | ||
| 83 | cpumask_clear_cpu(cpu, mm_cpumask(mm)); | ||
| 84 | } | ||
| 85 | } | ||
| 86 | |||
| 87 | void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) | ||
| 88 | { | ||
| 89 | unsigned long size, flags; | ||
| 90 | |||
| 91 | local_irq_save(flags); | ||
| 92 | size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; | ||
| 93 | size = (size + 1) >> 1; | ||
| 94 | if (size <= (current_cpu_data.tlbsizestlbsets ? | ||
| 95 | current_cpu_data.tlbsize / 8 : | ||
| 96 | current_cpu_data.tlbsize / 2)) { | ||
| 97 | |||
| 98 | start &= (PAGE_MASK << 1); | ||
| 99 | end += ((PAGE_SIZE << 1) - 1); | ||
| 100 | end &= (PAGE_MASK << 1); | ||
| 101 | |||
| 102 | while (start < end) { | ||
| 103 | invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start); | ||
| 104 | start += (PAGE_SIZE << 1); | ||
| 105 | } | ||
| 106 | } else { | ||
| 107 | local_flush_tlb_kernel(); | ||
| 108 | } | ||
| 109 | local_irq_restore(flags); | ||
| 110 | } | ||
| 111 | |||
| 112 | void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) | ||
| 113 | { | ||
| 114 | int cpu = smp_processor_id(); | ||
| 115 | |||
| 116 | if (asid_valid(vma->vm_mm, cpu)) { | ||
| 117 | int newpid; | ||
| 118 | |||
| 119 | newpid = cpu_asid(cpu, vma->vm_mm); | ||
| 120 | page &= (PAGE_MASK << 1); | ||
| 121 | invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page); | ||
| 122 | } else { | ||
| 123 | cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm)); | ||
| 124 | } | ||
| 125 | } | ||
| 126 | |||
| 127 | /* | ||
| 128 | * This one is only used for pages with the global bit set so we don't care | ||
| 129 | * much about the ASID. | ||
| 130 | */ | ||
| 131 | void local_flush_tlb_one(unsigned long page) | ||
| 132 | { | ||
| 133 | page &= (PAGE_MASK << 1); | ||
| 134 | invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page); | ||
| 135 | } | ||
| 136 | |||
| 137 | static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) | ||
| 138 | { | ||
| 139 | #ifdef CONFIG_HUGETLB_PAGE | ||
| 140 | int idx; | ||
| 141 | unsigned long lo; | ||
| 142 | unsigned long flags; | ||
| 143 | |||
| 144 | local_irq_save(flags); | ||
| 145 | |||
| 146 | address &= (PAGE_MASK << 1); | ||
| 147 | write_csr_entryhi(address); | ||
| 148 | tlb_probe(); | ||
| 149 | idx = read_csr_tlbidx(); | ||
| 150 | write_csr_pagesize(PS_HUGE_SIZE); | ||
| 151 | lo = pmd_to_entrylo(pte_val(*ptep)); | ||
| 152 | write_csr_entrylo0(lo); | ||
| 153 | write_csr_entrylo1(lo + (HPAGE_SIZE >> 1)); | ||
| 154 | |||
| 155 | if (idx < 0) | ||
| 156 | tlb_write_random(); | ||
| 157 | else | ||
| 158 | tlb_write_indexed(); | ||
| 159 | write_csr_pagesize(PS_DEFAULT_SIZE); | ||
| 160 | |||
| 161 | local_irq_restore(flags); | ||
| 162 | #endif | ||
| 163 | } | ||
| 164 | |||
| 165 | void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) | ||
| 166 | { | ||
| 167 | int idx; | ||
| 168 | unsigned long flags; | ||
| 169 | |||
| 170 | /* | ||
| 171 | * Handle debugger faulting in for debugee. | ||
| 172 | */ | ||
| 173 | if (current->active_mm != vma->vm_mm) | ||
| 174 | return; | ||
| 175 | |||
| 176 | if (pte_val(*ptep) & _PAGE_HUGE) { | ||
| 177 | __update_hugetlb(vma, address, ptep); | ||
| 178 | return; | ||
| 179 | } | ||
| 180 | |||
| 181 | local_irq_save(flags); | ||
| 182 | |||
| 183 | if ((unsigned long)ptep & sizeof(pte_t)) | ||
| 184 | ptep--; | ||
| 185 | |||
| 186 | address &= (PAGE_MASK << 1); | ||
| 187 | write_csr_entryhi(address); | ||
| 188 | tlb_probe(); | ||
| 189 | idx = read_csr_tlbidx(); | ||
| 190 | write_csr_pagesize(PS_DEFAULT_SIZE); | ||
| 191 | write_csr_entrylo0(pte_val(*ptep++)); | ||
| 192 | write_csr_entrylo1(pte_val(*ptep)); | ||
| 193 | if (idx < 0) | ||
| 194 | tlb_write_random(); | ||
| 195 | else | ||
| 196 | tlb_write_indexed(); | ||
| 197 | |||
| 198 | local_irq_restore(flags); | ||
| 199 | } | ||
| 200 | |||
| 201 | static void setup_ptwalker(void) | ||
| 202 | { | ||
| 203 | unsigned long pwctl0, pwctl1; | ||
| 204 | unsigned long pgd_i = 0, pgd_w = 0; | ||
| 205 | unsigned long pud_i = 0, pud_w = 0; | ||
| 206 | unsigned long pmd_i = 0, pmd_w = 0; | ||
| 207 | unsigned long pte_i = 0, pte_w = 0; | ||
| 208 | |||
| 209 | pgd_i = PGDIR_SHIFT; | ||
| 210 | pgd_w = PAGE_SHIFT - 3; | ||
| 211 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 212 | pud_i = PUD_SHIFT; | ||
| 213 | pud_w = PAGE_SHIFT - 3; | ||
| 214 | #endif | ||
| 215 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 216 | pmd_i = PMD_SHIFT; | ||
| 217 | pmd_w = PAGE_SHIFT - 3; | ||
| 218 | #endif | ||
| 219 | pte_i = PAGE_SHIFT; | ||
| 220 | pte_w = PAGE_SHIFT - 3; | ||
| 221 | |||
| 222 | pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25; | ||
| 223 | pwctl1 = pgd_i | pgd_w << 6; | ||
| 224 | |||
| 225 | csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0); | ||
| 226 | csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1); | ||
| 227 | csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH); | ||
| 228 | csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL); | ||
| 229 | csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID); | ||
| 230 | } | ||
| 231 | |||
| 232 | static void output_pgtable_bits_defines(void) | ||
| 233 | { | ||
| 234 | #define pr_define(fmt, ...) \ | ||
| 235 | pr_debug("#define " fmt, ##__VA_ARGS__) | ||
| 236 | |||
| 237 | pr_debug("#include <asm/asm.h>\n"); | ||
| 238 | pr_debug("#include <asm/regdef.h>\n"); | ||
| 239 | pr_debug("\n"); | ||
| 240 | |||
| 241 | pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); | ||
| 242 | pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); | ||
| 243 | pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); | ||
| 244 | pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); | ||
| 245 | pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); | ||
| 246 | pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); | ||
| 247 | pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); | ||
| 248 | pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); | ||
| 249 | pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); | ||
| 250 | pr_debug("\n"); | ||
| 251 | } | ||
| 252 | |||
| 253 | void setup_tlb_handler(void) | ||
| 254 | { | ||
| 255 | static int run_once = 0; | ||
| 256 | |||
| 257 | setup_ptwalker(); | ||
| 258 | output_pgtable_bits_defines(); | ||
| 259 | |||
| 260 | /* The tlb handlers are generated only once */ | ||
| 261 | if (!run_once) { | ||
| 262 | memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); | ||
| 263 | local_flush_icache_range(tlbrentry, tlbrentry + 0x80); | ||
| 264 | set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); | ||
| 265 | set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); | ||
| 266 | set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); | ||
| 267 | set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); | ||
| 268 | set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); | ||
| 269 | set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); | ||
| 270 | set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); | ||
| 271 | run_once++; | ||
| 272 | } | ||
| 273 | } | ||
| 274 | |||
| 275 | void tlb_init(void) | ||
| 276 | { | ||
| 277 | write_csr_pagesize(PS_DEFAULT_SIZE); | ||
| 278 | write_csr_stlbpgsize(PS_DEFAULT_SIZE); | ||
| 279 | write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE); | ||
| 280 | setup_tlb_handler(); | ||
| 281 | local_flush_tlb_all(); | ||
| 282 | } | ||
| @@ -0,0 +1,477 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | ||
| 4 | */ | ||
| 5 | #include <asm/asm.h> | ||
| 6 | #include <asm/export.h> | ||
| 7 | #include <asm/loongarch.h> | ||
| 8 | #include <asm/page.h> | ||
| 9 | #include <asm/pgtable.h> | ||
| 10 | #include <asm/regdef.h> | ||
| 11 | #include <asm/stackframe.h> | ||
| 12 | |||
| 13 | .macro tlb_do_page_fault, write | ||
| 14 | SYM_FUNC_START(tlb_do_page_fault_\write) | ||
| 15 | SAVE_ALL | ||
| 16 | csrrd a2, LOONGARCH_CSR_BADV | ||
| 17 | move a0, sp | ||
| 18 | REG_S a2, sp, PT_BVADDR | ||
| 19 | li.w a1, \write | ||
| 20 | la.abs t0, do_page_fault | ||
| 21 | jirl ra, t0, 0 | ||
| 22 | RESTORE_ALL_AND_RET | ||
| 23 | SYM_FUNC_END(tlb_do_page_fault_\write) | ||
| 24 | .endm | ||
| 25 | |||
| 26 | tlb_do_page_fault 0 | ||
| 27 | tlb_do_page_fault 1 | ||
| 28 | |||
| 29 | SYM_FUNC_START(handle_tlb_protect) | ||
| 30 | BACKUP_T0T1 | ||
| 31 | SAVE_ALL | ||
| 32 | move a0, sp | ||
| 33 | move a1, zero | ||
| 34 | csrrd a2, LOONGARCH_CSR_BADV | ||
| 35 | REG_S a2, sp, PT_BVADDR | ||
| 36 | la.abs t0, do_page_fault | ||
| 37 | jirl ra, t0, 0 | ||
| 38 | RESTORE_ALL_AND_RET | ||
| 39 | SYM_FUNC_END(handle_tlb_protect) | ||
| 40 | |||
| 41 | SYM_FUNC_START(handle_tlb_load) | ||
| 42 | csrwr t0, EXCEPTION_KS0 | ||
| 43 | csrwr t1, EXCEPTION_KS1 | ||
| 44 | csrwr ra, EXCEPTION_KS2 | ||
| 45 | |||
| 46 | /* | ||
| 47 | * The vmalloc handling is not in the hotpath. | ||
| 48 | */ | ||
| 49 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 50 | blt t0, $r0, vmalloc_load | ||
| 51 | csrrd t1, LOONGARCH_CSR_PGDL | ||
| 52 | |||
| 53 | vmalloc_done_load: | ||
| 54 | /* Get PGD offset in bytes */ | ||
| 55 | srli.d t0, t0, PGDIR_SHIFT | ||
| 56 | andi t0, t0, (PTRS_PER_PGD - 1) | ||
| 57 | slli.d t0, t0, 3 | ||
| 58 | add.d t1, t1, t0 | ||
| 59 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 60 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 61 | ld.d t1, t1, 0 | ||
| 62 | srli.d t0, t0, PUD_SHIFT | ||
| 63 | andi t0, t0, (PTRS_PER_PUD - 1) | ||
| 64 | slli.d t0, t0, 3 | ||
| 65 | add.d t1, t1, t0 | ||
| 66 | #endif | ||
| 67 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 68 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 69 | ld.d t1, t1, 0 | ||
| 70 | srli.d t0, t0, PMD_SHIFT | ||
| 71 | andi t0, t0, (PTRS_PER_PMD - 1) | ||
| 72 | slli.d t0, t0, 3 | ||
| 73 | add.d t1, t1, t0 | ||
| 74 | #endif | ||
| 75 | ld.d ra, t1, 0 | ||
| 76 | |||
| 77 | /* | ||
| 78 | * For huge tlb entries, pmde doesn't contain an address but | ||
| 79 | * instead contains the tlb pte. Check the PAGE_HUGE bit and | ||
| 80 | * see if we need to jump to huge tlb processing. | ||
| 81 | */ | ||
| 82 | andi t0, ra, _PAGE_HUGE | ||
| 83 | bne t0, $r0, tlb_huge_update_load | ||
| 84 | |||
| 85 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 86 | srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | ||
| 87 | andi t0, t0, (PTRS_PER_PTE - 1) | ||
| 88 | slli.d t0, t0, _PTE_T_LOG2 | ||
| 89 | add.d t1, ra, t0 | ||
| 90 | |||
| 91 | ld.d t0, t1, 0 | ||
| 92 | tlbsrch | ||
| 93 | |||
| 94 | srli.d ra, t0, _PAGE_PRESENT_SHIFT | ||
| 95 | andi ra, ra, 1 | ||
| 96 | beq ra, $r0, nopage_tlb_load | ||
| 97 | |||
| 98 | ori t0, t0, _PAGE_VALID | ||
| 99 | st.d t0, t1, 0 | ||
| 100 | ori t1, t1, 8 | ||
| 101 | xori t1, t1, 8 | ||
| 102 | ld.d t0, t1, 0 | ||
| 103 | ld.d t1, t1, 8 | ||
| 104 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 105 | csrwr t1, LOONGARCH_CSR_TLBELO1 | ||
| 106 | tlbwr | ||
| 107 | leave_load: | ||
| 108 | csrrd t0, EXCEPTION_KS0 | ||
| 109 | csrrd t1, EXCEPTION_KS1 | ||
| 110 | csrrd ra, EXCEPTION_KS2 | ||
| 111 | ertn | ||
| 112 | #ifdef CONFIG_64BIT | ||
| 113 | vmalloc_load: | ||
| 114 | la.abs t1, swapper_pg_dir | ||
| 115 | b vmalloc_done_load | ||
| 116 | #endif | ||
| 117 | |||
| 118 | /* | ||
| 119 | * This is the entry point when build_tlbchange_handler_head | ||
| 120 | * spots a huge page. | ||
| 121 | */ | ||
| 122 | tlb_huge_update_load: | ||
| 123 | ld.d t0, t1, 0 | ||
| 124 | srli.d ra, t0, _PAGE_PRESENT_SHIFT | ||
| 125 | andi ra, ra, 1 | ||
| 126 | beq ra, $r0, nopage_tlb_load | ||
| 127 | tlbsrch | ||
| 128 | |||
| 129 | ori t0, t0, _PAGE_VALID | ||
| 130 | st.d t0, t1, 0 | ||
| 131 | addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) | ||
| 132 | addi.d ra, t1, 0 | ||
| 133 | csrxchg ra, t1, LOONGARCH_CSR_TLBIDX | ||
| 134 | tlbwr | ||
| 135 | |||
| 136 | csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX | ||
| 137 | |||
| 138 | /* | ||
| 139 | * A huge PTE describes an area the size of the | ||
| 140 | * configured huge page size. This is twice the | ||
| 141 | * of the large TLB entry size we intend to use. | ||
| 142 | * A TLB entry half the size of the configured | ||
| 143 | * huge page size is configured into entrylo0 | ||
| 144 | * and entrylo1 to cover the contiguous huge PTE | ||
| 145 | * address space. | ||
| 146 | */ | ||
| 147 | /* Huge page: Move Global bit */ | ||
| 148 | xori t0, t0, _PAGE_HUGE | ||
| 149 | lu12i.w t1, _PAGE_HGLOBAL >> 12 | ||
| 150 | and t1, t0, t1 | ||
| 151 | srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | ||
| 152 | or t0, t0, t1 | ||
| 153 | |||
| 154 | addi.d ra, t0, 0 | ||
| 155 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 156 | addi.d t0, ra, 0 | ||
| 157 | |||
| 158 | /* Convert to entrylo1 */ | ||
| 159 | addi.d t1, $r0, 1 | ||
| 160 | slli.d t1, t1, (HPAGE_SHIFT - 1) | ||
| 161 | add.d t0, t0, t1 | ||
| 162 | csrwr t0, LOONGARCH_CSR_TLBELO1 | ||
| 163 | |||
| 164 | /* Set huge page tlb entry size */ | ||
| 165 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 166 | addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 167 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 168 | |||
| 169 | tlbfill | ||
| 170 | |||
| 171 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 172 | addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 173 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 174 | |||
| 175 | nopage_tlb_load: | ||
| 176 | csrrd ra, EXCEPTION_KS2 | ||
| 177 | la.abs t0, tlb_do_page_fault_0 | ||
| 178 | jirl $r0, t0, 0 | ||
| 179 | SYM_FUNC_END(handle_tlb_load) | ||
| 180 | |||
| 181 | SYM_FUNC_START(handle_tlb_store) | ||
| 182 | csrwr t0, EXCEPTION_KS0 | ||
| 183 | csrwr t1, EXCEPTION_KS1 | ||
| 184 | csrwr ra, EXCEPTION_KS2 | ||
| 185 | |||
| 186 | /* | ||
| 187 | * The vmalloc handling is not in the hotpath. | ||
| 188 | */ | ||
| 189 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 190 | blt t0, $r0, vmalloc_store | ||
| 191 | csrrd t1, LOONGARCH_CSR_PGDL | ||
| 192 | |||
| 193 | vmalloc_done_store: | ||
| 194 | /* Get PGD offset in bytes */ | ||
| 195 | srli.d t0, t0, PGDIR_SHIFT | ||
| 196 | andi t0, t0, (PTRS_PER_PGD - 1) | ||
| 197 | slli.d t0, t0, 3 | ||
| 198 | add.d t1, t1, t0 | ||
| 199 | |||
| 200 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 201 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 202 | ld.d t1, t1, 0 | ||
| 203 | srli.d t0, t0, PUD_SHIFT | ||
| 204 | andi t0, t0, (PTRS_PER_PUD - 1) | ||
| 205 | slli.d t0, t0, 3 | ||
| 206 | add.d t1, t1, t0 | ||
| 207 | #endif | ||
| 208 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 209 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 210 | ld.d t1, t1, 0 | ||
| 211 | srli.d t0, t0, PMD_SHIFT | ||
| 212 | andi t0, t0, (PTRS_PER_PMD - 1) | ||
| 213 | slli.d t0, t0, 3 | ||
| 214 | add.d t1, t1, t0 | ||
| 215 | #endif | ||
| 216 | ld.d ra, t1, 0 | ||
| 217 | |||
| 218 | /* | ||
| 219 | * For huge tlb entries, pmde doesn't contain an address but | ||
| 220 | * instead contains the tlb pte. Check the PAGE_HUGE bit and | ||
| 221 | * see if we need to jump to huge tlb processing. | ||
| 222 | */ | ||
| 223 | andi t0, ra, _PAGE_HUGE | ||
| 224 | bne t0, $r0, tlb_huge_update_store | ||
| 225 | |||
| 226 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 227 | srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | ||
| 228 | andi t0, t0, (PTRS_PER_PTE - 1) | ||
| 229 | slli.d t0, t0, _PTE_T_LOG2 | ||
| 230 | add.d t1, ra, t0 | ||
| 231 | |||
| 232 | ld.d t0, t1, 0 | ||
| 233 | tlbsrch | ||
| 234 | |||
| 235 | srli.d ra, t0, _PAGE_PRESENT_SHIFT | ||
| 236 | andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | ||
| 237 | xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | ||
| 238 | bne ra, $r0, nopage_tlb_store | ||
| 239 | |||
| 240 | ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | ||
| 241 | st.d t0, t1, 0 | ||
| 242 | |||
| 243 | ori t1, t1, 8 | ||
| 244 | xori t1, t1, 8 | ||
| 245 | ld.d t0, t1, 0 | ||
| 246 | ld.d t1, t1, 8 | ||
| 247 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 248 | csrwr t1, LOONGARCH_CSR_TLBELO1 | ||
| 249 | tlbwr | ||
| 250 | leave_store: | ||
| 251 | csrrd t0, EXCEPTION_KS0 | ||
| 252 | csrrd t1, EXCEPTION_KS1 | ||
| 253 | csrrd ra, EXCEPTION_KS2 | ||
| 254 | ertn | ||
| 255 | #ifdef CONFIG_64BIT | ||
| 256 | vmalloc_store: | ||
| 257 | la.abs t1, swapper_pg_dir | ||
| 258 | b vmalloc_done_store | ||
| 259 | #endif | ||
| 260 | |||
| 261 | /* | ||
| 262 | * This is the entry point when build_tlbchange_handler_head | ||
| 263 | * spots a huge page. | ||
| 264 | */ | ||
| 265 | tlb_huge_update_store: | ||
| 266 | ld.d t0, t1, 0 | ||
| 267 | srli.d ra, t0, _PAGE_PRESENT_SHIFT | ||
| 268 | andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | ||
| 269 | xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | ||
| 270 | bne ra, $r0, nopage_tlb_store | ||
| 271 | |||
| 272 | tlbsrch | ||
| 273 | ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | ||
| 274 | |||
| 275 | st.d t0, t1, 0 | ||
| 276 | addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) | ||
| 277 | addi.d ra, t1, 0 | ||
| 278 | csrxchg ra, t1, LOONGARCH_CSR_TLBIDX | ||
| 279 | tlbwr | ||
| 280 | |||
| 281 | csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX | ||
| 282 | /* | ||
| 283 | * A huge PTE describes an area the size of the | ||
| 284 | * configured huge page size. This is twice the | ||
| 285 | * of the large TLB entry size we intend to use. | ||
| 286 | * A TLB entry half the size of the configured | ||
| 287 | * huge page size is configured into entrylo0 | ||
| 288 | * and entrylo1 to cover the contiguous huge PTE | ||
| 289 | * address space. | ||
| 290 | */ | ||
| 291 | /* Huge page: Move Global bit */ | ||
| 292 | xori t0, t0, _PAGE_HUGE | ||
| 293 | lu12i.w t1, _PAGE_HGLOBAL >> 12 | ||
| 294 | and t1, t0, t1 | ||
| 295 | srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | ||
| 296 | or t0, t0, t1 | ||
| 297 | |||
| 298 | addi.d ra, t0, 0 | ||
| 299 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 300 | addi.d t0, ra, 0 | ||
| 301 | |||
| 302 | /* Convert to entrylo1 */ | ||
| 303 | addi.d t1, $r0, 1 | ||
| 304 | slli.d t1, t1, (HPAGE_SHIFT - 1) | ||
| 305 | add.d t0, t0, t1 | ||
| 306 | csrwr t0, LOONGARCH_CSR_TLBELO1 | ||
| 307 | |||
| 308 | /* Set huge page tlb entry size */ | ||
| 309 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 310 | addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 311 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 312 | |||
| 313 | tlbfill | ||
| 314 | |||
| 315 | /* Reset default page size */ | ||
| 316 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 317 | addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 318 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 319 | |||
| 320 | nopage_tlb_store: | ||
| 321 | csrrd ra, EXCEPTION_KS2 | ||
| 322 | la.abs t0, tlb_do_page_fault_1 | ||
| 323 | jirl $r0, t0, 0 | ||
| 324 | SYM_FUNC_END(handle_tlb_store) | ||
| 325 | |||
| 326 | SYM_FUNC_START(handle_tlb_modify) | ||
| 327 | csrwr t0, EXCEPTION_KS0 | ||
| 328 | csrwr t1, EXCEPTION_KS1 | ||
| 329 | csrwr ra, EXCEPTION_KS2 | ||
| 330 | |||
| 331 | /* | ||
| 332 | * The vmalloc handling is not in the hotpath. | ||
| 333 | */ | ||
| 334 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 335 | blt t0, $r0, vmalloc_modify | ||
| 336 | csrrd t1, LOONGARCH_CSR_PGDL | ||
| 337 | |||
| 338 | vmalloc_done_modify: | ||
| 339 | /* Get PGD offset in bytes */ | ||
| 340 | srli.d t0, t0, PGDIR_SHIFT | ||
| 341 | andi t0, t0, (PTRS_PER_PGD - 1) | ||
| 342 | slli.d t0, t0, 3 | ||
| 343 | add.d t1, t1, t0 | ||
| 344 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 345 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 346 | ld.d t1, t1, 0 | ||
| 347 | srli.d t0, t0, PUD_SHIFT | ||
| 348 | andi t0, t0, (PTRS_PER_PUD - 1) | ||
| 349 | slli.d t0, t0, 3 | ||
| 350 | add.d t1, t1, t0 | ||
| 351 | #endif | ||
| 352 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 353 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 354 | ld.d t1, t1, 0 | ||
| 355 | srli.d t0, t0, PMD_SHIFT | ||
| 356 | andi t0, t0, (PTRS_PER_PMD - 1) | ||
| 357 | slli.d t0, t0, 3 | ||
| 358 | add.d t1, t1, t0 | ||
| 359 | #endif | ||
| 360 | ld.d ra, t1, 0 | ||
| 361 | |||
| 362 | /* | ||
| 363 | * For huge tlb entries, pmde doesn't contain an address but | ||
| 364 | * instead contains the tlb pte. Check the PAGE_HUGE bit and | ||
| 365 | * see if we need to jump to huge tlb processing. | ||
| 366 | */ | ||
| 367 | andi t0, ra, _PAGE_HUGE | ||
| 368 | bne t0, $r0, tlb_huge_update_modify | ||
| 369 | |||
| 370 | csrrd t0, LOONGARCH_CSR_BADV | ||
| 371 | srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | ||
| 372 | andi t0, t0, (PTRS_PER_PTE - 1) | ||
| 373 | slli.d t0, t0, _PTE_T_LOG2 | ||
| 374 | add.d t1, ra, t0 | ||
| 375 | |||
| 376 | ld.d t0, t1, 0 | ||
| 377 | tlbsrch | ||
| 378 | |||
| 379 | srli.d ra, t0, _PAGE_WRITE_SHIFT | ||
| 380 | andi ra, ra, 1 | ||
| 381 | beq ra, $r0, nopage_tlb_modify | ||
| 382 | |||
| 383 | ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | ||
| 384 | st.d t0, t1, 0 | ||
| 385 | ori t1, t1, 8 | ||
| 386 | xori t1, t1, 8 | ||
| 387 | ld.d t0, t1, 0 | ||
| 388 | ld.d t1, t1, 8 | ||
| 389 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 390 | csrwr t1, LOONGARCH_CSR_TLBELO1 | ||
| 391 | tlbwr | ||
| 392 | leave_modify: | ||
| 393 | csrrd t0, EXCEPTION_KS0 | ||
| 394 | csrrd t1, EXCEPTION_KS1 | ||
| 395 | csrrd ra, EXCEPTION_KS2 | ||
| 396 | ertn | ||
| 397 | #ifdef CONFIG_64BIT | ||
| 398 | vmalloc_modify: | ||
| 399 | la.abs t1, swapper_pg_dir | ||
| 400 | b vmalloc_done_modify | ||
| 401 | #endif | ||
| 402 | |||
| 403 | /* | ||
| 404 | * This is the entry point when | ||
| 405 | * build_tlbchange_handler_head spots a huge page. | ||
| 406 | */ | ||
| 407 | tlb_huge_update_modify: | ||
| 408 | ld.d t0, t1, 0 | ||
| 409 | |||
| 410 | srli.d ra, t0, _PAGE_WRITE_SHIFT | ||
| 411 | andi ra, ra, 1 | ||
| 412 | beq ra, $r0, nopage_tlb_modify | ||
| 413 | |||
| 414 | tlbsrch | ||
| 415 | ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | ||
| 416 | |||
| 417 | st.d t0, t1, 0 | ||
| 418 | /* | ||
| 419 | * A huge PTE describes an area the size of the | ||
| 420 | * configured huge page size. This is twice the | ||
| 421 | * of the large TLB entry size we intend to use. | ||
| 422 | * A TLB entry half the size of the configured | ||
| 423 | * huge page size is configured into entrylo0 | ||
| 424 | * and entrylo1 to cover the contiguous huge PTE | ||
| 425 | * address space. | ||
| 426 | */ | ||
| 427 | /* Huge page: Move Global bit */ | ||
| 428 | xori t0, t0, _PAGE_HUGE | ||
| 429 | lu12i.w t1, _PAGE_HGLOBAL >> 12 | ||
| 430 | and t1, t0, t1 | ||
| 431 | srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | ||
| 432 | or t0, t0, t1 | ||
| 433 | |||
| 434 | addi.d ra, t0, 0 | ||
| 435 | csrwr t0, LOONGARCH_CSR_TLBELO0 | ||
| 436 | addi.d t0, ra, 0 | ||
| 437 | |||
| 438 | /* Convert to entrylo1 */ | ||
| 439 | addi.d t1, $r0, 1 | ||
| 440 | slli.d t1, t1, (HPAGE_SHIFT - 1) | ||
| 441 | add.d t0, t0, t1 | ||
| 442 | csrwr t0, LOONGARCH_CSR_TLBELO1 | ||
| 443 | |||
| 444 | /* Set huge page tlb entry size */ | ||
| 445 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 446 | addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 447 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 448 | |||
| 449 | tlbwr | ||
| 450 | |||
| 451 | /* Reset default page size */ | ||
| 452 | addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | ||
| 453 | addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | ||
| 454 | csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | ||
| 455 | |||
| 456 | nopage_tlb_modify: | ||
| 457 | csrrd ra, EXCEPTION_KS2 | ||
| 458 | la.abs t0, tlb_do_page_fault_1 | ||
| 459 | jirl $r0, t0, 0 | ||
| 460 | SYM_FUNC_END(handle_tlb_modify) | ||
| 461 | |||
| 462 | SYM_FUNC_START(handle_tlb_refill) | ||
| 463 | csrwr t0, LOONGARCH_CSR_TLBRSAVE | ||
| 464 | csrrd t0, LOONGARCH_CSR_PGD | ||
| 465 | lddir t0, t0, 3 | ||
| 466 | #if CONFIG_PGTABLE_LEVELS > 3 | ||
| 467 | lddir t0, t0, 2 | ||
| 468 | #endif | ||
| 469 | #if CONFIG_PGTABLE_LEVELS > 2 | ||
| 470 | lddir t0, t0, 1 | ||
| 471 | #endif | ||
| 472 | ldpte t0, 0 | ||
| 473 | ldpte t0, 1 | ||
| 474 | tlbfill | ||
| 475 | csrrd t0, LOONGARCH_CSR_TLBRSAVE | ||
| 476 | ertn | ||
| 477 | SYM_FUNC_END(handle_tlb_refill) | ||