Add memory management support for LoongArch, including: cache and tlb management, page fault handling and ioremap/mmap support.
Unified · Split
| @@ -0,0 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_CACHE_H | |
| 6 | +#define _ASM_CACHE_H | |
| 7 | + | |
| 8 | +#define L1_CACHE_SHIFT CONFIG_L1_CACHE_SHIFT | |
| 9 | +#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) | |
| 10 | + | |
| 11 | +#define __read_mostly __section(".data..read_mostly") | |
| 12 | + | |
| 13 | +#endif /* _ASM_CACHE_H */ | |
| @@ -0,0 +1,80 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_CACHEFLUSH_H | |
| 6 | +#define _ASM_CACHEFLUSH_H | |
| 7 | + | |
| 8 | +#include <linux/mm.h> | |
| 9 | +#include <asm/cpu-features.h> | |
| 10 | +#include <asm/cacheops.h> | |
| 11 | + | |
| 12 | +extern void local_flush_icache_range(unsigned long start, unsigned long end); | |
| 13 | + | |
| 14 | +#define flush_icache_range local_flush_icache_range | |
| 15 | +#define flush_icache_user_range local_flush_icache_range | |
| 16 | + | |
| 17 | +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 | |
| 18 | + | |
| 19 | +#define flush_cache_all() do { } while (0) | |
| 20 | +#define flush_cache_mm(mm) do { } while (0) | |
| 21 | +#define flush_cache_dup_mm(mm) do { } while (0) | |
| 22 | +#define flush_cache_range(vma, start, end) do { } while (0) | |
| 23 | +#define flush_cache_page(vma, vmaddr, pfn) do { } while (0) | |
| 24 | +#define flush_cache_vmap(start, end) do { } while (0) | |
| 25 | +#define flush_cache_vunmap(start, end) do { } while (0) | |
| 26 | +#define flush_icache_page(vma, page) do { } while (0) | |
| 27 | +#define flush_icache_user_page(vma, page, addr, len) do { } while (0) | |
| 28 | +#define flush_dcache_page(page) do { } while (0) | |
| 29 | +#define flush_dcache_mmap_lock(mapping) do { } while (0) | |
| 30 | +#define flush_dcache_mmap_unlock(mapping) do { } while (0) | |
| 31 | + | |
| 32 | +#define cache_op(op, addr) \ | |
| 33 | + __asm__ __volatile__( \ | |
| 34 | + " cacop %0, %1 \n" \ | |
| 35 | + : \ | |
| 36 | + : "i" (op), "ZC" (*(unsigned char *)(addr))) | |
| 37 | + | |
| 38 | +static inline void flush_icache_line_indexed(unsigned long addr) | |
| 39 | +{ | |
| 40 | + cache_op(Index_Invalidate_I, addr); | |
| 41 | +} | |
| 42 | + | |
| 43 | +static inline void flush_dcache_line_indexed(unsigned long addr) | |
| 44 | +{ | |
| 45 | + cache_op(Index_Writeback_Inv_D, addr); | |
| 46 | +} | |
| 47 | + | |
| 48 | +static inline void flush_vcache_line_indexed(unsigned long addr) | |
| 49 | +{ | |
| 50 | + cache_op(Index_Writeback_Inv_V, addr); | |
| 51 | +} | |
| 52 | + | |
| 53 | +static inline void flush_scache_line_indexed(unsigned long addr) | |
| 54 | +{ | |
| 55 | + cache_op(Index_Writeback_Inv_S, addr); | |
| 56 | +} | |
| 57 | + | |
| 58 | +static inline void flush_icache_line(unsigned long addr) | |
| 59 | +{ | |
| 60 | + cache_op(Hit_Invalidate_I, addr); | |
| 61 | +} | |
| 62 | + | |
| 63 | +static inline void flush_dcache_line(unsigned long addr) | |
| 64 | +{ | |
| 65 | + cache_op(Hit_Writeback_Inv_D, addr); | |
| 66 | +} | |
| 67 | + | |
| 68 | +static inline void flush_vcache_line(unsigned long addr) | |
| 69 | +{ | |
| 70 | + cache_op(Hit_Writeback_Inv_V, addr); | |
| 71 | +} | |
| 72 | + | |
| 73 | +static inline void flush_scache_line(unsigned long addr) | |
| 74 | +{ | |
| 75 | + cache_op(Hit_Writeback_Inv_S, addr); | |
| 76 | +} | |
| 77 | + | |
| 78 | +#include <asm-generic/cacheflush.h> | |
| 79 | + | |
| 80 | +#endif /* _ASM_CACHEFLUSH_H */ | |
| @@ -0,0 +1,37 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Cache operations for the cache instruction. | |
| 4 | + * | |
| 5 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 6 | + */ | |
| 7 | +#ifndef __ASM_CACHEOPS_H | |
| 8 | +#define __ASM_CACHEOPS_H | |
| 9 | + | |
| 10 | +/* | |
| 11 | + * Most cache ops are split into a 2 bit field identifying the cache, and a 3 | |
| 12 | + * bit field identifying the cache operation. | |
| 13 | + */ | |
| 14 | +#define CacheOp_Cache 0x03 | |
| 15 | +#define CacheOp_Op 0x1c | |
| 16 | + | |
| 17 | +#define Cache_I 0x00 | |
| 18 | +#define Cache_D 0x01 | |
| 19 | +#define Cache_V 0x02 | |
| 20 | +#define Cache_S 0x03 | |
| 21 | + | |
| 22 | +#define Index_Invalidate 0x08 | |
| 23 | +#define Index_Writeback_Inv 0x08 | |
| 24 | +#define Hit_Invalidate 0x10 | |
| 25 | +#define Hit_Writeback_Inv 0x10 | |
| 26 | +#define CacheOp_User_Defined 0x18 | |
| 27 | + | |
| 28 | +#define Index_Invalidate_I (Cache_I | Index_Invalidate) | |
| 29 | +#define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv) | |
| 30 | +#define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv) | |
| 31 | +#define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv) | |
| 32 | +#define Hit_Invalidate_I (Cache_I | Hit_Invalidate) | |
| 33 | +#define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv) | |
| 34 | +#define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv) | |
| 35 | +#define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv) | |
| 36 | + | |
| 37 | +#endif /* __ASM_CACHEOPS_H */ | |
| @@ -0,0 +1,13 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * fixmap.h: compile-time virtual memory allocation | |
| 4 | + * | |
| 5 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 6 | + */ | |
| 7 | + | |
| 8 | +#ifndef _ASM_FIXMAP_H | |
| 9 | +#define _ASM_FIXMAP_H | |
| 10 | + | |
| 11 | +#define NR_FIX_BTMAPS 64 | |
| 12 | + | |
| 13 | +#endif | |
| @@ -0,0 +1,83 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#ifndef __ASM_HUGETLB_H | |
| 7 | +#define __ASM_HUGETLB_H | |
| 8 | + | |
| 9 | +#include <asm/page.h> | |
| 10 | + | |
| 11 | +uint64_t pmd_to_entrylo(unsigned long pmd_val); | |
| 12 | + | |
| 13 | +#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE | |
| 14 | +static inline int prepare_hugepage_range(struct file *file, | |
| 15 | + unsigned long addr, | |
| 16 | + unsigned long len) | |
| 17 | +{ | |
| 18 | + unsigned long task_size = STACK_TOP; | |
| 19 | + struct hstate *h = hstate_file(file); | |
| 20 | + | |
| 21 | + if (len & ~huge_page_mask(h)) | |
| 22 | + return -EINVAL; | |
| 23 | + if (addr & ~huge_page_mask(h)) | |
| 24 | + return -EINVAL; | |
| 25 | + if (len > task_size) | |
| 26 | + return -ENOMEM; | |
| 27 | + if (task_size - len < addr) | |
| 28 | + return -EINVAL; | |
| 29 | + return 0; | |
| 30 | +} | |
| 31 | + | |
| 32 | +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR | |
| 33 | +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
| 34 | + unsigned long addr, pte_t *ptep) | |
| 35 | +{ | |
| 36 | + pte_t clear; | |
| 37 | + pte_t pte = *ptep; | |
| 38 | + | |
| 39 | + pte_val(clear) = (unsigned long)invalid_pte_table; | |
| 40 | + set_pte_at(mm, addr, ptep, clear); | |
| 41 | + return pte; | |
| 42 | +} | |
| 43 | + | |
| 44 | +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH | |
| 45 | +static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, | |
| 46 | + unsigned long addr, pte_t *ptep) | |
| 47 | +{ | |
| 48 | + pte_t pte; | |
| 49 | + | |
| 50 | + pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); | |
| 51 | + flush_tlb_page(vma, addr); | |
| 52 | + return pte; | |
| 53 | +} | |
| 54 | + | |
| 55 | +#define __HAVE_ARCH_HUGE_PTE_NONE | |
| 56 | +static inline int huge_pte_none(pte_t pte) | |
| 57 | +{ | |
| 58 | + unsigned long val = pte_val(pte) & ~_PAGE_GLOBAL; | |
| 59 | + return !val || (val == (unsigned long)invalid_pte_table); | |
| 60 | +} | |
| 61 | + | |
| 62 | +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS | |
| 63 | +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, | |
| 64 | + unsigned long addr, | |
| 65 | + pte_t *ptep, pte_t pte, | |
| 66 | + int dirty) | |
| 67 | +{ | |
| 68 | + int changed = !pte_same(*ptep, pte); | |
| 69 | + | |
| 70 | + if (changed) { | |
| 71 | + set_pte_at(vma->vm_mm, addr, ptep, pte); | |
| 72 | + /* | |
| 73 | + * There could be some standard sized pages in there, | |
| 74 | + * get them all. | |
| 75 | + */ | |
| 76 | + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); | |
| 77 | + } | |
| 78 | + return changed; | |
| 79 | +} | |
| 80 | + | |
| 81 | +#include <asm-generic/hugetlb.h> | |
| 82 | + | |
| 83 | +#endif /* __ASM_HUGETLB_H */ | |
| @@ -0,0 +1,115 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_PAGE_H | |
| 6 | +#define _ASM_PAGE_H | |
| 7 | + | |
| 8 | +#include <linux/const.h> | |
| 9 | + | |
| 10 | +/* | |
| 11 | + * PAGE_SHIFT determines the page size | |
| 12 | + */ | |
| 13 | +#ifdef CONFIG_PAGE_SIZE_4KB | |
| 14 | +#define PAGE_SHIFT 12 | |
| 15 | +#endif | |
| 16 | +#ifdef CONFIG_PAGE_SIZE_16KB | |
| 17 | +#define PAGE_SHIFT 14 | |
| 18 | +#endif | |
| 19 | +#ifdef CONFIG_PAGE_SIZE_64KB | |
| 20 | +#define PAGE_SHIFT 16 | |
| 21 | +#endif | |
| 22 | +#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) | |
| 23 | +#define PAGE_MASK (~(PAGE_SIZE - 1)) | |
| 24 | + | |
| 25 | +#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) | |
| 26 | +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) | |
| 27 | +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) | |
| 28 | +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) | |
| 29 | + | |
| 30 | +#ifndef __ASSEMBLY__ | |
| 31 | + | |
| 32 | +#include <linux/kernel.h> | |
| 33 | +#include <linux/pfn.h> | |
| 34 | + | |
| 35 | +#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) | |
| 36 | + | |
| 37 | +/* | |
| 38 | + * It's normally defined only for FLATMEM config but it's | |
| 39 | + * used in our early mem init code for all memory models. | |
| 40 | + * So always define it. | |
| 41 | + */ | |
| 42 | +#define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET) | |
| 43 | + | |
| 44 | +extern void clear_page(void *page); | |
| 45 | +extern void copy_page(void *to, void *from); | |
| 46 | + | |
| 47 | +#define clear_user_page(page, vaddr, pg) clear_page(page) | |
| 48 | +#define copy_user_page(to, from, vaddr, pg) copy_page(to, from) | |
| 49 | + | |
| 50 | +extern unsigned long shm_align_mask; | |
| 51 | + | |
| 52 | +struct page; | |
| 53 | +struct vm_area_struct; | |
| 54 | +void copy_user_highpage(struct page *to, struct page *from, | |
| 55 | + unsigned long vaddr, struct vm_area_struct *vma); | |
| 56 | + | |
| 57 | +#define __HAVE_ARCH_COPY_USER_HIGHPAGE | |
| 58 | + | |
| 59 | +typedef struct { unsigned long pte; } pte_t; | |
| 60 | +#define pte_val(x) ((x).pte) | |
| 61 | +#define __pte(x) ((pte_t) { (x) }) | |
| 62 | +typedef struct page *pgtable_t; | |
| 63 | + | |
| 64 | +typedef struct { unsigned long pgd; } pgd_t; | |
| 65 | +#define pgd_val(x) ((x).pgd) | |
| 66 | +#define __pgd(x) ((pgd_t) { (x) }) | |
| 67 | + | |
| 68 | +/* | |
| 69 | + * Manipulate page protection bits | |
| 70 | + */ | |
| 71 | +typedef struct { unsigned long pgprot; } pgprot_t; | |
| 72 | +#define pgprot_val(x) ((x).pgprot) | |
| 73 | +#define __pgprot(x) ((pgprot_t) { (x) }) | |
| 74 | +#define pte_pgprot(x) __pgprot(pte_val(x) & ~_PFN_MASK) | |
| 75 | + | |
| 76 | +#define ptep_buddy(x) ((pte_t *)((unsigned long)(x) ^ sizeof(pte_t))) | |
| 77 | + | |
| 78 | +/* | |
| 79 | + * __pa()/__va() should be used only during mem init. | |
| 80 | + */ | |
| 81 | +#define __pa(x) PHYSADDR(x) | |
| 82 | +#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET)) | |
| 83 | + | |
| 84 | +#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) | |
| 85 | + | |
| 86 | +#ifdef CONFIG_FLATMEM | |
| 87 | + | |
| 88 | +static inline int pfn_valid(unsigned long pfn) | |
| 89 | +{ | |
| 90 | + /* avoid <linux/mm.h> include hell */ | |
| 91 | + extern unsigned long max_mapnr; | |
| 92 | + unsigned long pfn_offset = ARCH_PFN_OFFSET; | |
| 93 | + | |
| 94 | + return pfn >= pfn_offset && pfn < max_mapnr; | |
| 95 | +} | |
| 96 | + | |
| 97 | +#endif | |
| 98 | + | |
| 99 | +#define virt_to_pfn(kaddr) PFN_DOWN(virt_to_phys((void *)(kaddr))) | |
| 100 | +#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) | |
| 101 | + | |
| 102 | +extern int __virt_addr_valid(volatile void *kaddr); | |
| 103 | +#define virt_addr_valid(kaddr) __virt_addr_valid((volatile void *)(kaddr)) | |
| 104 | + | |
| 105 | +#define VM_DATA_DEFAULT_FLAGS \ | |
| 106 | + (VM_READ | VM_WRITE | \ | |
| 107 | + ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \ | |
| 108 | + VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) | |
| 109 | + | |
| 110 | +#include <asm-generic/memory_model.h> | |
| 111 | +#include <asm-generic/getorder.h> | |
| 112 | + | |
| 113 | +#endif /* !__ASSEMBLY__ */ | |
| 114 | + | |
| 115 | +#endif /* _ASM_PAGE_H */ | |
| @@ -0,0 +1,103 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_PGALLOC_H | |
| 6 | +#define _ASM_PGALLOC_H | |
| 7 | + | |
| 8 | +#include <linux/mm.h> | |
| 9 | +#include <linux/sched.h> | |
| 10 | + | |
| 11 | +#define __HAVE_ARCH_PMD_ALLOC_ONE | |
| 12 | +#define __HAVE_ARCH_PUD_ALLOC_ONE | |
| 13 | +#include <asm-generic/pgalloc.h> | |
| 14 | + | |
| 15 | +static inline void pmd_populate_kernel(struct mm_struct *mm, | |
| 16 | + pmd_t *pmd, pte_t *pte) | |
| 17 | +{ | |
| 18 | + set_pmd(pmd, __pmd((unsigned long)pte)); | |
| 19 | +} | |
| 20 | + | |
| 21 | +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte) | |
| 22 | +{ | |
| 23 | + set_pmd(pmd, __pmd((unsigned long)page_address(pte))); | |
| 24 | +} | |
| 25 | + | |
| 26 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 27 | + | |
| 28 | +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) | |
| 29 | +{ | |
| 30 | + set_pud(pud, __pud((unsigned long)pmd)); | |
| 31 | +} | |
| 32 | +#endif | |
| 33 | + | |
| 34 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 35 | + | |
| 36 | +static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) | |
| 37 | +{ | |
| 38 | + set_p4d(p4d, __p4d((unsigned long)pud)); | |
| 39 | +} | |
| 40 | + | |
| 41 | +#endif /* __PAGETABLE_PUD_FOLDED */ | |
| 42 | + | |
| 43 | +extern void pagetable_init(void); | |
| 44 | + | |
| 45 | +/* | |
| 46 | + * Initialize a new pmd table with invalid pointers. | |
| 47 | + */ | |
| 48 | +extern void pmd_init(unsigned long page, unsigned long pagetable); | |
| 49 | + | |
| 50 | +/* | |
| 51 | + * Initialize a new pgd / pmd table with invalid pointers. | |
| 52 | + */ | |
| 53 | +extern void pgd_init(unsigned long page); | |
| 54 | +extern pgd_t *pgd_alloc(struct mm_struct *mm); | |
| 55 | + | |
| 56 | +#define __pte_free_tlb(tlb, pte, address) \ | |
| 57 | +do { \ | |
| 58 | + pgtable_pte_page_dtor(pte); \ | |
| 59 | + tlb_remove_page((tlb), pte); \ | |
| 60 | +} while (0) | |
| 61 | + | |
| 62 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 63 | + | |
| 64 | +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) | |
| 65 | +{ | |
| 66 | + pmd_t *pmd; | |
| 67 | + struct page *pg; | |
| 68 | + | |
| 69 | + pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_ORDER); | |
| 70 | + if (!pg) | |
| 71 | + return NULL; | |
| 72 | + | |
| 73 | + if (!pgtable_pmd_page_ctor(pg)) { | |
| 74 | + __free_pages(pg, PMD_ORDER); | |
| 75 | + return NULL; | |
| 76 | + } | |
| 77 | + | |
| 78 | + pmd = (pmd_t *)page_address(pg); | |
| 79 | + pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table); | |
| 80 | + return pmd; | |
| 81 | +} | |
| 82 | + | |
| 83 | +#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) | |
| 84 | + | |
| 85 | +#endif | |
| 86 | + | |
| 87 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 88 | + | |
| 89 | +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address) | |
| 90 | +{ | |
| 91 | + pud_t *pud; | |
| 92 | + | |
| 93 | + pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER); | |
| 94 | + if (pud) | |
| 95 | + pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table); | |
| 96 | + return pud; | |
| 97 | +} | |
| 98 | + | |
| 99 | +#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x) | |
| 100 | + | |
| 101 | +#endif /* __PAGETABLE_PUD_FOLDED */ | |
| 102 | + | |
| 103 | +#endif /* _ASM_PGALLOC_H */ | |
| @@ -0,0 +1,131 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_PGTABLE_BITS_H | |
| 6 | +#define _ASM_PGTABLE_BITS_H | |
| 7 | + | |
| 8 | +/* Page table bits */ | |
| 9 | +#define _PAGE_VALID_SHIFT 0 | |
| 10 | +#define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ | |
| 11 | +#define _PAGE_DIRTY_SHIFT 1 | |
| 12 | +#define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */ | |
| 13 | +#define _CACHE_SHIFT 4 /* 4~5, two bits */ | |
| 14 | +#define _PAGE_GLOBAL_SHIFT 6 | |
| 15 | +#define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */ | |
| 16 | +#define _PAGE_PRESENT_SHIFT 7 | |
| 17 | +#define _PAGE_WRITE_SHIFT 8 | |
| 18 | +#define _PAGE_MODIFIED_SHIFT 9 | |
| 19 | +#define _PAGE_PROTNONE_SHIFT 10 | |
| 20 | +#define _PAGE_SPECIAL_SHIFT 11 | |
| 21 | +#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ | |
| 22 | +#define _PAGE_PFN_SHIFT 12 | |
| 23 | +#define _PAGE_PFN_END_SHIFT 48 | |
| 24 | +#define _PAGE_NO_READ_SHIFT 61 | |
| 25 | +#define _PAGE_NO_EXEC_SHIFT 62 | |
| 26 | +#define _PAGE_RPLV_SHIFT 63 | |
| 27 | + | |
| 28 | +/* Used by software */ | |
| 29 | +#define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT) | |
| 30 | +#define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT) | |
| 31 | +#define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT) | |
| 32 | +#define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT) | |
| 33 | +#define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT) | |
| 34 | +#define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT) | |
| 35 | + | |
| 36 | +/* Used by TLB hardware (placed in EntryLo*) */ | |
| 37 | +#define _PAGE_VALID (_ULCAST_(1) << _PAGE_VALID_SHIFT) | |
| 38 | +#define _PAGE_DIRTY (_ULCAST_(1) << _PAGE_DIRTY_SHIFT) | |
| 39 | +#define _PAGE_PLV (_ULCAST_(3) << _PAGE_PLV_SHIFT) | |
| 40 | +#define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT) | |
| 41 | +#define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT) | |
| 42 | +#define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT) | |
| 43 | +#define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT) | |
| 44 | +#define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT) | |
| 45 | +#define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT) | |
| 46 | +#define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT) | |
| 47 | +#define _PFN_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT) | |
| 48 | + | |
| 49 | +#define _PAGE_USER (PLV_USER << _PAGE_PLV_SHIFT) | |
| 50 | +#define _PAGE_KERN (PLV_KERN << _PAGE_PLV_SHIFT) | |
| 51 | + | |
| 52 | +#define _PFN_MASK (~((_ULCAST_(1) << (_PFN_SHIFT)) - 1) & \ | |
| 53 | + ((_ULCAST_(1) << (_PAGE_PFN_END_SHIFT)) - 1)) | |
| 54 | + | |
| 55 | +/* | |
| 56 | + * Cache attributes | |
| 57 | + */ | |
| 58 | +#ifndef _CACHE_SUC | |
| 59 | +#define _CACHE_SUC (0<<_CACHE_SHIFT) /* Strong-ordered UnCached */ | |
| 60 | +#endif | |
| 61 | +#ifndef _CACHE_CC | |
| 62 | +#define _CACHE_CC (1<<_CACHE_SHIFT) /* Coherent Cached */ | |
| 63 | +#endif | |
| 64 | +#ifndef _CACHE_WUC | |
| 65 | +#define _CACHE_WUC (2<<_CACHE_SHIFT) /* Weak-ordered UnCached */ | |
| 66 | +#endif | |
| 67 | + | |
| 68 | +#define __READABLE (_PAGE_VALID) | |
| 69 | +#define __WRITEABLE (_PAGE_DIRTY | _PAGE_WRITE) | |
| 70 | + | |
| 71 | +#define _PAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV) | |
| 72 | +#define _HPAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV | _PAGE_HUGE) | |
| 73 | + | |
| 74 | +#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_NO_READ | \ | |
| 75 | + _PAGE_USER | _CACHE_CC) | |
| 76 | +#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \ | |
| 77 | + _PAGE_USER | _CACHE_CC) | |
| 78 | +#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _CACHE_CC) | |
| 79 | + | |
| 80 | +#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | |
| 81 | + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_CC) | |
| 82 | +#define PAGE_KERNEL_SUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | |
| 83 | + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC) | |
| 84 | +#define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ | |
| 85 | + _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC) | |
| 86 | + | |
| 87 | +#define __P000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) | |
| 88 | +#define __P001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | |
| 89 | +#define __P010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | |
| 90 | +#define __P011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | |
| 91 | +#define __P100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 92 | +#define __P101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 93 | +#define __P110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 94 | +#define __P111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 95 | + | |
| 96 | +#define __S000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ) | |
| 97 | +#define __S001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC) | |
| 98 | +#define __S010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) | |
| 99 | +#define __S011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE) | |
| 100 | +#define __S100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 101 | +#define __S101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT) | |
| 102 | +#define __S110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) | |
| 103 | +#define __S111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE) | |
| 104 | + | |
| 105 | +#ifndef __ASSEMBLY__ | |
| 106 | + | |
| 107 | +#define pgprot_noncached pgprot_noncached | |
| 108 | + | |
| 109 | +static inline pgprot_t pgprot_noncached(pgprot_t _prot) | |
| 110 | +{ | |
| 111 | + unsigned long prot = pgprot_val(_prot); | |
| 112 | + | |
| 113 | + prot = (prot & ~_CACHE_MASK) | _CACHE_SUC; | |
| 114 | + | |
| 115 | + return __pgprot(prot); | |
| 116 | +} | |
| 117 | + | |
| 118 | +#define pgprot_writecombine pgprot_writecombine | |
| 119 | + | |
| 120 | +static inline pgprot_t pgprot_writecombine(pgprot_t _prot) | |
| 121 | +{ | |
| 122 | + unsigned long prot = pgprot_val(_prot); | |
| 123 | + | |
| 124 | + prot = (prot & ~_CACHE_MASK) | _CACHE_WUC; | |
| 125 | + | |
| 126 | + return __pgprot(prot); | |
| 127 | +} | |
| 128 | + | |
| 129 | +#endif /* !__ASSEMBLY__ */ | |
| 130 | + | |
| 131 | +#endif /* _ASM_PGTABLE_BITS_H */ | |
| @@ -0,0 +1,532 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + * | |
| 5 | + * Derived from MIPS: | |
| 6 | + * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle | |
| 7 | + * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc. | |
| 8 | + */ | |
| 9 | +#ifndef _ASM_PGTABLE_H | |
| 10 | +#define _ASM_PGTABLE_H | |
| 11 | + | |
| 12 | +#include <linux/compiler.h> | |
| 13 | +#include <asm/addrspace.h> | |
| 14 | +#include <asm/pgtable-bits.h> | |
| 15 | + | |
| 16 | +#if CONFIG_PGTABLE_LEVELS == 2 | |
| 17 | +#include <asm-generic/pgtable-nopmd.h> | |
| 18 | +#elif CONFIG_PGTABLE_LEVELS == 3 | |
| 19 | +#include <asm-generic/pgtable-nopud.h> | |
| 20 | +#else | |
| 21 | +#include <asm-generic/pgtable-nop4d.h> | |
| 22 | +#endif | |
| 23 | + | |
| 24 | +#define PGD_ORDER 0 | |
| 25 | +#define PUD_ORDER 0 | |
| 26 | +#define PMD_ORDER 0 | |
| 27 | +#define PTE_ORDER 0 | |
| 28 | + | |
| 29 | +#if CONFIG_PGTABLE_LEVELS == 2 | |
| 30 | +#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | |
| 31 | +#elif CONFIG_PGTABLE_LEVELS == 3 | |
| 32 | +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | |
| 33 | +#define PMD_SIZE (1UL << PMD_SHIFT) | |
| 34 | +#define PMD_MASK (~(PMD_SIZE-1)) | |
| 35 | +#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) | |
| 36 | +#elif CONFIG_PGTABLE_LEVELS == 4 | |
| 37 | +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3)) | |
| 38 | +#define PMD_SIZE (1UL << PMD_SHIFT) | |
| 39 | +#define PMD_MASK (~(PMD_SIZE-1)) | |
| 40 | +#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3)) | |
| 41 | +#define PUD_SIZE (1UL << PUD_SHIFT) | |
| 42 | +#define PUD_MASK (~(PUD_SIZE-1)) | |
| 43 | +#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3)) | |
| 44 | +#endif | |
| 45 | + | |
| 46 | +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) | |
| 47 | +#define PGDIR_MASK (~(PGDIR_SIZE-1)) | |
| 48 | + | |
| 49 | +#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT + PGD_ORDER - 3)) | |
| 50 | + | |
| 51 | +#define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) >> 3) | |
| 52 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 53 | +#define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) >> 3) | |
| 54 | +#endif | |
| 55 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 56 | +#define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) >> 3) | |
| 57 | +#endif | |
| 58 | +#define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) >> 3) | |
| 59 | + | |
| 60 | +#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) | |
| 61 | + | |
| 62 | +#ifndef __ASSEMBLY__ | |
| 63 | + | |
| 64 | +#include <linux/mm_types.h> | |
| 65 | +#include <linux/mmzone.h> | |
| 66 | +#include <asm/fixmap.h> | |
| 67 | +#include <asm/io.h> | |
| 68 | + | |
| 69 | +struct mm_struct; | |
| 70 | +struct vm_area_struct; | |
| 71 | + | |
| 72 | +/* | |
| 73 | + * ZERO_PAGE is a global shared page that is always zero; used | |
| 74 | + * for zero-mapped memory areas etc.. | |
| 75 | + */ | |
| 76 | + | |
| 77 | +extern unsigned long empty_zero_page; | |
| 78 | +extern unsigned long zero_page_mask; | |
| 79 | + | |
| 80 | +#define ZERO_PAGE(vaddr) \ | |
| 81 | + (virt_to_page((void *)(empty_zero_page + (((unsigned long)(vaddr)) & zero_page_mask)))) | |
| 82 | +#define __HAVE_COLOR_ZERO_PAGE | |
| 83 | + | |
| 84 | +/* | |
| 85 | + * TLB refill handlers may also map the vmalloc area into xkvrange. | |
| 86 | + * Avoid the first couple of pages so NULL pointer dereferences will | |
| 87 | + * still reliably trap. | |
| 88 | + */ | |
| 89 | +#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) | |
| 90 | +#define MODULES_END (MODULES_VADDR + SZ_256M) | |
| 91 | + | |
| 92 | +#define VMALLOC_START MODULES_END | |
| 93 | +#define VMALLOC_END \ | |
| 94 | + (vm_map_base + \ | |
| 95 | + min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE) | |
| 96 | + | |
| 97 | +#define pte_ERROR(e) \ | |
| 98 | + pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) | |
| 99 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 100 | +#define pmd_ERROR(e) \ | |
| 101 | + pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) | |
| 102 | +#endif | |
| 103 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 104 | +#define pud_ERROR(e) \ | |
| 105 | + pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) | |
| 106 | +#endif | |
| 107 | +#define pgd_ERROR(e) \ | |
| 108 | + pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) | |
| 109 | + | |
| 110 | +extern pte_t invalid_pte_table[PTRS_PER_PTE]; | |
| 111 | + | |
| 112 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 113 | + | |
| 114 | +typedef struct { unsigned long pud; } pud_t; | |
| 115 | +#define pud_val(x) ((x).pud) | |
| 116 | +#define __pud(x) ((pud_t) { (x) }) | |
| 117 | + | |
| 118 | +extern pud_t invalid_pud_table[PTRS_PER_PUD]; | |
| 119 | + | |
| 120 | +/* | |
| 121 | + * Empty pgd/p4d entries point to the invalid_pud_table. | |
| 122 | + */ | |
| 123 | +static inline int p4d_none(p4d_t p4d) | |
| 124 | +{ | |
| 125 | + return p4d_val(p4d) == (unsigned long)invalid_pud_table; | |
| 126 | +} | |
| 127 | + | |
| 128 | +static inline int p4d_bad(p4d_t p4d) | |
| 129 | +{ | |
| 130 | + return p4d_val(p4d) & ~PAGE_MASK; | |
| 131 | +} | |
| 132 | + | |
| 133 | +static inline int p4d_present(p4d_t p4d) | |
| 134 | +{ | |
| 135 | + return p4d_val(p4d) != (unsigned long)invalid_pud_table; | |
| 136 | +} | |
| 137 | + | |
| 138 | +static inline void p4d_clear(p4d_t *p4dp) | |
| 139 | +{ | |
| 140 | + p4d_val(*p4dp) = (unsigned long)invalid_pud_table; | |
| 141 | +} | |
| 142 | + | |
| 143 | +static inline pud_t *p4d_pgtable(p4d_t p4d) | |
| 144 | +{ | |
| 145 | + return (pud_t *)p4d_val(p4d); | |
| 146 | +} | |
| 147 | + | |
| 148 | +static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) | |
| 149 | +{ | |
| 150 | + *p4d = p4dval; | |
| 151 | +} | |
| 152 | + | |
| 153 | +#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) | |
| 154 | +#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) | |
| 155 | + | |
| 156 | +#endif | |
| 157 | + | |
| 158 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 159 | + | |
| 160 | +typedef struct { unsigned long pmd; } pmd_t; | |
| 161 | +#define pmd_val(x) ((x).pmd) | |
| 162 | +#define __pmd(x) ((pmd_t) { (x) }) | |
| 163 | + | |
| 164 | +extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; | |
| 165 | + | |
| 166 | +/* | |
| 167 | + * Empty pud entries point to the invalid_pmd_table. | |
| 168 | + */ | |
| 169 | +static inline int pud_none(pud_t pud) | |
| 170 | +{ | |
| 171 | + return pud_val(pud) == (unsigned long)invalid_pmd_table; | |
| 172 | +} | |
| 173 | + | |
| 174 | +static inline int pud_bad(pud_t pud) | |
| 175 | +{ | |
| 176 | + return pud_val(pud) & ~PAGE_MASK; | |
| 177 | +} | |
| 178 | + | |
| 179 | +static inline int pud_present(pud_t pud) | |
| 180 | +{ | |
| 181 | + return pud_val(pud) != (unsigned long)invalid_pmd_table; | |
| 182 | +} | |
| 183 | + | |
| 184 | +static inline void pud_clear(pud_t *pudp) | |
| 185 | +{ | |
| 186 | + pud_val(*pudp) = ((unsigned long)invalid_pmd_table); | |
| 187 | +} | |
| 188 | + | |
| 189 | +static inline pmd_t *pud_pgtable(pud_t pud) | |
| 190 | +{ | |
| 191 | + return (pmd_t *)pud_val(pud); | |
| 192 | +} | |
| 193 | + | |
| 194 | +#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0) | |
| 195 | + | |
| 196 | +#define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) | |
| 197 | +#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) | |
| 198 | + | |
| 199 | +#endif | |
| 200 | + | |
| 201 | +/* | |
| 202 | + * Empty pmd entries point to the invalid_pte_table. | |
| 203 | + */ | |
| 204 | +static inline int pmd_none(pmd_t pmd) | |
| 205 | +{ | |
| 206 | + return pmd_val(pmd) == (unsigned long)invalid_pte_table; | |
| 207 | +} | |
| 208 | + | |
| 209 | +static inline int pmd_bad(pmd_t pmd) | |
| 210 | +{ | |
| 211 | + return (pmd_val(pmd) & ~PAGE_MASK); | |
| 212 | +} | |
| 213 | + | |
| 214 | +static inline int pmd_present(pmd_t pmd) | |
| 215 | +{ | |
| 216 | + if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) | |
| 217 | + return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE)); | |
| 218 | + | |
| 219 | + return pmd_val(pmd) != (unsigned long)invalid_pte_table; | |
| 220 | +} | |
| 221 | + | |
| 222 | +static inline void pmd_clear(pmd_t *pmdp) | |
| 223 | +{ | |
| 224 | + pmd_val(*pmdp) = ((unsigned long)invalid_pte_table); | |
| 225 | +} | |
| 226 | + | |
| 227 | +#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0) | |
| 228 | + | |
| 229 | +#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) | |
| 230 | + | |
| 231 | +#ifndef CONFIG_TRANSPARENT_HUGEPAGE | |
| 232 | +#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) | |
| 233 | +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
| 234 | + | |
| 235 | +#define pmd_page_vaddr(pmd) pmd_val(pmd) | |
| 236 | + | |
| 237 | +extern pmd_t mk_pmd(struct page *page, pgprot_t prot); | |
| 238 | +extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd); | |
| 239 | + | |
| 240 | +#define pte_page(x) pfn_to_page(pte_pfn(x)) | |
| 241 | +#define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> _PFN_SHIFT)) | |
| 242 | +#define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) | |
| 243 | +#define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) | |
| 244 | + | |
| 245 | +/* | |
| 246 | + * Initialize a new pgd / pmd table with invalid pointers. | |
| 247 | + */ | |
| 248 | +extern void pgd_init(unsigned long page); | |
| 249 | +extern void pud_init(unsigned long page, unsigned long pagetable); | |
| 250 | +extern void pmd_init(unsigned long page, unsigned long pagetable); | |
| 251 | + | |
| 252 | +/* | |
| 253 | + * Non-present pages: high 40 bits are offset, next 8 bits type, | |
| 254 | + * low 16 bits zero. | |
| 255 | + */ | |
| 256 | +static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) | |
| 257 | +{ pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; } | |
| 258 | + | |
| 259 | +#define __swp_type(x) (((x).val >> 16) & 0xff) | |
| 260 | +#define __swp_offset(x) ((x).val >> 24) | |
| 261 | +#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) | |
| 262 | +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) | |
| 263 | +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) | |
| 264 | +#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) | |
| 265 | +#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE }) | |
| 266 | + | |
| 267 | +extern void paging_init(void); | |
| 268 | + | |
| 269 | +#define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) | |
| 270 | +#define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE)) | |
| 271 | +#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC) | |
| 272 | + | |
| 273 | +static inline void set_pte(pte_t *ptep, pte_t pteval) | |
| 274 | +{ | |
| 275 | + *ptep = pteval; | |
| 276 | + if (pte_val(pteval) & _PAGE_GLOBAL) { | |
| 277 | + pte_t *buddy = ptep_buddy(ptep); | |
| 278 | + /* | |
| 279 | + * Make sure the buddy is global too (if it's !none, | |
| 280 | + * it better already be global) | |
| 281 | + */ | |
| 282 | + if (pte_none(*buddy)) | |
| 283 | + pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL; | |
| 284 | + } | |
| 285 | +} | |
| 286 | + | |
| 287 | +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, | |
| 288 | + pte_t *ptep, pte_t pteval) | |
| 289 | +{ | |
| 290 | + set_pte(ptep, pteval); | |
| 291 | +} | |
| 292 | + | |
| 293 | +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
| 294 | +{ | |
| 295 | + /* Preserve global status for the pair */ | |
| 296 | + if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL) | |
| 297 | + set_pte_at(mm, addr, ptep, __pte(_PAGE_GLOBAL)); | |
| 298 | + else | |
| 299 | + set_pte_at(mm, addr, ptep, __pte(0)); | |
| 300 | +} | |
| 301 | + | |
| 302 | +#define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1) | |
| 303 | +#define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1) | |
| 304 | +#define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1) | |
| 305 | + | |
| 306 | +extern pgd_t swapper_pg_dir[]; | |
| 307 | +extern pgd_t invalid_pg_dir[]; | |
| 308 | + | |
| 309 | +/* | |
| 310 | + * The following only work if pte_present() is true. | |
| 311 | + * Undefined behaviour if not.. | |
| 312 | + */ | |
| 313 | +static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; } | |
| 314 | +static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } | |
| 315 | +static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; } | |
| 316 | + | |
| 317 | +static inline pte_t pte_mkold(pte_t pte) | |
| 318 | +{ | |
| 319 | + pte_val(pte) &= ~_PAGE_ACCESSED; | |
| 320 | + return pte; | |
| 321 | +} | |
| 322 | + | |
| 323 | +static inline pte_t pte_mkyoung(pte_t pte) | |
| 324 | +{ | |
| 325 | + pte_val(pte) |= _PAGE_ACCESSED; | |
| 326 | + return pte; | |
| 327 | +} | |
| 328 | + | |
| 329 | +static inline pte_t pte_mkclean(pte_t pte) | |
| 330 | +{ | |
| 331 | + pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); | |
| 332 | + return pte; | |
| 333 | +} | |
| 334 | + | |
| 335 | +static inline pte_t pte_mkdirty(pte_t pte) | |
| 336 | +{ | |
| 337 | + pte_val(pte) |= (_PAGE_DIRTY | _PAGE_MODIFIED); | |
| 338 | + return pte; | |
| 339 | +} | |
| 340 | + | |
| 341 | +static inline pte_t pte_mkwrite(pte_t pte) | |
| 342 | +{ | |
| 343 | + pte_val(pte) |= (_PAGE_WRITE | _PAGE_DIRTY); | |
| 344 | + return pte; | |
| 345 | +} | |
| 346 | + | |
| 347 | +static inline pte_t pte_wrprotect(pte_t pte) | |
| 348 | +{ | |
| 349 | + pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY); | |
| 350 | + return pte; | |
| 351 | +} | |
| 352 | + | |
| 353 | +static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; } | |
| 354 | + | |
| 355 | +static inline pte_t pte_mkhuge(pte_t pte) | |
| 356 | +{ | |
| 357 | + pte_val(pte) |= _PAGE_HUGE; | |
| 358 | + return pte; | |
| 359 | +} | |
| 360 | + | |
| 361 | +#if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) | |
| 362 | +static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } | |
| 363 | +static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; } | |
| 364 | +#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ | |
| 365 | + | |
| 366 | +#define pte_accessible pte_accessible | |
| 367 | +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) | |
| 368 | +{ | |
| 369 | + if (pte_val(a) & _PAGE_PRESENT) | |
| 370 | + return true; | |
| 371 | + | |
| 372 | + if ((pte_val(a) & _PAGE_PROTNONE) && | |
| 373 | + atomic_read(&mm->tlb_flush_pending)) | |
| 374 | + return true; | |
| 375 | + | |
| 376 | + return false; | |
| 377 | +} | |
| 378 | + | |
| 379 | +/* | |
| 380 | + * Conversion functions: convert a page and protection to a page entry, | |
| 381 | + * and a page entry and page directory to the page they refer to. | |
| 382 | + */ | |
| 383 | +#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | |
| 384 | + | |
| 385 | +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) | |
| 386 | +{ | |
| 387 | + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | | |
| 388 | + (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); | |
| 389 | +} | |
| 390 | + | |
| 391 | +extern void __update_tlb(struct vm_area_struct *vma, | |
| 392 | + unsigned long address, pte_t *ptep); | |
| 393 | + | |
| 394 | +static inline void update_mmu_cache(struct vm_area_struct *vma, | |
| 395 | + unsigned long address, pte_t *ptep) | |
| 396 | +{ | |
| 397 | + __update_tlb(vma, address, ptep); | |
| 398 | +} | |
| 399 | + | |
| 400 | +static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, | |
| 401 | + unsigned long address, pmd_t *pmdp) | |
| 402 | +{ | |
| 403 | + __update_tlb(vma, address, (pte_t *)pmdp); | |
| 404 | +} | |
| 405 | + | |
| 406 | +#define kern_addr_valid(addr) (1) | |
| 407 | + | |
| 408 | +#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
| 409 | + | |
| 410 | +/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/ | |
| 411 | +#define pmdp_establish generic_pmdp_establish | |
| 412 | + | |
| 413 | +static inline int pmd_trans_huge(pmd_t pmd) | |
| 414 | +{ | |
| 415 | + return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd); | |
| 416 | +} | |
| 417 | + | |
| 418 | +static inline pmd_t pmd_mkhuge(pmd_t pmd) | |
| 419 | +{ | |
| 420 | + pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) | | |
| 421 | + ((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); | |
| 422 | + pmd_val(pmd) |= _PAGE_HUGE; | |
| 423 | + | |
| 424 | + return pmd; | |
| 425 | +} | |
| 426 | + | |
| 427 | +#define pmd_write pmd_write | |
| 428 | +static inline int pmd_write(pmd_t pmd) | |
| 429 | +{ | |
| 430 | + return !!(pmd_val(pmd) & _PAGE_WRITE); | |
| 431 | +} | |
| 432 | + | |
| 433 | +static inline pmd_t pmd_mkwrite(pmd_t pmd) | |
| 434 | +{ | |
| 435 | + pmd_val(pmd) |= (_PAGE_WRITE | _PAGE_DIRTY); | |
| 436 | + return pmd; | |
| 437 | +} | |
| 438 | + | |
| 439 | +static inline pmd_t pmd_wrprotect(pmd_t pmd) | |
| 440 | +{ | |
| 441 | + pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY); | |
| 442 | + return pmd; | |
| 443 | +} | |
| 444 | + | |
| 445 | +static inline int pmd_dirty(pmd_t pmd) | |
| 446 | +{ | |
| 447 | + return !!(pmd_val(pmd) & _PAGE_MODIFIED); | |
| 448 | +} | |
| 449 | + | |
| 450 | +static inline pmd_t pmd_mkclean(pmd_t pmd) | |
| 451 | +{ | |
| 452 | + pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED); | |
| 453 | + return pmd; | |
| 454 | +} | |
| 455 | + | |
| 456 | +static inline pmd_t pmd_mkdirty(pmd_t pmd) | |
| 457 | +{ | |
| 458 | + pmd_val(pmd) |= (_PAGE_DIRTY | _PAGE_MODIFIED); | |
| 459 | + return pmd; | |
| 460 | +} | |
| 461 | + | |
| 462 | +static inline int pmd_young(pmd_t pmd) | |
| 463 | +{ | |
| 464 | + return !!(pmd_val(pmd) & _PAGE_ACCESSED); | |
| 465 | +} | |
| 466 | + | |
| 467 | +static inline pmd_t pmd_mkold(pmd_t pmd) | |
| 468 | +{ | |
| 469 | + pmd_val(pmd) &= ~_PAGE_ACCESSED; | |
| 470 | + return pmd; | |
| 471 | +} | |
| 472 | + | |
| 473 | +static inline pmd_t pmd_mkyoung(pmd_t pmd) | |
| 474 | +{ | |
| 475 | + pmd_val(pmd) |= _PAGE_ACCESSED; | |
| 476 | + return pmd; | |
| 477 | +} | |
| 478 | + | |
| 479 | +static inline unsigned long pmd_pfn(pmd_t pmd) | |
| 480 | +{ | |
| 481 | + return (pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT; | |
| 482 | +} | |
| 483 | + | |
| 484 | +static inline struct page *pmd_page(pmd_t pmd) | |
| 485 | +{ | |
| 486 | + if (pmd_trans_huge(pmd)) | |
| 487 | + return pfn_to_page(pmd_pfn(pmd)); | |
| 488 | + | |
| 489 | + return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT); | |
| 490 | +} | |
| 491 | + | |
| 492 | +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) | |
| 493 | +{ | |
| 494 | + pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | | |
| 495 | + (pgprot_val(newprot) & ~_HPAGE_CHG_MASK); | |
| 496 | + return pmd; | |
| 497 | +} | |
| 498 | + | |
| 499 | +static inline pmd_t pmd_mkinvalid(pmd_t pmd) | |
| 500 | +{ | |
| 501 | + pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE); | |
| 502 | + | |
| 503 | + return pmd; | |
| 504 | +} | |
| 505 | + | |
| 506 | +/* | |
| 507 | + * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a | |
| 508 | + * different prototype. | |
| 509 | + */ | |
| 510 | +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR | |
| 511 | +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, | |
| 512 | + unsigned long address, pmd_t *pmdp) | |
| 513 | +{ | |
| 514 | + pmd_t old = *pmdp; | |
| 515 | + | |
| 516 | + pmd_clear(pmdp); | |
| 517 | + | |
| 518 | + return old; | |
| 519 | +} | |
| 520 | + | |
| 521 | +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
| 522 | + | |
| 523 | +/* | |
| 524 | + * We provide our own get_unmapped area to cope with the virtual aliasing | |
| 525 | + * constraints placed on us by the cache architecture. | |
| 526 | + */ | |
| 527 | +#define HAVE_ARCH_UNMAPPED_AREA | |
| 528 | +#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN | |
| 529 | + | |
| 530 | +#endif /* !__ASSEMBLY__ */ | |
| 531 | + | |
| 532 | +#endif /* _ASM_PGTABLE_H */ | |
| @@ -0,0 +1,12 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef _ASM_SHMPARAM_H | |
| 6 | +#define _ASM_SHMPARAM_H | |
| 7 | + | |
| 8 | +#define __ARCH_FORCE_SHMLBA 1 | |
| 9 | + | |
| 10 | +#define SHMLBA SZ_64K /* attach addr a multiple of this */ | |
| 11 | + | |
| 12 | +#endif /* _ASM_SHMPARAM_H */ | |
| @@ -0,0 +1,23 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +#ifndef _LOONGARCH_SPARSEMEM_H | |
| 3 | +#define _LOONGARCH_SPARSEMEM_H | |
| 4 | + | |
| 5 | +#ifdef CONFIG_SPARSEMEM | |
| 6 | + | |
| 7 | +/* | |
| 8 | + * SECTION_SIZE_BITS 2^N: how big each section will be | |
| 9 | + * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space | |
| 10 | + */ | |
| 11 | +#define SECTION_SIZE_BITS 29 /* 2^29 = Largest Huge Page Size */ | |
| 12 | +#define MAX_PHYSMEM_BITS 48 | |
| 13 | + | |
| 14 | +#endif /* CONFIG_SPARSEMEM */ | |
| 15 | + | |
| 16 | +#ifdef CONFIG_MEMORY_HOTPLUG | |
| 17 | +int memory_add_physaddr_to_nid(u64 addr); | |
| 18 | +#define memory_add_physaddr_to_nid memory_add_physaddr_to_nid | |
| 19 | +#endif | |
| 20 | + | |
| 21 | +#define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS) | |
| 22 | + | |
| 23 | +#endif /* _LOONGARCH_SPARSEMEM_H */ | |
| @@ -0,0 +1,180 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef __ASM_TLB_H | |
| 6 | +#define __ASM_TLB_H | |
| 7 | + | |
| 8 | +#include <linux/mm_types.h> | |
| 9 | +#include <asm/cpu-features.h> | |
| 10 | +#include <asm/loongarch.h> | |
| 11 | + | |
| 12 | +/* | |
| 13 | + * TLB Invalidate Flush | |
| 14 | + */ | |
| 15 | +static inline void tlbclr(void) | |
| 16 | +{ | |
| 17 | + __asm__ __volatile__("tlbclr"); | |
| 18 | +} | |
| 19 | + | |
| 20 | +static inline void tlbflush(void) | |
| 21 | +{ | |
| 22 | + __asm__ __volatile__("tlbflush"); | |
| 23 | +} | |
| 24 | + | |
| 25 | +/* | |
| 26 | + * TLB R/W operations. | |
| 27 | + */ | |
| 28 | +static inline void tlb_probe(void) | |
| 29 | +{ | |
| 30 | + __asm__ __volatile__("tlbsrch"); | |
| 31 | +} | |
| 32 | + | |
| 33 | +static inline void tlb_read(void) | |
| 34 | +{ | |
| 35 | + __asm__ __volatile__("tlbrd"); | |
| 36 | +} | |
| 37 | + | |
| 38 | +static inline void tlb_write_indexed(void) | |
| 39 | +{ | |
| 40 | + __asm__ __volatile__("tlbwr"); | |
| 41 | +} | |
| 42 | + | |
| 43 | +static inline void tlb_write_random(void) | |
| 44 | +{ | |
| 45 | + __asm__ __volatile__("tlbfill"); | |
| 46 | +} | |
| 47 | + | |
| 48 | +enum invtlb_ops { | |
| 49 | + /* Invalid all tlb */ | |
| 50 | + INVTLB_ALL = 0x0, | |
| 51 | + /* Invalid current tlb */ | |
| 52 | + INVTLB_CURRENT_ALL = 0x1, | |
| 53 | + /* Invalid all global=1 lines in current tlb */ | |
| 54 | + INVTLB_CURRENT_GTRUE = 0x2, | |
| 55 | + /* Invalid all global=0 lines in current tlb */ | |
| 56 | + INVTLB_CURRENT_GFALSE = 0x3, | |
| 57 | + /* Invalid global=0 and matched asid lines in current tlb */ | |
| 58 | + INVTLB_GFALSE_AND_ASID = 0x4, | |
| 59 | + /* Invalid addr with global=0 and matched asid in current tlb */ | |
| 60 | + INVTLB_ADDR_GFALSE_AND_ASID = 0x5, | |
| 61 | + /* Invalid addr with global=1 or matched asid in current tlb */ | |
| 62 | + INVTLB_ADDR_GTRUE_OR_ASID = 0x6, | |
| 63 | + /* Invalid matched gid in guest tlb */ | |
| 64 | + INVGTLB_GID = 0x9, | |
| 65 | + /* Invalid global=1, matched gid in guest tlb */ | |
| 66 | + INVGTLB_GID_GTRUE = 0xa, | |
| 67 | + /* Invalid global=0, matched gid in guest tlb */ | |
| 68 | + INVGTLB_GID_GFALSE = 0xb, | |
| 69 | + /* Invalid global=0, matched gid and asid in guest tlb */ | |
| 70 | + INVGTLB_GID_GFALSE_ASID = 0xc, | |
| 71 | + /* Invalid global=0 , matched gid, asid and addr in guest tlb */ | |
| 72 | + INVGTLB_GID_GFALSE_ASID_ADDR = 0xd, | |
| 73 | + /* Invalid global=1 , matched gid, asid and addr in guest tlb */ | |
| 74 | + INVGTLB_GID_GTRUE_ASID_ADDR = 0xe, | |
| 75 | + /* Invalid all gid gva-->gpa guest tlb */ | |
| 76 | + INVGTLB_ALLGID_GVA_TO_GPA = 0x10, | |
| 77 | + /* Invalid all gid gpa-->hpa tlb */ | |
| 78 | + INVTLB_ALLGID_GPA_TO_HPA = 0x11, | |
| 79 | + /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */ | |
| 80 | + INVTLB_ALLGID = 0x12, | |
| 81 | + /* Invalid matched gid gva-->gpa guest tlb */ | |
| 82 | + INVGTLB_GID_GVA_TO_GPA = 0x13, | |
| 83 | + /* Invalid matched gid gpa-->hpa tlb */ | |
| 84 | + INVTLB_GID_GPA_TO_HPA = 0x14, | |
| 85 | + /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */ | |
| 86 | + INVTLB_GID_ALL = 0x15, | |
| 87 | + /* Invalid matched gid and addr gpa-->hpa tlb */ | |
| 88 | + INVTLB_GID_ADDR = 0x16, | |
| 89 | +}; | |
| 90 | + | |
| 91 | +/* | |
| 92 | + * invtlb op info addr | |
| 93 | + * (0x1 << 26) | (0x24 << 20) | (0x13 << 15) | | |
| 94 | + * (addr << 10) | (info << 5) | op | |
| 95 | + */ | |
| 96 | +static inline void invtlb(u32 op, u32 info, u64 addr) | |
| 97 | +{ | |
| 98 | + __asm__ __volatile__( | |
| 99 | + "parse_r addr,%0\n\t" | |
| 100 | + "parse_r info,%1\n\t" | |
| 101 | + ".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t" | |
| 102 | + : | |
| 103 | + : "r"(addr), "r"(info), "i"(op) | |
| 104 | + : | |
| 105 | + ); | |
| 106 | +} | |
| 107 | + | |
| 108 | +static inline void invtlb_addr(u32 op, u32 info, u64 addr) | |
| 109 | +{ | |
| 110 | + __asm__ __volatile__( | |
| 111 | + "parse_r addr,%0\n\t" | |
| 112 | + ".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t" | |
| 113 | + : | |
| 114 | + : "r"(addr), "i"(op) | |
| 115 | + : | |
| 116 | + ); | |
| 117 | +} | |
| 118 | + | |
| 119 | +static inline void invtlb_info(u32 op, u32 info, u64 addr) | |
| 120 | +{ | |
| 121 | + __asm__ __volatile__( | |
| 122 | + "parse_r info,%0\n\t" | |
| 123 | + ".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t" | |
| 124 | + : | |
| 125 | + : "r"(info), "i"(op) | |
| 126 | + : | |
| 127 | + ); | |
| 128 | +} | |
| 129 | + | |
| 130 | +static inline void invtlb_all(u32 op, u32 info, u64 addr) | |
| 131 | +{ | |
| 132 | + __asm__ __volatile__( | |
| 133 | + ".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t" | |
| 134 | + : | |
| 135 | + : "i"(op) | |
| 136 | + : | |
| 137 | + ); | |
| 138 | +} | |
| 139 | + | |
| 140 | +/* | |
| 141 | + * LoongArch doesn't need any special per-pte or per-vma handling, except | |
| 142 | + * we need to flush cache for area to be unmapped. | |
| 143 | + */ | |
| 144 | +#define tlb_start_vma(tlb, vma) \ | |
| 145 | + do { \ | |
| 146 | + if (!(tlb)->fullmm) \ | |
| 147 | + flush_cache_range(vma, vma->vm_start, vma->vm_end); \ | |
| 148 | + } while (0) | |
| 149 | +#define tlb_end_vma(tlb, vma) do { } while (0) | |
| 150 | +#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) | |
| 151 | + | |
| 152 | +static void tlb_flush(struct mmu_gather *tlb); | |
| 153 | + | |
| 154 | +#define tlb_flush tlb_flush | |
| 155 | +#include <asm-generic/tlb.h> | |
| 156 | + | |
| 157 | +static inline void tlb_flush(struct mmu_gather *tlb) | |
| 158 | +{ | |
| 159 | + struct vm_area_struct vma; | |
| 160 | + | |
| 161 | + vma.vm_mm = tlb->mm; | |
| 162 | + vma.vm_flags = 0; | |
| 163 | + if (tlb->fullmm) { | |
| 164 | + flush_tlb_mm(tlb->mm); | |
| 165 | + return; | |
| 166 | + } | |
| 167 | + | |
| 168 | + flush_tlb_range(&vma, tlb->start, tlb->end); | |
| 169 | +} | |
| 170 | + | |
| 171 | +extern void handle_tlb_load(void); | |
| 172 | +extern void handle_tlb_store(void); | |
| 173 | +extern void handle_tlb_modify(void); | |
| 174 | +extern void handle_tlb_refill(void); | |
| 175 | +extern void handle_tlb_protect(void); | |
| 176 | + | |
| 177 | +extern void dump_tlb_all(void); | |
| 178 | +extern void dump_tlb_regs(void); | |
| 179 | + | |
| 180 | +#endif /* __ASM_TLB_H */ | |
| @@ -0,0 +1,35 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#ifndef __ASM_TLBFLUSH_H | |
| 6 | +#define __ASM_TLBFLUSH_H | |
| 7 | + | |
| 8 | +#include <linux/mm.h> | |
| 9 | + | |
| 10 | +/* | |
| 11 | + * TLB flushing: | |
| 12 | + * | |
| 13 | + * - flush_tlb_all() flushes all processes TLB entries | |
| 14 | + * - flush_tlb_mm(mm) flushes the specified mm context TLB entries | |
| 15 | + * - flush_tlb_page(vma, vmaddr) flushes one page | |
| 16 | + * - flush_tlb_range(vma, start, end) flushes a range of pages | |
| 17 | + * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages | |
| 18 | + */ | |
| 19 | +extern void local_flush_tlb_all(void); | |
| 20 | +extern void local_flush_tlb_user(void); | |
| 21 | +extern void local_flush_tlb_kernel(void); | |
| 22 | +extern void local_flush_tlb_mm(struct mm_struct *mm); | |
| 23 | +extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); | |
| 24 | +extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); | |
| 25 | +extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); | |
| 26 | +extern void local_flush_tlb_one(unsigned long vaddr); | |
| 27 | + | |
| 28 | +#define flush_tlb_all() local_flush_tlb_all() | |
| 29 | +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) | |
| 30 | +#define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end) | |
| 31 | +#define flush_tlb_kernel_range(vmaddr, end) local_flush_tlb_kernel_range(vmaddr, end) | |
| 32 | +#define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page) | |
| 33 | +#define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr) | |
| 34 | + | |
| 35 | +#endif /* __ASM_TLBFLUSH_H */ | |
| @@ -0,0 +1,4 @@ | ||
| 1 | +#ifndef _ASM_LOONGARCH_VMALLOC_H | |
| 2 | +#define _ASM_LOONGARCH_VMALLOC_H | |
| 3 | + | |
| 4 | +#endif /* _ASM_LOONGARCH_VMALLOC_H */ | |
| @@ -0,0 +1,141 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + * | |
| 5 | + * Derived from MIPS: | |
| 6 | + * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) | |
| 7 | + * Copyright (C) 2007 MIPS Technologies, Inc. | |
| 8 | + */ | |
| 9 | +#include <linux/export.h> | |
| 10 | +#include <linux/fcntl.h> | |
| 11 | +#include <linux/fs.h> | |
| 12 | +#include <linux/highmem.h> | |
| 13 | +#include <linux/kernel.h> | |
| 14 | +#include <linux/linkage.h> | |
| 15 | +#include <linux/mm.h> | |
| 16 | +#include <linux/sched.h> | |
| 17 | +#include <linux/syscalls.h> | |
| 18 | + | |
| 19 | +#include <asm/cacheflush.h> | |
| 20 | +#include <asm/cpu.h> | |
| 21 | +#include <asm/cpu-features.h> | |
| 22 | +#include <asm/dma.h> | |
| 23 | +#include <asm/loongarch.h> | |
| 24 | +#include <asm/processor.h> | |
| 25 | +#include <asm/setup.h> | |
| 26 | + | |
| 27 | +/* | |
| 28 | + * LoongArch maintains ICache/DCache coherency by hardware, | |
| 29 | + * we just need "ibar" to avoid instruction hazard here. | |
| 30 | + */ | |
| 31 | +void local_flush_icache_range(unsigned long start, unsigned long end) | |
| 32 | +{ | |
| 33 | + asm volatile ("\tibar 0\n"::); | |
| 34 | +} | |
| 35 | +EXPORT_SYMBOL(local_flush_icache_range); | |
| 36 | + | |
| 37 | +void cache_error_setup(void) | |
| 38 | +{ | |
| 39 | + extern char __weak except_vec_cex; | |
| 40 | + set_merr_handler(0x0, &except_vec_cex, 0x80); | |
| 41 | +} | |
| 42 | + | |
| 43 | +static unsigned long icache_size __read_mostly; | |
| 44 | +static unsigned long dcache_size __read_mostly; | |
| 45 | +static unsigned long vcache_size __read_mostly; | |
| 46 | +static unsigned long scache_size __read_mostly; | |
| 47 | + | |
| 48 | +static char *way_string[] = { NULL, "direct mapped", "2-way", | |
| 49 | + "3-way", "4-way", "5-way", "6-way", "7-way", "8-way", | |
| 50 | + "9-way", "10-way", "11-way", "12-way", | |
| 51 | + "13-way", "14-way", "15-way", "16-way", | |
| 52 | +}; | |
| 53 | + | |
| 54 | +static void probe_pcache(void) | |
| 55 | +{ | |
| 56 | + struct cpuinfo_loongarch *c = ¤t_cpu_data; | |
| 57 | + unsigned int lsize, sets, ways; | |
| 58 | + unsigned int config; | |
| 59 | + | |
| 60 | + config = read_cpucfg(LOONGARCH_CPUCFG17); | |
| 61 | + lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE); | |
| 62 | + sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS); | |
| 63 | + ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1; | |
| 64 | + | |
| 65 | + c->icache.linesz = lsize; | |
| 66 | + c->icache.sets = sets; | |
| 67 | + c->icache.ways = ways; | |
| 68 | + icache_size = sets * ways * lsize; | |
| 69 | + c->icache.waysize = icache_size / c->icache.ways; | |
| 70 | + | |
| 71 | + config = read_cpucfg(LOONGARCH_CPUCFG18); | |
| 72 | + lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE); | |
| 73 | + sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS); | |
| 74 | + ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1; | |
| 75 | + | |
| 76 | + c->dcache.linesz = lsize; | |
| 77 | + c->dcache.sets = sets; | |
| 78 | + c->dcache.ways = ways; | |
| 79 | + dcache_size = sets * ways * lsize; | |
| 80 | + c->dcache.waysize = dcache_size / c->dcache.ways; | |
| 81 | + | |
| 82 | + c->options |= LOONGARCH_CPU_PREFETCH; | |
| 83 | + | |
| 84 | + pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", | |
| 85 | + icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz); | |
| 86 | + | |
| 87 | + pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", | |
| 88 | + dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz); | |
| 89 | +} | |
| 90 | + | |
| 91 | +static void probe_vcache(void) | |
| 92 | +{ | |
| 93 | + struct cpuinfo_loongarch *c = ¤t_cpu_data; | |
| 94 | + unsigned int lsize, sets, ways; | |
| 95 | + unsigned int config; | |
| 96 | + | |
| 97 | + config = read_cpucfg(LOONGARCH_CPUCFG19); | |
| 98 | + lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE); | |
| 99 | + sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS); | |
| 100 | + ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1; | |
| 101 | + | |
| 102 | + c->vcache.linesz = lsize; | |
| 103 | + c->vcache.sets = sets; | |
| 104 | + c->vcache.ways = ways; | |
| 105 | + vcache_size = lsize * sets * ways; | |
| 106 | + c->vcache.waysize = vcache_size / c->vcache.ways; | |
| 107 | + | |
| 108 | + pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n", | |
| 109 | + vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz); | |
| 110 | +} | |
| 111 | + | |
| 112 | +static void probe_scache(void) | |
| 113 | +{ | |
| 114 | + struct cpuinfo_loongarch *c = ¤t_cpu_data; | |
| 115 | + unsigned int lsize, sets, ways; | |
| 116 | + unsigned int config; | |
| 117 | + | |
| 118 | + config = read_cpucfg(LOONGARCH_CPUCFG20); | |
| 119 | + lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE); | |
| 120 | + sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS); | |
| 121 | + ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1; | |
| 122 | + | |
| 123 | + c->scache.linesz = lsize; | |
| 124 | + c->scache.sets = sets; | |
| 125 | + c->scache.ways = ways; | |
| 126 | + /* 4 cores. scaches are shared */ | |
| 127 | + scache_size = lsize * sets * ways; | |
| 128 | + c->scache.waysize = scache_size / c->scache.ways; | |
| 129 | + | |
| 130 | + pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", | |
| 131 | + scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); | |
| 132 | +} | |
| 133 | + | |
| 134 | +void cpu_cache_init(void) | |
| 135 | +{ | |
| 136 | + probe_pcache(); | |
| 137 | + probe_vcache(); | |
| 138 | + probe_scache(); | |
| 139 | + | |
| 140 | + shm_align_mask = PAGE_SIZE - 1; | |
| 141 | +} | |
| @@ -0,0 +1,22 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/extable.h> | |
| 6 | +#include <linux/spinlock.h> | |
| 7 | +#include <asm/branch.h> | |
| 8 | +#include <linux/uaccess.h> | |
| 9 | + | |
| 10 | +int fixup_exception(struct pt_regs *regs) | |
| 11 | +{ | |
| 12 | + const struct exception_table_entry *fixup; | |
| 13 | + | |
| 14 | + fixup = search_exception_tables(exception_era(regs)); | |
| 15 | + if (fixup) { | |
| 16 | + regs->csr_era = fixup->fixup; | |
| 17 | + | |
| 18 | + return 1; | |
| 19 | + } | |
| 20 | + | |
| 21 | + return 0; | |
| 22 | +} | |
| @@ -0,0 +1,261 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + * | |
| 5 | + * Derived from MIPS: | |
| 6 | + * Copyright (C) 1995 - 2000 by Ralf Baechle | |
| 7 | + */ | |
| 8 | +#include <linux/context_tracking.h> | |
| 9 | +#include <linux/signal.h> | |
| 10 | +#include <linux/sched.h> | |
| 11 | +#include <linux/interrupt.h> | |
| 12 | +#include <linux/kernel.h> | |
| 13 | +#include <linux/entry-common.h> | |
| 14 | +#include <linux/errno.h> | |
| 15 | +#include <linux/string.h> | |
| 16 | +#include <linux/types.h> | |
| 17 | +#include <linux/ptrace.h> | |
| 18 | +#include <linux/ratelimit.h> | |
| 19 | +#include <linux/mman.h> | |
| 20 | +#include <linux/mm.h> | |
| 21 | +#include <linux/smp.h> | |
| 22 | +#include <linux/kdebug.h> | |
| 23 | +#include <linux/kprobes.h> | |
| 24 | +#include <linux/perf_event.h> | |
| 25 | +#include <linux/uaccess.h> | |
| 26 | + | |
| 27 | +#include <asm/branch.h> | |
| 28 | +#include <asm/mmu_context.h> | |
| 29 | +#include <asm/ptrace.h> | |
| 30 | + | |
| 31 | +int show_unhandled_signals = 1; | |
| 32 | + | |
| 33 | +static void __kprobes no_context(struct pt_regs *regs, unsigned long address) | |
| 34 | +{ | |
| 35 | + const int field = sizeof(unsigned long) * 2; | |
| 36 | + | |
| 37 | + /* Are we prepared to handle this kernel fault? */ | |
| 38 | + if (fixup_exception(regs)) | |
| 39 | + return; | |
| 40 | + | |
| 41 | + /* | |
| 42 | + * Oops. The kernel tried to access some bad page. We'll have to | |
| 43 | + * terminate things with extreme prejudice. | |
| 44 | + */ | |
| 45 | + bust_spinlocks(1); | |
| 46 | + | |
| 47 | + pr_alert("CPU %d Unable to handle kernel paging request at " | |
| 48 | + "virtual address %0*lx, era == %0*lx, ra == %0*lx\n", | |
| 49 | + raw_smp_processor_id(), field, address, field, regs->csr_era, | |
| 50 | + field, regs->regs[1]); | |
| 51 | + die("Oops", regs); | |
| 52 | +} | |
| 53 | + | |
| 54 | +static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address) | |
| 55 | +{ | |
| 56 | + /* | |
| 57 | + * We ran out of memory, call the OOM killer, and return the userspace | |
| 58 | + * (which will retry the fault, or kill us if we got oom-killed). | |
| 59 | + */ | |
| 60 | + if (!user_mode(regs)) { | |
| 61 | + no_context(regs, address); | |
| 62 | + return; | |
| 63 | + } | |
| 64 | + pagefault_out_of_memory(); | |
| 65 | +} | |
| 66 | + | |
| 67 | +static void __kprobes do_sigbus(struct pt_regs *regs, | |
| 68 | + unsigned long write, unsigned long address, int si_code) | |
| 69 | +{ | |
| 70 | + /* Kernel mode? Handle exceptions or die */ | |
| 71 | + if (!user_mode(regs)) { | |
| 72 | + no_context(regs, address); | |
| 73 | + return; | |
| 74 | + } | |
| 75 | + | |
| 76 | + /* | |
| 77 | + * Send a sigbus, regardless of whether we were in kernel | |
| 78 | + * or user mode. | |
| 79 | + */ | |
| 80 | + current->thread.csr_badvaddr = address; | |
| 81 | + current->thread.trap_nr = read_csr_excode(); | |
| 82 | + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); | |
| 83 | +} | |
| 84 | + | |
| 85 | +static void __kprobes do_sigsegv(struct pt_regs *regs, | |
| 86 | + unsigned long write, unsigned long address, int si_code) | |
| 87 | +{ | |
| 88 | + const int field = sizeof(unsigned long) * 2; | |
| 89 | + static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); | |
| 90 | + | |
| 91 | + /* Kernel mode? Handle exceptions or die */ | |
| 92 | + if (!user_mode(regs)) { | |
| 93 | + no_context(regs, address); | |
| 94 | + return; | |
| 95 | + } | |
| 96 | + | |
| 97 | + /* User mode accesses just cause a SIGSEGV */ | |
| 98 | + current->thread.csr_badvaddr = address; | |
| 99 | + if (!write) | |
| 100 | + current->thread.error_code = 1; | |
| 101 | + else | |
| 102 | + current->thread.error_code = 2; | |
| 103 | + current->thread.trap_nr = read_csr_excode(); | |
| 104 | + | |
| 105 | + if (show_unhandled_signals && | |
| 106 | + unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) { | |
| 107 | + pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n", | |
| 108 | + current->comm, | |
| 109 | + write ? "write access to" : "read access from", | |
| 110 | + field, address); | |
| 111 | + pr_info("era = %0*lx in", field, | |
| 112 | + (unsigned long) regs->csr_era); | |
| 113 | + print_vma_addr(KERN_CONT " ", regs->csr_era); | |
| 114 | + pr_cont("\n"); | |
| 115 | + pr_info("ra = %0*lx in", field, | |
| 116 | + (unsigned long) regs->regs[1]); | |
| 117 | + print_vma_addr(KERN_CONT " ", regs->regs[1]); | |
| 118 | + pr_cont("\n"); | |
| 119 | + } | |
| 120 | + force_sig_fault(SIGSEGV, si_code, (void __user *)address); | |
| 121 | +} | |
| 122 | + | |
| 123 | +/* | |
| 124 | + * This routine handles page faults. It determines the address, | |
| 125 | + * and the problem, and then passes it off to one of the appropriate | |
| 126 | + * routines. | |
| 127 | + */ | |
| 128 | +static void __kprobes __do_page_fault(struct pt_regs *regs, | |
| 129 | + unsigned long write, unsigned long address) | |
| 130 | +{ | |
| 131 | + int si_code = SEGV_MAPERR; | |
| 132 | + unsigned int flags = FAULT_FLAG_DEFAULT; | |
| 133 | + struct task_struct *tsk = current; | |
| 134 | + struct mm_struct *mm = tsk->mm; | |
| 135 | + struct vm_area_struct *vma = NULL; | |
| 136 | + vm_fault_t fault; | |
| 137 | + | |
| 138 | + /* | |
| 139 | + * We fault-in kernel-space virtual memory on-demand. The | |
| 140 | + * 'reference' page table is init_mm.pgd. | |
| 141 | + * | |
| 142 | + * NOTE! We MUST NOT take any locks for this case. We may | |
| 143 | + * be in an interrupt or a critical region, and should | |
| 144 | + * only copy the information from the master page table, | |
| 145 | + * nothing more. | |
| 146 | + */ | |
| 147 | + if (address & __UA_LIMIT) { | |
| 148 | + if (!user_mode(regs)) | |
| 149 | + no_context(regs, address); | |
| 150 | + else | |
| 151 | + do_sigsegv(regs, write, address, si_code); | |
| 152 | + return; | |
| 153 | + } | |
| 154 | + | |
| 155 | + /* | |
| 156 | + * If we're in an interrupt or have no user | |
| 157 | + * context, we must not take the fault.. | |
| 158 | + */ | |
| 159 | + if (faulthandler_disabled() || !mm) { | |
| 160 | + do_sigsegv(regs, write, address, si_code); | |
| 161 | + return; | |
| 162 | + } | |
| 163 | + | |
| 164 | + if (user_mode(regs)) | |
| 165 | + flags |= FAULT_FLAG_USER; | |
| 166 | + | |
| 167 | + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); | |
| 168 | +retry: | |
| 169 | + mmap_read_lock(mm); | |
| 170 | + vma = find_vma(mm, address); | |
| 171 | + if (!vma) | |
| 172 | + goto bad_area; | |
| 173 | + if (vma->vm_start <= address) | |
| 174 | + goto good_area; | |
| 175 | + if (!(vma->vm_flags & VM_GROWSDOWN)) | |
| 176 | + goto bad_area; | |
| 177 | + if (!expand_stack(vma, address)) | |
| 178 | + goto good_area; | |
| 179 | +/* | |
| 180 | + * Something tried to access memory that isn't in our memory map.. | |
| 181 | + * Fix it, but check if it's kernel or user first.. | |
| 182 | + */ | |
| 183 | +bad_area: | |
| 184 | + mmap_read_unlock(mm); | |
| 185 | + do_sigsegv(regs, write, address, si_code); | |
| 186 | + return; | |
| 187 | + | |
| 188 | +/* | |
| 189 | + * Ok, we have a good vm_area for this memory access, so | |
| 190 | + * we can handle it.. | |
| 191 | + */ | |
| 192 | +good_area: | |
| 193 | + si_code = SEGV_ACCERR; | |
| 194 | + | |
| 195 | + if (write) { | |
| 196 | + flags |= FAULT_FLAG_WRITE; | |
| 197 | + if (!(vma->vm_flags & VM_WRITE)) | |
| 198 | + goto bad_area; | |
| 199 | + } else { | |
| 200 | + if (!(vma->vm_flags & VM_READ) && address != exception_era(regs)) | |
| 201 | + goto bad_area; | |
| 202 | + if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs)) | |
| 203 | + goto bad_area; | |
| 204 | + } | |
| 205 | + | |
| 206 | + /* | |
| 207 | + * If for any reason at all we couldn't handle the fault, | |
| 208 | + * make sure we exit gracefully rather than endlessly redo | |
| 209 | + * the fault. | |
| 210 | + */ | |
| 211 | + fault = handle_mm_fault(vma, address, flags, regs); | |
| 212 | + | |
| 213 | + if (fault_signal_pending(fault, regs)) { | |
| 214 | + if (!user_mode(regs)) | |
| 215 | + no_context(regs, address); | |
| 216 | + return; | |
| 217 | + } | |
| 218 | + | |
| 219 | + if (unlikely(fault & VM_FAULT_RETRY)) { | |
| 220 | + flags |= FAULT_FLAG_TRIED; | |
| 221 | + | |
| 222 | + /* | |
| 223 | + * No need to mmap_read_unlock(mm) as we would | |
| 224 | + * have already released it in __lock_page_or_retry | |
| 225 | + * in mm/filemap.c. | |
| 226 | + */ | |
| 227 | + goto retry; | |
| 228 | + } | |
| 229 | + if (unlikely(fault & VM_FAULT_ERROR)) { | |
| 230 | + mmap_read_unlock(mm); | |
| 231 | + if (fault & VM_FAULT_OOM) { | |
| 232 | + do_out_of_memory(regs, address); | |
| 233 | + return; | |
| 234 | + } else if (fault & VM_FAULT_SIGSEGV) { | |
| 235 | + do_sigsegv(regs, write, address, si_code); | |
| 236 | + return; | |
| 237 | + } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) { | |
| 238 | + do_sigbus(regs, write, address, si_code); | |
| 239 | + return; | |
| 240 | + } | |
| 241 | + BUG(); | |
| 242 | + } | |
| 243 | + | |
| 244 | + mmap_read_unlock(mm); | |
| 245 | +} | |
| 246 | + | |
| 247 | +asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, | |
| 248 | + unsigned long write, unsigned long address) | |
| 249 | +{ | |
| 250 | + irqentry_state_t state = irqentry_enter(regs); | |
| 251 | + | |
| 252 | + /* Enable interrupt if enabled in parent context */ | |
| 253 | + if (likely(regs->csr_prmd & CSR_PRMD_PIE)) | |
| 254 | + local_irq_enable(); | |
| 255 | + | |
| 256 | + __do_page_fault(regs, write, address); | |
| 257 | + | |
| 258 | + local_irq_disable(); | |
| 259 | + | |
| 260 | + irqentry_exit(regs, state); | |
| 261 | +} | |
| @@ -0,0 +1,87 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <linux/fs.h> | |
| 7 | +#include <linux/mm.h> | |
| 8 | +#include <linux/hugetlb.h> | |
| 9 | +#include <linux/pagemap.h> | |
| 10 | +#include <linux/err.h> | |
| 11 | +#include <linux/sysctl.h> | |
| 12 | +#include <asm/mman.h> | |
| 13 | +#include <asm/tlb.h> | |
| 14 | +#include <asm/tlbflush.h> | |
| 15 | + | |
| 16 | +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, | |
| 17 | + unsigned long addr, unsigned long sz) | |
| 18 | +{ | |
| 19 | + pgd_t *pgd; | |
| 20 | + p4d_t *p4d; | |
| 21 | + pud_t *pud; | |
| 22 | + pte_t *pte = NULL; | |
| 23 | + | |
| 24 | + pgd = pgd_offset(mm, addr); | |
| 25 | + p4d = p4d_alloc(mm, pgd, addr); | |
| 26 | + pud = pud_alloc(mm, p4d, addr); | |
| 27 | + if (pud) | |
| 28 | + pte = (pte_t *)pmd_alloc(mm, pud, addr); | |
| 29 | + | |
| 30 | + return pte; | |
| 31 | +} | |
| 32 | + | |
| 33 | +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, | |
| 34 | + unsigned long sz) | |
| 35 | +{ | |
| 36 | + pgd_t *pgd; | |
| 37 | + p4d_t *p4d; | |
| 38 | + pud_t *pud; | |
| 39 | + pmd_t *pmd = NULL; | |
| 40 | + | |
| 41 | + pgd = pgd_offset(mm, addr); | |
| 42 | + if (pgd_present(*pgd)) { | |
| 43 | + p4d = p4d_offset(pgd, addr); | |
| 44 | + if (p4d_present(*p4d)) { | |
| 45 | + pud = pud_offset(p4d, addr); | |
| 46 | + if (pud_present(*pud)) | |
| 47 | + pmd = pmd_offset(pud, addr); | |
| 48 | + } | |
| 49 | + } | |
| 50 | + return (pte_t *) pmd; | |
| 51 | +} | |
| 52 | + | |
| 53 | +/* | |
| 54 | + * This function checks for proper alignment of input addr and len parameters. | |
| 55 | + */ | |
| 56 | +int is_aligned_hugepage_range(unsigned long addr, unsigned long len) | |
| 57 | +{ | |
| 58 | + if (len & ~HPAGE_MASK) | |
| 59 | + return -EINVAL; | |
| 60 | + if (addr & ~HPAGE_MASK) | |
| 61 | + return -EINVAL; | |
| 62 | + return 0; | |
| 63 | +} | |
| 64 | + | |
| 65 | +int pmd_huge(pmd_t pmd) | |
| 66 | +{ | |
| 67 | + return (pmd_val(pmd) & _PAGE_HUGE) != 0; | |
| 68 | +} | |
| 69 | + | |
| 70 | +int pud_huge(pud_t pud) | |
| 71 | +{ | |
| 72 | + return (pud_val(pud) & _PAGE_HUGE) != 0; | |
| 73 | +} | |
| 74 | + | |
| 75 | +uint64_t pmd_to_entrylo(unsigned long pmd_val) | |
| 76 | +{ | |
| 77 | + uint64_t val; | |
| 78 | + /* PMD as PTE. Must be huge page */ | |
| 79 | + if (!pmd_huge(__pmd(pmd_val))) | |
| 80 | + panic("%s", __func__); | |
| 81 | + | |
| 82 | + val = pmd_val ^ _PAGE_HUGE; | |
| 83 | + val |= ((val & _PAGE_HGLOBAL) >> | |
| 84 | + (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)); | |
| 85 | + | |
| 86 | + return val; | |
| 87 | +} | |
| @@ -0,0 +1,165 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/init.h> | |
| 6 | +#include <linux/export.h> | |
| 7 | +#include <linux/signal.h> | |
| 8 | +#include <linux/sched.h> | |
| 9 | +#include <linux/smp.h> | |
| 10 | +#include <linux/kernel.h> | |
| 11 | +#include <linux/errno.h> | |
| 12 | +#include <linux/string.h> | |
| 13 | +#include <linux/types.h> | |
| 14 | +#include <linux/pagemap.h> | |
| 15 | +#include <linux/memblock.h> | |
| 16 | +#include <linux/memremap.h> | |
| 17 | +#include <linux/mm.h> | |
| 18 | +#include <linux/mman.h> | |
| 19 | +#include <linux/highmem.h> | |
| 20 | +#include <linux/swap.h> | |
| 21 | +#include <linux/proc_fs.h> | |
| 22 | +#include <linux/pfn.h> | |
| 23 | +#include <linux/hardirq.h> | |
| 24 | +#include <linux/gfp.h> | |
| 25 | +#include <linux/initrd.h> | |
| 26 | +#include <linux/mmzone.h> | |
| 27 | + | |
| 28 | +#include <asm/asm-offsets.h> | |
| 29 | +#include <asm/bootinfo.h> | |
| 30 | +#include <asm/cpu.h> | |
| 31 | +#include <asm/dma.h> | |
| 32 | +#include <asm/mmu_context.h> | |
| 33 | +#include <asm/sections.h> | |
| 34 | +#include <asm/pgtable.h> | |
| 35 | +#include <asm/pgalloc.h> | |
| 36 | +#include <asm/tlb.h> | |
| 37 | + | |
| 38 | +/* | |
| 39 | + * We have up to 8 empty zeroed pages so we can map one of the right colour | |
| 40 | + * when needed. Since page is never written to after the initialization we | |
| 41 | + * don't have to care about aliases on other CPUs. | |
| 42 | + */ | |
| 43 | +unsigned long empty_zero_page, zero_page_mask; | |
| 44 | +EXPORT_SYMBOL_GPL(empty_zero_page); | |
| 45 | +EXPORT_SYMBOL(zero_page_mask); | |
| 46 | + | |
| 47 | +void setup_zero_pages(void) | |
| 48 | +{ | |
| 49 | + unsigned int order, i; | |
| 50 | + struct page *page; | |
| 51 | + | |
| 52 | + order = 0; | |
| 53 | + | |
| 54 | + empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); | |
| 55 | + if (!empty_zero_page) | |
| 56 | + panic("Oh boy, that early out of memory?"); | |
| 57 | + | |
| 58 | + page = virt_to_page((void *)empty_zero_page); | |
| 59 | + split_page(page, order); | |
| 60 | + for (i = 0; i < (1 << order); i++, page++) | |
| 61 | + mark_page_reserved(page); | |
| 62 | + | |
| 63 | + zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; | |
| 64 | +} | |
| 65 | + | |
| 66 | +void copy_user_highpage(struct page *to, struct page *from, | |
| 67 | + unsigned long vaddr, struct vm_area_struct *vma) | |
| 68 | +{ | |
| 69 | + void *vfrom, *vto; | |
| 70 | + | |
| 71 | + vto = kmap_atomic(to); | |
| 72 | + vfrom = kmap_atomic(from); | |
| 73 | + copy_page(vto, vfrom); | |
| 74 | + kunmap_atomic(vfrom); | |
| 75 | + kunmap_atomic(vto); | |
| 76 | + /* Make sure this page is cleared on other CPU's too before using it */ | |
| 77 | + smp_wmb(); | |
| 78 | +} | |
| 79 | + | |
| 80 | +int __ref page_is_ram(unsigned long pfn) | |
| 81 | +{ | |
| 82 | + unsigned long addr = PFN_PHYS(pfn); | |
| 83 | + | |
| 84 | + return memblock_is_memory(addr) && !memblock_is_reserved(addr); | |
| 85 | +} | |
| 86 | + | |
| 87 | +void __init paging_init(void) | |
| 88 | +{ | |
| 89 | + unsigned long max_zone_pfns[MAX_NR_ZONES]; | |
| 90 | + | |
| 91 | +#ifdef CONFIG_ZONE_DMA | |
| 92 | + max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; | |
| 93 | +#endif | |
| 94 | +#ifdef CONFIG_ZONE_DMA32 | |
| 95 | + max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; | |
| 96 | +#endif | |
| 97 | + max_zone_pfns[ZONE_NORMAL] = max_low_pfn; | |
| 98 | + | |
| 99 | + free_area_init(max_zone_pfns); | |
| 100 | +} | |
| 101 | + | |
| 102 | +void __init mem_init(void) | |
| 103 | +{ | |
| 104 | + max_mapnr = max_low_pfn; | |
| 105 | + high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); | |
| 106 | + | |
| 107 | + memblock_free_all(); | |
| 108 | + setup_zero_pages(); /* Setup zeroed pages. */ | |
| 109 | +} | |
| 110 | + | |
| 111 | +void __ref free_initmem(void) | |
| 112 | +{ | |
| 113 | + free_initmem_default(POISON_FREE_INITMEM); | |
| 114 | +} | |
| 115 | + | |
| 116 | +#ifdef CONFIG_MEMORY_HOTPLUG | |
| 117 | +int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) | |
| 118 | +{ | |
| 119 | + unsigned long start_pfn = start >> PAGE_SHIFT; | |
| 120 | + unsigned long nr_pages = size >> PAGE_SHIFT; | |
| 121 | + int ret; | |
| 122 | + | |
| 123 | + ret = __add_pages(nid, start_pfn, nr_pages, params); | |
| 124 | + | |
| 125 | + if (ret) | |
| 126 | + pr_warn("%s: Problem encountered in __add_pages() as ret=%d\n", | |
| 127 | + __func__, ret); | |
| 128 | + | |
| 129 | + return ret; | |
| 130 | +} | |
| 131 | + | |
| 132 | +#ifdef CONFIG_MEMORY_HOTREMOVE | |
| 133 | +void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) | |
| 134 | +{ | |
| 135 | + unsigned long start_pfn = start >> PAGE_SHIFT; | |
| 136 | + unsigned long nr_pages = size >> PAGE_SHIFT; | |
| 137 | + struct page *page = pfn_to_page(start_pfn); | |
| 138 | + | |
| 139 | + /* With altmap the first mapped page is offset from @start */ | |
| 140 | + if (altmap) | |
| 141 | + page += vmem_altmap_offset(altmap); | |
| 142 | + __remove_pages(start_pfn, nr_pages, altmap); | |
| 143 | +} | |
| 144 | +#endif | |
| 145 | +#endif | |
| 146 | + | |
| 147 | +/* | |
| 148 | + * Align swapper_pg_dir in to 64K, allows its address to be loaded | |
| 149 | + * with a single LUI instruction in the TLB handlers. If we used | |
| 150 | + * __aligned(64K), its size would get rounded up to the alignment | |
| 151 | + * size, and waste space. So we place it in its own section and align | |
| 152 | + * it in the linker script. | |
| 153 | + */ | |
| 154 | +pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); | |
| 155 | + | |
| 156 | +pgd_t invalid_pg_dir[_PTRS_PER_PGD] __page_aligned_bss; | |
| 157 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 158 | +pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss; | |
| 159 | +#endif | |
| 160 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 161 | +pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss; | |
| 162 | +EXPORT_SYMBOL_GPL(invalid_pmd_table); | |
| 163 | +#endif | |
| 164 | +pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; | |
| 165 | +EXPORT_SYMBOL(invalid_pte_table); | |
| @@ -0,0 +1,27 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | + | |
| 6 | +#include <asm/io.h> | |
| 7 | + | |
| 8 | +void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size) | |
| 9 | +{ | |
| 10 | + return ((void __iomem *)TO_CACHE(phys_addr)); | |
| 11 | +} | |
| 12 | + | |
| 13 | +void __init early_iounmap(void __iomem *addr, unsigned long size) | |
| 14 | +{ | |
| 15 | + | |
| 16 | +} | |
| 17 | + | |
| 18 | +void *early_memremap_ro(resource_size_t phys_addr, unsigned long size) | |
| 19 | +{ | |
| 20 | + return early_memremap(phys_addr, size); | |
| 21 | +} | |
| 22 | + | |
| 23 | +void *early_memremap_prot(resource_size_t phys_addr, unsigned long size, | |
| 24 | + unsigned long prot_val) | |
| 25 | +{ | |
| 26 | + return early_memremap(phys_addr, size); | |
| 27 | +} | |
| @@ -0,0 +1,10 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | + | |
| 3 | +#include <linux/uaccess.h> | |
| 4 | +#include <linux/kernel.h> | |
| 5 | + | |
| 6 | +bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) | |
| 7 | +{ | |
| 8 | + /* highest bit set means kernel space */ | |
| 9 | + return (unsigned long)unsafe_src >> (BITS_PER_LONG - 1); | |
| 10 | +} | |
| @@ -0,0 +1,125 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/compiler.h> | |
| 6 | +#include <linux/elf-randomize.h> | |
| 7 | +#include <linux/errno.h> | |
| 8 | +#include <linux/mm.h> | |
| 9 | +#include <linux/mman.h> | |
| 10 | +#include <linux/export.h> | |
| 11 | +#include <linux/personality.h> | |
| 12 | +#include <linux/random.h> | |
| 13 | +#include <linux/sched/signal.h> | |
| 14 | +#include <linux/sched/mm.h> | |
| 15 | + | |
| 16 | +unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */ | |
| 17 | +EXPORT_SYMBOL(shm_align_mask); | |
| 18 | + | |
| 19 | +#define COLOUR_ALIGN(addr, pgoff) \ | |
| 20 | + ((((addr) + shm_align_mask) & ~shm_align_mask) + \ | |
| 21 | + (((pgoff) << PAGE_SHIFT) & shm_align_mask)) | |
| 22 | + | |
| 23 | +enum mmap_allocation_direction {UP, DOWN}; | |
| 24 | + | |
| 25 | +static unsigned long arch_get_unmapped_area_common(struct file *filp, | |
| 26 | + unsigned long addr0, unsigned long len, unsigned long pgoff, | |
| 27 | + unsigned long flags, enum mmap_allocation_direction dir) | |
| 28 | +{ | |
| 29 | + struct mm_struct *mm = current->mm; | |
| 30 | + struct vm_area_struct *vma; | |
| 31 | + unsigned long addr = addr0; | |
| 32 | + int do_color_align; | |
| 33 | + struct vm_unmapped_area_info info; | |
| 34 | + | |
| 35 | + if (unlikely(len > TASK_SIZE)) | |
| 36 | + return -ENOMEM; | |
| 37 | + | |
| 38 | + if (flags & MAP_FIXED) { | |
| 39 | + /* Even MAP_FIXED mappings must reside within TASK_SIZE */ | |
| 40 | + if (TASK_SIZE - len < addr) | |
| 41 | + return -EINVAL; | |
| 42 | + | |
| 43 | + /* | |
| 44 | + * We do not accept a shared mapping if it would violate | |
| 45 | + * cache aliasing constraints. | |
| 46 | + */ | |
| 47 | + if ((flags & MAP_SHARED) && | |
| 48 | + ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask)) | |
| 49 | + return -EINVAL; | |
| 50 | + return addr; | |
| 51 | + } | |
| 52 | + | |
| 53 | + do_color_align = 0; | |
| 54 | + if (filp || (flags & MAP_SHARED)) | |
| 55 | + do_color_align = 1; | |
| 56 | + | |
| 57 | + /* requesting a specific address */ | |
| 58 | + if (addr) { | |
| 59 | + if (do_color_align) | |
| 60 | + addr = COLOUR_ALIGN(addr, pgoff); | |
| 61 | + else | |
| 62 | + addr = PAGE_ALIGN(addr); | |
| 63 | + | |
| 64 | + vma = find_vma(mm, addr); | |
| 65 | + if (TASK_SIZE - len >= addr && | |
| 66 | + (!vma || addr + len <= vm_start_gap(vma))) | |
| 67 | + return addr; | |
| 68 | + } | |
| 69 | + | |
| 70 | + info.length = len; | |
| 71 | + info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0; | |
| 72 | + info.align_offset = pgoff << PAGE_SHIFT; | |
| 73 | + | |
| 74 | + if (dir == DOWN) { | |
| 75 | + info.flags = VM_UNMAPPED_AREA_TOPDOWN; | |
| 76 | + info.low_limit = PAGE_SIZE; | |
| 77 | + info.high_limit = mm->mmap_base; | |
| 78 | + addr = vm_unmapped_area(&info); | |
| 79 | + | |
| 80 | + if (!(addr & ~PAGE_MASK)) | |
| 81 | + return addr; | |
| 82 | + | |
| 83 | + /* | |
| 84 | + * A failed mmap() very likely causes application failure, | |
| 85 | + * so fall back to the bottom-up function here. This scenario | |
| 86 | + * can happen with large stack limits and large mmap() | |
| 87 | + * allocations. | |
| 88 | + */ | |
| 89 | + } | |
| 90 | + | |
| 91 | + info.flags = 0; | |
| 92 | + info.low_limit = mm->mmap_base; | |
| 93 | + info.high_limit = TASK_SIZE; | |
| 94 | + return vm_unmapped_area(&info); | |
| 95 | +} | |
| 96 | + | |
| 97 | +unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0, | |
| 98 | + unsigned long len, unsigned long pgoff, unsigned long flags) | |
| 99 | +{ | |
| 100 | + return arch_get_unmapped_area_common(filp, | |
| 101 | + addr0, len, pgoff, flags, UP); | |
| 102 | +} | |
| 103 | + | |
| 104 | +/* | |
| 105 | + * There is no need to export this but sched.h declares the function as | |
| 106 | + * extern so making it static here results in an error. | |
| 107 | + */ | |
| 108 | +unsigned long arch_get_unmapped_area_topdown(struct file *filp, | |
| 109 | + unsigned long addr0, unsigned long len, unsigned long pgoff, | |
| 110 | + unsigned long flags) | |
| 111 | +{ | |
| 112 | + return arch_get_unmapped_area_common(filp, | |
| 113 | + addr0, len, pgoff, flags, DOWN); | |
| 114 | +} | |
| 115 | + | |
| 116 | +int __virt_addr_valid(volatile void *kaddr) | |
| 117 | +{ | |
| 118 | + unsigned long vaddr = (unsigned long)kaddr; | |
| 119 | + | |
| 120 | + if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base)) | |
| 121 | + return 0; | |
| 122 | + | |
| 123 | + return pfn_valid(PFN_DOWN(virt_to_phys(kaddr))); | |
| 124 | +} | |
| 125 | +EXPORT_SYMBOL_GPL(__virt_addr_valid); | |
| @@ -0,0 +1,84 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/linkage.h> | |
| 6 | +#include <asm/asm.h> | |
| 7 | +#include <asm/export.h> | |
| 8 | +#include <asm/page.h> | |
| 9 | +#include <asm/regdef.h> | |
| 10 | + | |
| 11 | + .align 5 | |
| 12 | +SYM_FUNC_START(clear_page) | |
| 13 | + lu12i.w t0, 1 << (PAGE_SHIFT - 12) | |
| 14 | + add.d t0, t0, a0 | |
| 15 | +1: | |
| 16 | + st.d zero, a0, 0 | |
| 17 | + st.d zero, a0, 8 | |
| 18 | + st.d zero, a0, 16 | |
| 19 | + st.d zero, a0, 24 | |
| 20 | + st.d zero, a0, 32 | |
| 21 | + st.d zero, a0, 40 | |
| 22 | + st.d zero, a0, 48 | |
| 23 | + st.d zero, a0, 56 | |
| 24 | + addi.d a0, a0, 128 | |
| 25 | + st.d zero, a0, -64 | |
| 26 | + st.d zero, a0, -56 | |
| 27 | + st.d zero, a0, -48 | |
| 28 | + st.d zero, a0, -40 | |
| 29 | + st.d zero, a0, -32 | |
| 30 | + st.d zero, a0, -24 | |
| 31 | + st.d zero, a0, -16 | |
| 32 | + st.d zero, a0, -8 | |
| 33 | + bne t0, a0, 1b | |
| 34 | + | |
| 35 | + jirl $r0, ra, 0 | |
| 36 | +SYM_FUNC_END(clear_page) | |
| 37 | +EXPORT_SYMBOL(clear_page) | |
| 38 | + | |
| 39 | +.align 5 | |
| 40 | +SYM_FUNC_START(copy_page) | |
| 41 | + lu12i.w t8, 1 << (PAGE_SHIFT - 12) | |
| 42 | + add.d t8, t8, a0 | |
| 43 | +1: | |
| 44 | + ld.d t0, a1, 0 | |
| 45 | + ld.d t1, a1, 8 | |
| 46 | + ld.d t2, a1, 16 | |
| 47 | + ld.d t3, a1, 24 | |
| 48 | + ld.d t4, a1, 32 | |
| 49 | + ld.d t5, a1, 40 | |
| 50 | + ld.d t6, a1, 48 | |
| 51 | + ld.d t7, a1, 56 | |
| 52 | + | |
| 53 | + st.d t0, a0, 0 | |
| 54 | + st.d t1, a0, 8 | |
| 55 | + ld.d t0, a1, 64 | |
| 56 | + ld.d t1, a1, 72 | |
| 57 | + st.d t2, a0, 16 | |
| 58 | + st.d t3, a0, 24 | |
| 59 | + ld.d t2, a1, 80 | |
| 60 | + ld.d t3, a1, 88 | |
| 61 | + st.d t4, a0, 32 | |
| 62 | + st.d t5, a0, 40 | |
| 63 | + ld.d t4, a1, 96 | |
| 64 | + ld.d t5, a1, 104 | |
| 65 | + st.d t6, a0, 48 | |
| 66 | + st.d t7, a0, 56 | |
| 67 | + ld.d t6, a1, 112 | |
| 68 | + ld.d t7, a1, 120 | |
| 69 | + addi.d a0, a0, 128 | |
| 70 | + addi.d a1, a1, 128 | |
| 71 | + | |
| 72 | + st.d t0, a0, -64 | |
| 73 | + st.d t1, a0, -56 | |
| 74 | + st.d t2, a0, -48 | |
| 75 | + st.d t3, a0, -40 | |
| 76 | + st.d t4, a0, -32 | |
| 77 | + st.d t5, a0, -24 | |
| 78 | + st.d t6, a0, -16 | |
| 79 | + st.d t7, a0, -8 | |
| 80 | + | |
| 81 | + bne t8, a0, 1b | |
| 82 | + jirl $r0, ra, 0 | |
| 83 | +SYM_FUNC_END(copy_page) | |
| 84 | +EXPORT_SYMBOL(copy_page) | |
| @@ -0,0 +1,130 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/init.h> | |
| 6 | +#include <linux/export.h> | |
| 7 | +#include <linux/mm.h> | |
| 8 | +#include <asm/pgalloc.h> | |
| 9 | +#include <asm/pgtable.h> | |
| 10 | +#include <asm/tlbflush.h> | |
| 11 | + | |
| 12 | +pgd_t *pgd_alloc(struct mm_struct *mm) | |
| 13 | +{ | |
| 14 | + pgd_t *ret, *init; | |
| 15 | + | |
| 16 | + ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER); | |
| 17 | + if (ret) { | |
| 18 | + init = pgd_offset(&init_mm, 0UL); | |
| 19 | + pgd_init((unsigned long)ret); | |
| 20 | + memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, | |
| 21 | + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); | |
| 22 | + } | |
| 23 | + | |
| 24 | + return ret; | |
| 25 | +} | |
| 26 | +EXPORT_SYMBOL_GPL(pgd_alloc); | |
| 27 | + | |
| 28 | +void pgd_init(unsigned long page) | |
| 29 | +{ | |
| 30 | + unsigned long *p, *end; | |
| 31 | + unsigned long entry; | |
| 32 | + | |
| 33 | +#if !defined(__PAGETABLE_PUD_FOLDED) | |
| 34 | + entry = (unsigned long)invalid_pud_table; | |
| 35 | +#elif !defined(__PAGETABLE_PMD_FOLDED) | |
| 36 | + entry = (unsigned long)invalid_pmd_table; | |
| 37 | +#else | |
| 38 | + entry = (unsigned long)invalid_pte_table; | |
| 39 | +#endif | |
| 40 | + | |
| 41 | + p = (unsigned long *) page; | |
| 42 | + end = p + PTRS_PER_PGD; | |
| 43 | + | |
| 44 | + do { | |
| 45 | + p[0] = entry; | |
| 46 | + p[1] = entry; | |
| 47 | + p[2] = entry; | |
| 48 | + p[3] = entry; | |
| 49 | + p[4] = entry; | |
| 50 | + p += 8; | |
| 51 | + p[-3] = entry; | |
| 52 | + p[-2] = entry; | |
| 53 | + p[-1] = entry; | |
| 54 | + } while (p != end); | |
| 55 | +} | |
| 56 | +EXPORT_SYMBOL_GPL(pgd_init); | |
| 57 | + | |
| 58 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 59 | +void pmd_init(unsigned long addr, unsigned long pagetable) | |
| 60 | +{ | |
| 61 | + unsigned long *p, *end; | |
| 62 | + | |
| 63 | + p = (unsigned long *) addr; | |
| 64 | + end = p + PTRS_PER_PMD; | |
| 65 | + | |
| 66 | + do { | |
| 67 | + p[0] = pagetable; | |
| 68 | + p[1] = pagetable; | |
| 69 | + p[2] = pagetable; | |
| 70 | + p[3] = pagetable; | |
| 71 | + p[4] = pagetable; | |
| 72 | + p += 8; | |
| 73 | + p[-3] = pagetable; | |
| 74 | + p[-2] = pagetable; | |
| 75 | + p[-1] = pagetable; | |
| 76 | + } while (p != end); | |
| 77 | +} | |
| 78 | +EXPORT_SYMBOL_GPL(pmd_init); | |
| 79 | +#endif | |
| 80 | + | |
| 81 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 82 | +void pud_init(unsigned long addr, unsigned long pagetable) | |
| 83 | +{ | |
| 84 | + unsigned long *p, *end; | |
| 85 | + | |
| 86 | + p = (unsigned long *)addr; | |
| 87 | + end = p + PTRS_PER_PUD; | |
| 88 | + | |
| 89 | + do { | |
| 90 | + p[0] = pagetable; | |
| 91 | + p[1] = pagetable; | |
| 92 | + p[2] = pagetable; | |
| 93 | + p[3] = pagetable; | |
| 94 | + p[4] = pagetable; | |
| 95 | + p += 8; | |
| 96 | + p[-3] = pagetable; | |
| 97 | + p[-2] = pagetable; | |
| 98 | + p[-1] = pagetable; | |
| 99 | + } while (p != end); | |
| 100 | +} | |
| 101 | +#endif | |
| 102 | + | |
| 103 | +pmd_t mk_pmd(struct page *page, pgprot_t prot) | |
| 104 | +{ | |
| 105 | + pmd_t pmd; | |
| 106 | + | |
| 107 | + pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); | |
| 108 | + | |
| 109 | + return pmd; | |
| 110 | +} | |
| 111 | + | |
| 112 | +void set_pmd_at(struct mm_struct *mm, unsigned long addr, | |
| 113 | + pmd_t *pmdp, pmd_t pmd) | |
| 114 | +{ | |
| 115 | + *pmdp = pmd; | |
| 116 | + flush_tlb_all(); | |
| 117 | +} | |
| 118 | + | |
| 119 | +void __init pagetable_init(void) | |
| 120 | +{ | |
| 121 | + /* Initialize the entire pgd. */ | |
| 122 | + pgd_init((unsigned long)swapper_pg_dir); | |
| 123 | + pgd_init((unsigned long)invalid_pg_dir); | |
| 124 | +#ifndef __PAGETABLE_PUD_FOLDED | |
| 125 | + pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table); | |
| 126 | +#endif | |
| 127 | +#ifndef __PAGETABLE_PMD_FOLDED | |
| 128 | + pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table); | |
| 129 | +#endif | |
| 130 | +} | |
| @@ -0,0 +1,282 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0 | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <linux/init.h> | |
| 6 | +#include <linux/sched.h> | |
| 7 | +#include <linux/smp.h> | |
| 8 | +#include <linux/mm.h> | |
| 9 | +#include <linux/hugetlb.h> | |
| 10 | +#include <linux/export.h> | |
| 11 | + | |
| 12 | +#include <asm/cpu.h> | |
| 13 | +#include <asm/bootinfo.h> | |
| 14 | +#include <asm/mmu_context.h> | |
| 15 | +#include <asm/pgtable.h> | |
| 16 | +#include <asm/tlb.h> | |
| 17 | + | |
| 18 | +void local_flush_tlb_all(void) | |
| 19 | +{ | |
| 20 | + invtlb_all(INVTLB_CURRENT_ALL, 0, 0); | |
| 21 | +} | |
| 22 | +EXPORT_SYMBOL(local_flush_tlb_all); | |
| 23 | + | |
| 24 | +void local_flush_tlb_user(void) | |
| 25 | +{ | |
| 26 | + invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0); | |
| 27 | +} | |
| 28 | +EXPORT_SYMBOL(local_flush_tlb_user); | |
| 29 | + | |
| 30 | +void local_flush_tlb_kernel(void) | |
| 31 | +{ | |
| 32 | + invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0); | |
| 33 | +} | |
| 34 | +EXPORT_SYMBOL(local_flush_tlb_kernel); | |
| 35 | + | |
| 36 | +/* | |
| 37 | + * All entries common to a mm share an asid. To effectively flush | |
| 38 | + * these entries, we just bump the asid. | |
| 39 | + */ | |
| 40 | +void local_flush_tlb_mm(struct mm_struct *mm) | |
| 41 | +{ | |
| 42 | + int cpu; | |
| 43 | + | |
| 44 | + preempt_disable(); | |
| 45 | + | |
| 46 | + cpu = smp_processor_id(); | |
| 47 | + | |
| 48 | + if (asid_valid(mm, cpu)) | |
| 49 | + drop_mmu_context(mm, cpu); | |
| 50 | + else | |
| 51 | + cpumask_clear_cpu(cpu, mm_cpumask(mm)); | |
| 52 | + | |
| 53 | + preempt_enable(); | |
| 54 | +} | |
| 55 | + | |
| 56 | +void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, | |
| 57 | + unsigned long end) | |
| 58 | +{ | |
| 59 | + struct mm_struct *mm = vma->vm_mm; | |
| 60 | + int cpu = smp_processor_id(); | |
| 61 | + | |
| 62 | + if (asid_valid(mm, cpu)) { | |
| 63 | + unsigned long size, flags; | |
| 64 | + | |
| 65 | + local_irq_save(flags); | |
| 66 | + start = round_down(start, PAGE_SIZE << 1); | |
| 67 | + end = round_up(end, PAGE_SIZE << 1); | |
| 68 | + size = (end - start) >> (PAGE_SHIFT + 1); | |
| 69 | + if (size <= (current_cpu_data.tlbsizestlbsets ? | |
| 70 | + current_cpu_data.tlbsize / 8 : | |
| 71 | + current_cpu_data.tlbsize / 2)) { | |
| 72 | + int asid = cpu_asid(cpu, mm); | |
| 73 | + | |
| 74 | + while (start < end) { | |
| 75 | + invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start); | |
| 76 | + start += (PAGE_SIZE << 1); | |
| 77 | + } | |
| 78 | + } else { | |
| 79 | + drop_mmu_context(mm, cpu); | |
| 80 | + } | |
| 81 | + local_irq_restore(flags); | |
| 82 | + } else { | |
| 83 | + cpumask_clear_cpu(cpu, mm_cpumask(mm)); | |
| 84 | + } | |
| 85 | +} | |
| 86 | + | |
| 87 | +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) | |
| 88 | +{ | |
| 89 | + unsigned long size, flags; | |
| 90 | + | |
| 91 | + local_irq_save(flags); | |
| 92 | + size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; | |
| 93 | + size = (size + 1) >> 1; | |
| 94 | + if (size <= (current_cpu_data.tlbsizestlbsets ? | |
| 95 | + current_cpu_data.tlbsize / 8 : | |
| 96 | + current_cpu_data.tlbsize / 2)) { | |
| 97 | + | |
| 98 | + start &= (PAGE_MASK << 1); | |
| 99 | + end += ((PAGE_SIZE << 1) - 1); | |
| 100 | + end &= (PAGE_MASK << 1); | |
| 101 | + | |
| 102 | + while (start < end) { | |
| 103 | + invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start); | |
| 104 | + start += (PAGE_SIZE << 1); | |
| 105 | + } | |
| 106 | + } else { | |
| 107 | + local_flush_tlb_kernel(); | |
| 108 | + } | |
| 109 | + local_irq_restore(flags); | |
| 110 | +} | |
| 111 | + | |
| 112 | +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) | |
| 113 | +{ | |
| 114 | + int cpu = smp_processor_id(); | |
| 115 | + | |
| 116 | + if (asid_valid(vma->vm_mm, cpu)) { | |
| 117 | + int newpid; | |
| 118 | + | |
| 119 | + newpid = cpu_asid(cpu, vma->vm_mm); | |
| 120 | + page &= (PAGE_MASK << 1); | |
| 121 | + invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page); | |
| 122 | + } else { | |
| 123 | + cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm)); | |
| 124 | + } | |
| 125 | +} | |
| 126 | + | |
| 127 | +/* | |
| 128 | + * This one is only used for pages with the global bit set so we don't care | |
| 129 | + * much about the ASID. | |
| 130 | + */ | |
| 131 | +void local_flush_tlb_one(unsigned long page) | |
| 132 | +{ | |
| 133 | + page &= (PAGE_MASK << 1); | |
| 134 | + invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page); | |
| 135 | +} | |
| 136 | + | |
| 137 | +static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) | |
| 138 | +{ | |
| 139 | +#ifdef CONFIG_HUGETLB_PAGE | |
| 140 | + int idx; | |
| 141 | + unsigned long lo; | |
| 142 | + unsigned long flags; | |
| 143 | + | |
| 144 | + local_irq_save(flags); | |
| 145 | + | |
| 146 | + address &= (PAGE_MASK << 1); | |
| 147 | + write_csr_entryhi(address); | |
| 148 | + tlb_probe(); | |
| 149 | + idx = read_csr_tlbidx(); | |
| 150 | + write_csr_pagesize(PS_HUGE_SIZE); | |
| 151 | + lo = pmd_to_entrylo(pte_val(*ptep)); | |
| 152 | + write_csr_entrylo0(lo); | |
| 153 | + write_csr_entrylo1(lo + (HPAGE_SIZE >> 1)); | |
| 154 | + | |
| 155 | + if (idx < 0) | |
| 156 | + tlb_write_random(); | |
| 157 | + else | |
| 158 | + tlb_write_indexed(); | |
| 159 | + write_csr_pagesize(PS_DEFAULT_SIZE); | |
| 160 | + | |
| 161 | + local_irq_restore(flags); | |
| 162 | +#endif | |
| 163 | +} | |
| 164 | + | |
| 165 | +void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) | |
| 166 | +{ | |
| 167 | + int idx; | |
| 168 | + unsigned long flags; | |
| 169 | + | |
| 170 | + /* | |
| 171 | + * Handle debugger faulting in for debugee. | |
| 172 | + */ | |
| 173 | + if (current->active_mm != vma->vm_mm) | |
| 174 | + return; | |
| 175 | + | |
| 176 | + if (pte_val(*ptep) & _PAGE_HUGE) { | |
| 177 | + __update_hugetlb(vma, address, ptep); | |
| 178 | + return; | |
| 179 | + } | |
| 180 | + | |
| 181 | + local_irq_save(flags); | |
| 182 | + | |
| 183 | + if ((unsigned long)ptep & sizeof(pte_t)) | |
| 184 | + ptep--; | |
| 185 | + | |
| 186 | + address &= (PAGE_MASK << 1); | |
| 187 | + write_csr_entryhi(address); | |
| 188 | + tlb_probe(); | |
| 189 | + idx = read_csr_tlbidx(); | |
| 190 | + write_csr_pagesize(PS_DEFAULT_SIZE); | |
| 191 | + write_csr_entrylo0(pte_val(*ptep++)); | |
| 192 | + write_csr_entrylo1(pte_val(*ptep)); | |
| 193 | + if (idx < 0) | |
| 194 | + tlb_write_random(); | |
| 195 | + else | |
| 196 | + tlb_write_indexed(); | |
| 197 | + | |
| 198 | + local_irq_restore(flags); | |
| 199 | +} | |
| 200 | + | |
| 201 | +static void setup_ptwalker(void) | |
| 202 | +{ | |
| 203 | + unsigned long pwctl0, pwctl1; | |
| 204 | + unsigned long pgd_i = 0, pgd_w = 0; | |
| 205 | + unsigned long pud_i = 0, pud_w = 0; | |
| 206 | + unsigned long pmd_i = 0, pmd_w = 0; | |
| 207 | + unsigned long pte_i = 0, pte_w = 0; | |
| 208 | + | |
| 209 | + pgd_i = PGDIR_SHIFT; | |
| 210 | + pgd_w = PAGE_SHIFT - 3; | |
| 211 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 212 | + pud_i = PUD_SHIFT; | |
| 213 | + pud_w = PAGE_SHIFT - 3; | |
| 214 | +#endif | |
| 215 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 216 | + pmd_i = PMD_SHIFT; | |
| 217 | + pmd_w = PAGE_SHIFT - 3; | |
| 218 | +#endif | |
| 219 | + pte_i = PAGE_SHIFT; | |
| 220 | + pte_w = PAGE_SHIFT - 3; | |
| 221 | + | |
| 222 | + pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25; | |
| 223 | + pwctl1 = pgd_i | pgd_w << 6; | |
| 224 | + | |
| 225 | + csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0); | |
| 226 | + csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1); | |
| 227 | + csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH); | |
| 228 | + csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL); | |
| 229 | + csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID); | |
| 230 | +} | |
| 231 | + | |
| 232 | +static void output_pgtable_bits_defines(void) | |
| 233 | +{ | |
| 234 | +#define pr_define(fmt, ...) \ | |
| 235 | + pr_debug("#define " fmt, ##__VA_ARGS__) | |
| 236 | + | |
| 237 | + pr_debug("#include <asm/asm.h>\n"); | |
| 238 | + pr_debug("#include <asm/regdef.h>\n"); | |
| 239 | + pr_debug("\n"); | |
| 240 | + | |
| 241 | + pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); | |
| 242 | + pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); | |
| 243 | + pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); | |
| 244 | + pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); | |
| 245 | + pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); | |
| 246 | + pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); | |
| 247 | + pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); | |
| 248 | + pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); | |
| 249 | + pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); | |
| 250 | + pr_debug("\n"); | |
| 251 | +} | |
| 252 | + | |
| 253 | +void setup_tlb_handler(void) | |
| 254 | +{ | |
| 255 | + static int run_once = 0; | |
| 256 | + | |
| 257 | + setup_ptwalker(); | |
| 258 | + output_pgtable_bits_defines(); | |
| 259 | + | |
| 260 | + /* The tlb handlers are generated only once */ | |
| 261 | + if (!run_once) { | |
| 262 | + memcpy((void *)tlbrentry, handle_tlb_refill, 0x80); | |
| 263 | + local_flush_icache_range(tlbrentry, tlbrentry + 0x80); | |
| 264 | + set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE); | |
| 265 | + set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE); | |
| 266 | + set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE); | |
| 267 | + set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE); | |
| 268 | + set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE); | |
| 269 | + set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE); | |
| 270 | + set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE); | |
| 271 | + run_once++; | |
| 272 | + } | |
| 273 | +} | |
| 274 | + | |
| 275 | +void tlb_init(void) | |
| 276 | +{ | |
| 277 | + write_csr_pagesize(PS_DEFAULT_SIZE); | |
| 278 | + write_csr_stlbpgsize(PS_DEFAULT_SIZE); | |
| 279 | + write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE); | |
| 280 | + setup_tlb_handler(); | |
| 281 | + local_flush_tlb_all(); | |
| 282 | +} | |
| @@ -0,0 +1,477 @@ | ||
| 1 | +/* SPDX-License-Identifier: GPL-2.0 */ | |
| 2 | +/* | |
| 3 | + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited | |
| 4 | + */ | |
| 5 | +#include <asm/asm.h> | |
| 6 | +#include <asm/export.h> | |
| 7 | +#include <asm/loongarch.h> | |
| 8 | +#include <asm/page.h> | |
| 9 | +#include <asm/pgtable.h> | |
| 10 | +#include <asm/regdef.h> | |
| 11 | +#include <asm/stackframe.h> | |
| 12 | + | |
| 13 | + .macro tlb_do_page_fault, write | |
| 14 | + SYM_FUNC_START(tlb_do_page_fault_\write) | |
| 15 | + SAVE_ALL | |
| 16 | + csrrd a2, LOONGARCH_CSR_BADV | |
| 17 | + move a0, sp | |
| 18 | + REG_S a2, sp, PT_BVADDR | |
| 19 | + li.w a1, \write | |
| 20 | + la.abs t0, do_page_fault | |
| 21 | + jirl ra, t0, 0 | |
| 22 | + RESTORE_ALL_AND_RET | |
| 23 | + SYM_FUNC_END(tlb_do_page_fault_\write) | |
| 24 | + .endm | |
| 25 | + | |
| 26 | + tlb_do_page_fault 0 | |
| 27 | + tlb_do_page_fault 1 | |
| 28 | + | |
| 29 | +SYM_FUNC_START(handle_tlb_protect) | |
| 30 | + BACKUP_T0T1 | |
| 31 | + SAVE_ALL | |
| 32 | + move a0, sp | |
| 33 | + move a1, zero | |
| 34 | + csrrd a2, LOONGARCH_CSR_BADV | |
| 35 | + REG_S a2, sp, PT_BVADDR | |
| 36 | + la.abs t0, do_page_fault | |
| 37 | + jirl ra, t0, 0 | |
| 38 | + RESTORE_ALL_AND_RET | |
| 39 | +SYM_FUNC_END(handle_tlb_protect) | |
| 40 | + | |
| 41 | +SYM_FUNC_START(handle_tlb_load) | |
| 42 | + csrwr t0, EXCEPTION_KS0 | |
| 43 | + csrwr t1, EXCEPTION_KS1 | |
| 44 | + csrwr ra, EXCEPTION_KS2 | |
| 45 | + | |
| 46 | + /* | |
| 47 | + * The vmalloc handling is not in the hotpath. | |
| 48 | + */ | |
| 49 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 50 | + blt t0, $r0, vmalloc_load | |
| 51 | + csrrd t1, LOONGARCH_CSR_PGDL | |
| 52 | + | |
| 53 | +vmalloc_done_load: | |
| 54 | + /* Get PGD offset in bytes */ | |
| 55 | + srli.d t0, t0, PGDIR_SHIFT | |
| 56 | + andi t0, t0, (PTRS_PER_PGD - 1) | |
| 57 | + slli.d t0, t0, 3 | |
| 58 | + add.d t1, t1, t0 | |
| 59 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 60 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 61 | + ld.d t1, t1, 0 | |
| 62 | + srli.d t0, t0, PUD_SHIFT | |
| 63 | + andi t0, t0, (PTRS_PER_PUD - 1) | |
| 64 | + slli.d t0, t0, 3 | |
| 65 | + add.d t1, t1, t0 | |
| 66 | +#endif | |
| 67 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 68 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 69 | + ld.d t1, t1, 0 | |
| 70 | + srli.d t0, t0, PMD_SHIFT | |
| 71 | + andi t0, t0, (PTRS_PER_PMD - 1) | |
| 72 | + slli.d t0, t0, 3 | |
| 73 | + add.d t1, t1, t0 | |
| 74 | +#endif | |
| 75 | + ld.d ra, t1, 0 | |
| 76 | + | |
| 77 | + /* | |
| 78 | + * For huge tlb entries, pmde doesn't contain an address but | |
| 79 | + * instead contains the tlb pte. Check the PAGE_HUGE bit and | |
| 80 | + * see if we need to jump to huge tlb processing. | |
| 81 | + */ | |
| 82 | + andi t0, ra, _PAGE_HUGE | |
| 83 | + bne t0, $r0, tlb_huge_update_load | |
| 84 | + | |
| 85 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 86 | + srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | |
| 87 | + andi t0, t0, (PTRS_PER_PTE - 1) | |
| 88 | + slli.d t0, t0, _PTE_T_LOG2 | |
| 89 | + add.d t1, ra, t0 | |
| 90 | + | |
| 91 | + ld.d t0, t1, 0 | |
| 92 | + tlbsrch | |
| 93 | + | |
| 94 | + srli.d ra, t0, _PAGE_PRESENT_SHIFT | |
| 95 | + andi ra, ra, 1 | |
| 96 | + beq ra, $r0, nopage_tlb_load | |
| 97 | + | |
| 98 | + ori t0, t0, _PAGE_VALID | |
| 99 | + st.d t0, t1, 0 | |
| 100 | + ori t1, t1, 8 | |
| 101 | + xori t1, t1, 8 | |
| 102 | + ld.d t0, t1, 0 | |
| 103 | + ld.d t1, t1, 8 | |
| 104 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 105 | + csrwr t1, LOONGARCH_CSR_TLBELO1 | |
| 106 | + tlbwr | |
| 107 | +leave_load: | |
| 108 | + csrrd t0, EXCEPTION_KS0 | |
| 109 | + csrrd t1, EXCEPTION_KS1 | |
| 110 | + csrrd ra, EXCEPTION_KS2 | |
| 111 | + ertn | |
| 112 | +#ifdef CONFIG_64BIT | |
| 113 | +vmalloc_load: | |
| 114 | + la.abs t1, swapper_pg_dir | |
| 115 | + b vmalloc_done_load | |
| 116 | +#endif | |
| 117 | + | |
| 118 | + /* | |
| 119 | + * This is the entry point when build_tlbchange_handler_head | |
| 120 | + * spots a huge page. | |
| 121 | + */ | |
| 122 | +tlb_huge_update_load: | |
| 123 | + ld.d t0, t1, 0 | |
| 124 | + srli.d ra, t0, _PAGE_PRESENT_SHIFT | |
| 125 | + andi ra, ra, 1 | |
| 126 | + beq ra, $r0, nopage_tlb_load | |
| 127 | + tlbsrch | |
| 128 | + | |
| 129 | + ori t0, t0, _PAGE_VALID | |
| 130 | + st.d t0, t1, 0 | |
| 131 | + addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) | |
| 132 | + addi.d ra, t1, 0 | |
| 133 | + csrxchg ra, t1, LOONGARCH_CSR_TLBIDX | |
| 134 | + tlbwr | |
| 135 | + | |
| 136 | + csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX | |
| 137 | + | |
| 138 | + /* | |
| 139 | + * A huge PTE describes an area the size of the | |
| 140 | + * configured huge page size. This is twice the | |
| 141 | + * of the large TLB entry size we intend to use. | |
| 142 | + * A TLB entry half the size of the configured | |
| 143 | + * huge page size is configured into entrylo0 | |
| 144 | + * and entrylo1 to cover the contiguous huge PTE | |
| 145 | + * address space. | |
| 146 | + */ | |
| 147 | + /* Huge page: Move Global bit */ | |
| 148 | + xori t0, t0, _PAGE_HUGE | |
| 149 | + lu12i.w t1, _PAGE_HGLOBAL >> 12 | |
| 150 | + and t1, t0, t1 | |
| 151 | + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | |
| 152 | + or t0, t0, t1 | |
| 153 | + | |
| 154 | + addi.d ra, t0, 0 | |
| 155 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 156 | + addi.d t0, ra, 0 | |
| 157 | + | |
| 158 | + /* Convert to entrylo1 */ | |
| 159 | + addi.d t1, $r0, 1 | |
| 160 | + slli.d t1, t1, (HPAGE_SHIFT - 1) | |
| 161 | + add.d t0, t0, t1 | |
| 162 | + csrwr t0, LOONGARCH_CSR_TLBELO1 | |
| 163 | + | |
| 164 | + /* Set huge page tlb entry size */ | |
| 165 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 166 | + addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 167 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 168 | + | |
| 169 | + tlbfill | |
| 170 | + | |
| 171 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 172 | + addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 173 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 174 | + | |
| 175 | +nopage_tlb_load: | |
| 176 | + csrrd ra, EXCEPTION_KS2 | |
| 177 | + la.abs t0, tlb_do_page_fault_0 | |
| 178 | + jirl $r0, t0, 0 | |
| 179 | +SYM_FUNC_END(handle_tlb_load) | |
| 180 | + | |
| 181 | +SYM_FUNC_START(handle_tlb_store) | |
| 182 | + csrwr t0, EXCEPTION_KS0 | |
| 183 | + csrwr t1, EXCEPTION_KS1 | |
| 184 | + csrwr ra, EXCEPTION_KS2 | |
| 185 | + | |
| 186 | + /* | |
| 187 | + * The vmalloc handling is not in the hotpath. | |
| 188 | + */ | |
| 189 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 190 | + blt t0, $r0, vmalloc_store | |
| 191 | + csrrd t1, LOONGARCH_CSR_PGDL | |
| 192 | + | |
| 193 | +vmalloc_done_store: | |
| 194 | + /* Get PGD offset in bytes */ | |
| 195 | + srli.d t0, t0, PGDIR_SHIFT | |
| 196 | + andi t0, t0, (PTRS_PER_PGD - 1) | |
| 197 | + slli.d t0, t0, 3 | |
| 198 | + add.d t1, t1, t0 | |
| 199 | + | |
| 200 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 201 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 202 | + ld.d t1, t1, 0 | |
| 203 | + srli.d t0, t0, PUD_SHIFT | |
| 204 | + andi t0, t0, (PTRS_PER_PUD - 1) | |
| 205 | + slli.d t0, t0, 3 | |
| 206 | + add.d t1, t1, t0 | |
| 207 | +#endif | |
| 208 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 209 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 210 | + ld.d t1, t1, 0 | |
| 211 | + srli.d t0, t0, PMD_SHIFT | |
| 212 | + andi t0, t0, (PTRS_PER_PMD - 1) | |
| 213 | + slli.d t0, t0, 3 | |
| 214 | + add.d t1, t1, t0 | |
| 215 | +#endif | |
| 216 | + ld.d ra, t1, 0 | |
| 217 | + | |
| 218 | + /* | |
| 219 | + * For huge tlb entries, pmde doesn't contain an address but | |
| 220 | + * instead contains the tlb pte. Check the PAGE_HUGE bit and | |
| 221 | + * see if we need to jump to huge tlb processing. | |
| 222 | + */ | |
| 223 | + andi t0, ra, _PAGE_HUGE | |
| 224 | + bne t0, $r0, tlb_huge_update_store | |
| 225 | + | |
| 226 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 227 | + srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | |
| 228 | + andi t0, t0, (PTRS_PER_PTE - 1) | |
| 229 | + slli.d t0, t0, _PTE_T_LOG2 | |
| 230 | + add.d t1, ra, t0 | |
| 231 | + | |
| 232 | + ld.d t0, t1, 0 | |
| 233 | + tlbsrch | |
| 234 | + | |
| 235 | + srli.d ra, t0, _PAGE_PRESENT_SHIFT | |
| 236 | + andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | |
| 237 | + xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | |
| 238 | + bne ra, $r0, nopage_tlb_store | |
| 239 | + | |
| 240 | + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | |
| 241 | + st.d t0, t1, 0 | |
| 242 | + | |
| 243 | + ori t1, t1, 8 | |
| 244 | + xori t1, t1, 8 | |
| 245 | + ld.d t0, t1, 0 | |
| 246 | + ld.d t1, t1, 8 | |
| 247 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 248 | + csrwr t1, LOONGARCH_CSR_TLBELO1 | |
| 249 | + tlbwr | |
| 250 | +leave_store: | |
| 251 | + csrrd t0, EXCEPTION_KS0 | |
| 252 | + csrrd t1, EXCEPTION_KS1 | |
| 253 | + csrrd ra, EXCEPTION_KS2 | |
| 254 | + ertn | |
| 255 | +#ifdef CONFIG_64BIT | |
| 256 | +vmalloc_store: | |
| 257 | + la.abs t1, swapper_pg_dir | |
| 258 | + b vmalloc_done_store | |
| 259 | +#endif | |
| 260 | + | |
| 261 | + /* | |
| 262 | + * This is the entry point when build_tlbchange_handler_head | |
| 263 | + * spots a huge page. | |
| 264 | + */ | |
| 265 | +tlb_huge_update_store: | |
| 266 | + ld.d t0, t1, 0 | |
| 267 | + srli.d ra, t0, _PAGE_PRESENT_SHIFT | |
| 268 | + andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | |
| 269 | + xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) | |
| 270 | + bne ra, $r0, nopage_tlb_store | |
| 271 | + | |
| 272 | + tlbsrch | |
| 273 | + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | |
| 274 | + | |
| 275 | + st.d t0, t1, 0 | |
| 276 | + addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) | |
| 277 | + addi.d ra, t1, 0 | |
| 278 | + csrxchg ra, t1, LOONGARCH_CSR_TLBIDX | |
| 279 | + tlbwr | |
| 280 | + | |
| 281 | + csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX | |
| 282 | + /* | |
| 283 | + * A huge PTE describes an area the size of the | |
| 284 | + * configured huge page size. This is twice the | |
| 285 | + * of the large TLB entry size we intend to use. | |
| 286 | + * A TLB entry half the size of the configured | |
| 287 | + * huge page size is configured into entrylo0 | |
| 288 | + * and entrylo1 to cover the contiguous huge PTE | |
| 289 | + * address space. | |
| 290 | + */ | |
| 291 | + /* Huge page: Move Global bit */ | |
| 292 | + xori t0, t0, _PAGE_HUGE | |
| 293 | + lu12i.w t1, _PAGE_HGLOBAL >> 12 | |
| 294 | + and t1, t0, t1 | |
| 295 | + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | |
| 296 | + or t0, t0, t1 | |
| 297 | + | |
| 298 | + addi.d ra, t0, 0 | |
| 299 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 300 | + addi.d t0, ra, 0 | |
| 301 | + | |
| 302 | + /* Convert to entrylo1 */ | |
| 303 | + addi.d t1, $r0, 1 | |
| 304 | + slli.d t1, t1, (HPAGE_SHIFT - 1) | |
| 305 | + add.d t0, t0, t1 | |
| 306 | + csrwr t0, LOONGARCH_CSR_TLBELO1 | |
| 307 | + | |
| 308 | + /* Set huge page tlb entry size */ | |
| 309 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 310 | + addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 311 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 312 | + | |
| 313 | + tlbfill | |
| 314 | + | |
| 315 | + /* Reset default page size */ | |
| 316 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 317 | + addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 318 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 319 | + | |
| 320 | +nopage_tlb_store: | |
| 321 | + csrrd ra, EXCEPTION_KS2 | |
| 322 | + la.abs t0, tlb_do_page_fault_1 | |
| 323 | + jirl $r0, t0, 0 | |
| 324 | +SYM_FUNC_END(handle_tlb_store) | |
| 325 | + | |
| 326 | +SYM_FUNC_START(handle_tlb_modify) | |
| 327 | + csrwr t0, EXCEPTION_KS0 | |
| 328 | + csrwr t1, EXCEPTION_KS1 | |
| 329 | + csrwr ra, EXCEPTION_KS2 | |
| 330 | + | |
| 331 | + /* | |
| 332 | + * The vmalloc handling is not in the hotpath. | |
| 333 | + */ | |
| 334 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 335 | + blt t0, $r0, vmalloc_modify | |
| 336 | + csrrd t1, LOONGARCH_CSR_PGDL | |
| 337 | + | |
| 338 | +vmalloc_done_modify: | |
| 339 | + /* Get PGD offset in bytes */ | |
| 340 | + srli.d t0, t0, PGDIR_SHIFT | |
| 341 | + andi t0, t0, (PTRS_PER_PGD - 1) | |
| 342 | + slli.d t0, t0, 3 | |
| 343 | + add.d t1, t1, t0 | |
| 344 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 345 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 346 | + ld.d t1, t1, 0 | |
| 347 | + srli.d t0, t0, PUD_SHIFT | |
| 348 | + andi t0, t0, (PTRS_PER_PUD - 1) | |
| 349 | + slli.d t0, t0, 3 | |
| 350 | + add.d t1, t1, t0 | |
| 351 | +#endif | |
| 352 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 353 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 354 | + ld.d t1, t1, 0 | |
| 355 | + srli.d t0, t0, PMD_SHIFT | |
| 356 | + andi t0, t0, (PTRS_PER_PMD - 1) | |
| 357 | + slli.d t0, t0, 3 | |
| 358 | + add.d t1, t1, t0 | |
| 359 | +#endif | |
| 360 | + ld.d ra, t1, 0 | |
| 361 | + | |
| 362 | + /* | |
| 363 | + * For huge tlb entries, pmde doesn't contain an address but | |
| 364 | + * instead contains the tlb pte. Check the PAGE_HUGE bit and | |
| 365 | + * see if we need to jump to huge tlb processing. | |
| 366 | + */ | |
| 367 | + andi t0, ra, _PAGE_HUGE | |
| 368 | + bne t0, $r0, tlb_huge_update_modify | |
| 369 | + | |
| 370 | + csrrd t0, LOONGARCH_CSR_BADV | |
| 371 | + srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER) | |
| 372 | + andi t0, t0, (PTRS_PER_PTE - 1) | |
| 373 | + slli.d t0, t0, _PTE_T_LOG2 | |
| 374 | + add.d t1, ra, t0 | |
| 375 | + | |
| 376 | + ld.d t0, t1, 0 | |
| 377 | + tlbsrch | |
| 378 | + | |
| 379 | + srli.d ra, t0, _PAGE_WRITE_SHIFT | |
| 380 | + andi ra, ra, 1 | |
| 381 | + beq ra, $r0, nopage_tlb_modify | |
| 382 | + | |
| 383 | + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | |
| 384 | + st.d t0, t1, 0 | |
| 385 | + ori t1, t1, 8 | |
| 386 | + xori t1, t1, 8 | |
| 387 | + ld.d t0, t1, 0 | |
| 388 | + ld.d t1, t1, 8 | |
| 389 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 390 | + csrwr t1, LOONGARCH_CSR_TLBELO1 | |
| 391 | + tlbwr | |
| 392 | +leave_modify: | |
| 393 | + csrrd t0, EXCEPTION_KS0 | |
| 394 | + csrrd t1, EXCEPTION_KS1 | |
| 395 | + csrrd ra, EXCEPTION_KS2 | |
| 396 | + ertn | |
| 397 | +#ifdef CONFIG_64BIT | |
| 398 | +vmalloc_modify: | |
| 399 | + la.abs t1, swapper_pg_dir | |
| 400 | + b vmalloc_done_modify | |
| 401 | +#endif | |
| 402 | + | |
| 403 | + /* | |
| 404 | + * This is the entry point when | |
| 405 | + * build_tlbchange_handler_head spots a huge page. | |
| 406 | + */ | |
| 407 | +tlb_huge_update_modify: | |
| 408 | + ld.d t0, t1, 0 | |
| 409 | + | |
| 410 | + srli.d ra, t0, _PAGE_WRITE_SHIFT | |
| 411 | + andi ra, ra, 1 | |
| 412 | + beq ra, $r0, nopage_tlb_modify | |
| 413 | + | |
| 414 | + tlbsrch | |
| 415 | + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) | |
| 416 | + | |
| 417 | + st.d t0, t1, 0 | |
| 418 | + /* | |
| 419 | + * A huge PTE describes an area the size of the | |
| 420 | + * configured huge page size. This is twice the | |
| 421 | + * of the large TLB entry size we intend to use. | |
| 422 | + * A TLB entry half the size of the configured | |
| 423 | + * huge page size is configured into entrylo0 | |
| 424 | + * and entrylo1 to cover the contiguous huge PTE | |
| 425 | + * address space. | |
| 426 | + */ | |
| 427 | + /* Huge page: Move Global bit */ | |
| 428 | + xori t0, t0, _PAGE_HUGE | |
| 429 | + lu12i.w t1, _PAGE_HGLOBAL >> 12 | |
| 430 | + and t1, t0, t1 | |
| 431 | + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) | |
| 432 | + or t0, t0, t1 | |
| 433 | + | |
| 434 | + addi.d ra, t0, 0 | |
| 435 | + csrwr t0, LOONGARCH_CSR_TLBELO0 | |
| 436 | + addi.d t0, ra, 0 | |
| 437 | + | |
| 438 | + /* Convert to entrylo1 */ | |
| 439 | + addi.d t1, $r0, 1 | |
| 440 | + slli.d t1, t1, (HPAGE_SHIFT - 1) | |
| 441 | + add.d t0, t0, t1 | |
| 442 | + csrwr t0, LOONGARCH_CSR_TLBELO1 | |
| 443 | + | |
| 444 | + /* Set huge page tlb entry size */ | |
| 445 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 446 | + addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 447 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 448 | + | |
| 449 | + tlbwr | |
| 450 | + | |
| 451 | + /* Reset default page size */ | |
| 452 | + addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16) | |
| 453 | + addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) | |
| 454 | + csrxchg t1, t0, LOONGARCH_CSR_TLBIDX | |
| 455 | + | |
| 456 | +nopage_tlb_modify: | |
| 457 | + csrrd ra, EXCEPTION_KS2 | |
| 458 | + la.abs t0, tlb_do_page_fault_1 | |
| 459 | + jirl $r0, t0, 0 | |
| 460 | +SYM_FUNC_END(handle_tlb_modify) | |
| 461 | + | |
| 462 | +SYM_FUNC_START(handle_tlb_refill) | |
| 463 | + csrwr t0, LOONGARCH_CSR_TLBRSAVE | |
| 464 | + csrrd t0, LOONGARCH_CSR_PGD | |
| 465 | + lddir t0, t0, 3 | |
| 466 | +#if CONFIG_PGTABLE_LEVELS > 3 | |
| 467 | + lddir t0, t0, 2 | |
| 468 | +#endif | |
| 469 | +#if CONFIG_PGTABLE_LEVELS > 2 | |
| 470 | + lddir t0, t0, 1 | |
| 471 | +#endif | |
| 472 | + ldpte t0, 0 | |
| 473 | + ldpte t0, 1 | |
| 474 | + tlbfill | |
| 475 | + csrrd t0, LOONGARCH_CSR_TLBRSAVE | |
| 476 | + ertn | |
| 477 | +SYM_FUNC_END(handle_tlb_refill) | |