History

LoongArch: Add memory management

Add memory management support for LoongArch, including: cache and tlb management, page fault handling and ioremap/mmap support.

Huacai Chen committed 4 years ago

Commit 09cfefb1 parent 803b0fc26 files · +3172 −0Browse files

Unified · Split

26 files
  1. arch/loongarch/include/asm/cache.h +13 −0
  2. arch/loongarch/include/asm/cacheflush.h +80 −0
  3. arch/loongarch/include/asm/cacheops.h +37 −0
  4. arch/loongarch/include/asm/fixmap.h +13 −0
  5. arch/loongarch/include/asm/hugetlb.h +83 −0
  6. arch/loongarch/include/asm/page.h +115 −0
  7. arch/loongarch/include/asm/pgalloc.h +103 −0
  8. arch/loongarch/include/asm/pgtable-bits.h +131 −0
  9. arch/loongarch/include/asm/pgtable.h +532 −0
  10. arch/loongarch/include/asm/shmparam.h +12 −0
  11. arch/loongarch/include/asm/sparsemem.h +23 −0
  12. arch/loongarch/include/asm/tlb.h +180 −0
  13. arch/loongarch/include/asm/tlbflush.h +35 −0
  14. arch/loongarch/include/asm/vmalloc.h +4 −0
  15. arch/loongarch/mm/cache.c +141 −0
  16. arch/loongarch/mm/extable.c +22 −0
  17. arch/loongarch/mm/fault.c +261 −0
  18. arch/loongarch/mm/hugetlbpage.c +87 −0
  19. arch/loongarch/mm/init.c +165 −0
  20. arch/loongarch/mm/ioremap.c +27 −0
  21. arch/loongarch/mm/maccess.c +10 −0
  22. arch/loongarch/mm/mmap.c +125 −0
  23. arch/loongarch/mm/page.S +84 −0
  24. arch/loongarch/mm/pgtable.c +130 −0
  25. arch/loongarch/mm/tlb.c +282 −0
  26. arch/loongarch/mm/tlbex.S +477 −0
arch/loongarch/include/asm/cache.h+13−0
@@ -0,0 +1,13 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_CACHE_H
6+#define _ASM_CACHE_H
7+
8+#define L1_CACHE_SHIFT CONFIG_L1_CACHE_SHIFT
9+#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
10+
11+#define __read_mostly __section(".data..read_mostly")
12+
13+#endif /* _ASM_CACHE_H */
arch/loongarch/include/asm/cacheflush.h+80−0
@@ -0,0 +1,80 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_CACHEFLUSH_H
6+#define _ASM_CACHEFLUSH_H
7+
8+#include <linux/mm.h>
9+#include <asm/cpu-features.h>
10+#include <asm/cacheops.h>
11+
12+extern void local_flush_icache_range(unsigned long start, unsigned long end);
13+
14+#define flush_icache_range local_flush_icache_range
15+#define flush_icache_user_range local_flush_icache_range
16+
17+#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0
18+
19+#define flush_cache_all() do { } while (0)
20+#define flush_cache_mm(mm) do { } while (0)
21+#define flush_cache_dup_mm(mm) do { } while (0)
22+#define flush_cache_range(vma, start, end) do { } while (0)
23+#define flush_cache_page(vma, vmaddr, pfn) do { } while (0)
24+#define flush_cache_vmap(start, end) do { } while (0)
25+#define flush_cache_vunmap(start, end) do { } while (0)
26+#define flush_icache_page(vma, page) do { } while (0)
27+#define flush_icache_user_page(vma, page, addr, len) do { } while (0)
28+#define flush_dcache_page(page) do { } while (0)
29+#define flush_dcache_mmap_lock(mapping) do { } while (0)
30+#define flush_dcache_mmap_unlock(mapping) do { } while (0)
31+
32+#define cache_op(op, addr) \
33+ __asm__ __volatile__( \
34+ " cacop %0, %1 \n" \
35+ : \
36+ : "i" (op), "ZC" (*(unsigned char *)(addr)))
37+
38+static inline void flush_icache_line_indexed(unsigned long addr)
39+{
40+ cache_op(Index_Invalidate_I, addr);
41+}
42+
43+static inline void flush_dcache_line_indexed(unsigned long addr)
44+{
45+ cache_op(Index_Writeback_Inv_D, addr);
46+}
47+
48+static inline void flush_vcache_line_indexed(unsigned long addr)
49+{
50+ cache_op(Index_Writeback_Inv_V, addr);
51+}
52+
53+static inline void flush_scache_line_indexed(unsigned long addr)
54+{
55+ cache_op(Index_Writeback_Inv_S, addr);
56+}
57+
58+static inline void flush_icache_line(unsigned long addr)
59+{
60+ cache_op(Hit_Invalidate_I, addr);
61+}
62+
63+static inline void flush_dcache_line(unsigned long addr)
64+{
65+ cache_op(Hit_Writeback_Inv_D, addr);
66+}
67+
68+static inline void flush_vcache_line(unsigned long addr)
69+{
70+ cache_op(Hit_Writeback_Inv_V, addr);
71+}
72+
73+static inline void flush_scache_line(unsigned long addr)
74+{
75+ cache_op(Hit_Writeback_Inv_S, addr);
76+}
77+
78+#include <asm-generic/cacheflush.h>
79+
80+#endif /* _ASM_CACHEFLUSH_H */
arch/loongarch/include/asm/cacheops.h+37−0
@@ -0,0 +1,37 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Cache operations for the cache instruction.
4+ *
5+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6+ */
7+#ifndef __ASM_CACHEOPS_H
8+#define __ASM_CACHEOPS_H
9+
10+/*
11+ * Most cache ops are split into a 2 bit field identifying the cache, and a 3
12+ * bit field identifying the cache operation.
13+ */
14+#define CacheOp_Cache 0x03
15+#define CacheOp_Op 0x1c
16+
17+#define Cache_I 0x00
18+#define Cache_D 0x01
19+#define Cache_V 0x02
20+#define Cache_S 0x03
21+
22+#define Index_Invalidate 0x08
23+#define Index_Writeback_Inv 0x08
24+#define Hit_Invalidate 0x10
25+#define Hit_Writeback_Inv 0x10
26+#define CacheOp_User_Defined 0x18
27+
28+#define Index_Invalidate_I (Cache_I | Index_Invalidate)
29+#define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv)
30+#define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv)
31+#define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv)
32+#define Hit_Invalidate_I (Cache_I | Hit_Invalidate)
33+#define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv)
34+#define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv)
35+#define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv)
36+
37+#endif /* __ASM_CACHEOPS_H */
arch/loongarch/include/asm/fixmap.h+13−0
@@ -0,0 +1,13 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * fixmap.h: compile-time virtual memory allocation
4+ *
5+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6+ */
7+
8+#ifndef _ASM_FIXMAP_H
9+#define _ASM_FIXMAP_H
10+
11+#define NR_FIX_BTMAPS 64
12+
13+#endif
arch/loongarch/include/asm/hugetlb.h+83−0
@@ -0,0 +1,83 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+
6+#ifndef __ASM_HUGETLB_H
7+#define __ASM_HUGETLB_H
8+
9+#include <asm/page.h>
10+
11+uint64_t pmd_to_entrylo(unsigned long pmd_val);
12+
13+#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE
14+static inline int prepare_hugepage_range(struct file *file,
15+ unsigned long addr,
16+ unsigned long len)
17+{
18+ unsigned long task_size = STACK_TOP;
19+ struct hstate *h = hstate_file(file);
20+
21+ if (len & ~huge_page_mask(h))
22+ return -EINVAL;
23+ if (addr & ~huge_page_mask(h))
24+ return -EINVAL;
25+ if (len > task_size)
26+ return -ENOMEM;
27+ if (task_size - len < addr)
28+ return -EINVAL;
29+ return 0;
30+}
31+
32+#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
33+static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
34+ unsigned long addr, pte_t *ptep)
35+{
36+ pte_t clear;
37+ pte_t pte = *ptep;
38+
39+ pte_val(clear) = (unsigned long)invalid_pte_table;
40+ set_pte_at(mm, addr, ptep, clear);
41+ return pte;
42+}
43+
44+#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
45+static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
46+ unsigned long addr, pte_t *ptep)
47+{
48+ pte_t pte;
49+
50+ pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
51+ flush_tlb_page(vma, addr);
52+ return pte;
53+}
54+
55+#define __HAVE_ARCH_HUGE_PTE_NONE
56+static inline int huge_pte_none(pte_t pte)
57+{
58+ unsigned long val = pte_val(pte) & ~_PAGE_GLOBAL;
59+ return !val || (val == (unsigned long)invalid_pte_table);
60+}
61+
62+#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
63+static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
64+ unsigned long addr,
65+ pte_t *ptep, pte_t pte,
66+ int dirty)
67+{
68+ int changed = !pte_same(*ptep, pte);
69+
70+ if (changed) {
71+ set_pte_at(vma->vm_mm, addr, ptep, pte);
72+ /*
73+ * There could be some standard sized pages in there,
74+ * get them all.
75+ */
76+ flush_tlb_range(vma, addr, addr + HPAGE_SIZE);
77+ }
78+ return changed;
79+}
80+
81+#include <asm-generic/hugetlb.h>
82+
83+#endif /* __ASM_HUGETLB_H */
arch/loongarch/include/asm/page.h+115−0
@@ -0,0 +1,115 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_PAGE_H
6+#define _ASM_PAGE_H
7+
8+#include <linux/const.h>
9+
10+/*
11+ * PAGE_SHIFT determines the page size
12+ */
13+#ifdef CONFIG_PAGE_SIZE_4KB
14+#define PAGE_SHIFT 12
15+#endif
16+#ifdef CONFIG_PAGE_SIZE_16KB
17+#define PAGE_SHIFT 14
18+#endif
19+#ifdef CONFIG_PAGE_SIZE_64KB
20+#define PAGE_SHIFT 16
21+#endif
22+#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
23+#define PAGE_MASK (~(PAGE_SIZE - 1))
24+
25+#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3)
26+#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
27+#define HPAGE_MASK (~(HPAGE_SIZE - 1))
28+#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
29+
30+#ifndef __ASSEMBLY__
31+
32+#include <linux/kernel.h>
33+#include <linux/pfn.h>
34+
35+#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT))
36+
37+/*
38+ * It's normally defined only for FLATMEM config but it's
39+ * used in our early mem init code for all memory models.
40+ * So always define it.
41+ */
42+#define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET)
43+
44+extern void clear_page(void *page);
45+extern void copy_page(void *to, void *from);
46+
47+#define clear_user_page(page, vaddr, pg) clear_page(page)
48+#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
49+
50+extern unsigned long shm_align_mask;
51+
52+struct page;
53+struct vm_area_struct;
54+void copy_user_highpage(struct page *to, struct page *from,
55+ unsigned long vaddr, struct vm_area_struct *vma);
56+
57+#define __HAVE_ARCH_COPY_USER_HIGHPAGE
58+
59+typedef struct { unsigned long pte; } pte_t;
60+#define pte_val(x) ((x).pte)
61+#define __pte(x) ((pte_t) { (x) })
62+typedef struct page *pgtable_t;
63+
64+typedef struct { unsigned long pgd; } pgd_t;
65+#define pgd_val(x) ((x).pgd)
66+#define __pgd(x) ((pgd_t) { (x) })
67+
68+/*
69+ * Manipulate page protection bits
70+ */
71+typedef struct { unsigned long pgprot; } pgprot_t;
72+#define pgprot_val(x) ((x).pgprot)
73+#define __pgprot(x) ((pgprot_t) { (x) })
74+#define pte_pgprot(x) __pgprot(pte_val(x) & ~_PFN_MASK)
75+
76+#define ptep_buddy(x) ((pte_t *)((unsigned long)(x) ^ sizeof(pte_t)))
77+
78+/*
79+ * __pa()/__va() should be used only during mem init.
80+ */
81+#define __pa(x) PHYSADDR(x)
82+#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET))
83+
84+#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
85+
86+#ifdef CONFIG_FLATMEM
87+
88+static inline int pfn_valid(unsigned long pfn)
89+{
90+ /* avoid <linux/mm.h> include hell */
91+ extern unsigned long max_mapnr;
92+ unsigned long pfn_offset = ARCH_PFN_OFFSET;
93+
94+ return pfn >= pfn_offset && pfn < max_mapnr;
95+}
96+
97+#endif
98+
99+#define virt_to_pfn(kaddr) PFN_DOWN(virt_to_phys((void *)(kaddr)))
100+#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr))
101+
102+extern int __virt_addr_valid(volatile void *kaddr);
103+#define virt_addr_valid(kaddr) __virt_addr_valid((volatile void *)(kaddr))
104+
105+#define VM_DATA_DEFAULT_FLAGS \
106+ (VM_READ | VM_WRITE | \
107+ ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \
108+ VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
109+
110+#include <asm-generic/memory_model.h>
111+#include <asm-generic/getorder.h>
112+
113+#endif /* !__ASSEMBLY__ */
114+
115+#endif /* _ASM_PAGE_H */
arch/loongarch/include/asm/pgalloc.h+103−0
@@ -0,0 +1,103 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_PGALLOC_H
6+#define _ASM_PGALLOC_H
7+
8+#include <linux/mm.h>
9+#include <linux/sched.h>
10+
11+#define __HAVE_ARCH_PMD_ALLOC_ONE
12+#define __HAVE_ARCH_PUD_ALLOC_ONE
13+#include <asm-generic/pgalloc.h>
14+
15+static inline void pmd_populate_kernel(struct mm_struct *mm,
16+ pmd_t *pmd, pte_t *pte)
17+{
18+ set_pmd(pmd, __pmd((unsigned long)pte));
19+}
20+
21+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte)
22+{
23+ set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
24+}
25+
26+#ifndef __PAGETABLE_PMD_FOLDED
27+
28+static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
29+{
30+ set_pud(pud, __pud((unsigned long)pmd));
31+}
32+#endif
33+
34+#ifndef __PAGETABLE_PUD_FOLDED
35+
36+static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
37+{
38+ set_p4d(p4d, __p4d((unsigned long)pud));
39+}
40+
41+#endif /* __PAGETABLE_PUD_FOLDED */
42+
43+extern void pagetable_init(void);
44+
45+/*
46+ * Initialize a new pmd table with invalid pointers.
47+ */
48+extern void pmd_init(unsigned long page, unsigned long pagetable);
49+
50+/*
51+ * Initialize a new pgd / pmd table with invalid pointers.
52+ */
53+extern void pgd_init(unsigned long page);
54+extern pgd_t *pgd_alloc(struct mm_struct *mm);
55+
56+#define __pte_free_tlb(tlb, pte, address) \
57+do { \
58+ pgtable_pte_page_dtor(pte); \
59+ tlb_remove_page((tlb), pte); \
60+} while (0)
61+
62+#ifndef __PAGETABLE_PMD_FOLDED
63+
64+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
65+{
66+ pmd_t *pmd;
67+ struct page *pg;
68+
69+ pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_ORDER);
70+ if (!pg)
71+ return NULL;
72+
73+ if (!pgtable_pmd_page_ctor(pg)) {
74+ __free_pages(pg, PMD_ORDER);
75+ return NULL;
76+ }
77+
78+ pmd = (pmd_t *)page_address(pg);
79+ pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table);
80+ return pmd;
81+}
82+
83+#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
84+
85+#endif
86+
87+#ifndef __PAGETABLE_PUD_FOLDED
88+
89+static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
90+{
91+ pud_t *pud;
92+
93+ pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER);
94+ if (pud)
95+ pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
96+ return pud;
97+}
98+
99+#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x)
100+
101+#endif /* __PAGETABLE_PUD_FOLDED */
102+
103+#endif /* _ASM_PGALLOC_H */
arch/loongarch/include/asm/pgtable-bits.h+131−0
@@ -0,0 +1,131 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_PGTABLE_BITS_H
6+#define _ASM_PGTABLE_BITS_H
7+
8+/* Page table bits */
9+#define _PAGE_VALID_SHIFT 0
10+#define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */
11+#define _PAGE_DIRTY_SHIFT 1
12+#define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */
13+#define _CACHE_SHIFT 4 /* 4~5, two bits */
14+#define _PAGE_GLOBAL_SHIFT 6
15+#define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */
16+#define _PAGE_PRESENT_SHIFT 7
17+#define _PAGE_WRITE_SHIFT 8
18+#define _PAGE_MODIFIED_SHIFT 9
19+#define _PAGE_PROTNONE_SHIFT 10
20+#define _PAGE_SPECIAL_SHIFT 11
21+#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */
22+#define _PAGE_PFN_SHIFT 12
23+#define _PAGE_PFN_END_SHIFT 48
24+#define _PAGE_NO_READ_SHIFT 61
25+#define _PAGE_NO_EXEC_SHIFT 62
26+#define _PAGE_RPLV_SHIFT 63
27+
28+/* Used by software */
29+#define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT)
30+#define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT)
31+#define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT)
32+#define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT)
33+#define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT)
34+#define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT)
35+
36+/* Used by TLB hardware (placed in EntryLo*) */
37+#define _PAGE_VALID (_ULCAST_(1) << _PAGE_VALID_SHIFT)
38+#define _PAGE_DIRTY (_ULCAST_(1) << _PAGE_DIRTY_SHIFT)
39+#define _PAGE_PLV (_ULCAST_(3) << _PAGE_PLV_SHIFT)
40+#define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT)
41+#define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT)
42+#define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT)
43+#define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT)
44+#define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT)
45+#define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT)
46+#define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT)
47+#define _PFN_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT)
48+
49+#define _PAGE_USER (PLV_USER << _PAGE_PLV_SHIFT)
50+#define _PAGE_KERN (PLV_KERN << _PAGE_PLV_SHIFT)
51+
52+#define _PFN_MASK (~((_ULCAST_(1) << (_PFN_SHIFT)) - 1) & \
53+ ((_ULCAST_(1) << (_PAGE_PFN_END_SHIFT)) - 1))
54+
55+/*
56+ * Cache attributes
57+ */
58+#ifndef _CACHE_SUC
59+#define _CACHE_SUC (0<<_CACHE_SHIFT) /* Strong-ordered UnCached */
60+#endif
61+#ifndef _CACHE_CC
62+#define _CACHE_CC (1<<_CACHE_SHIFT) /* Coherent Cached */
63+#endif
64+#ifndef _CACHE_WUC
65+#define _CACHE_WUC (2<<_CACHE_SHIFT) /* Weak-ordered UnCached */
66+#endif
67+
68+#define __READABLE (_PAGE_VALID)
69+#define __WRITEABLE (_PAGE_DIRTY | _PAGE_WRITE)
70+
71+#define _PAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV)
72+#define _HPAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV | _PAGE_HUGE)
73+
74+#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_NO_READ | \
75+ _PAGE_USER | _CACHE_CC)
76+#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \
77+ _PAGE_USER | _CACHE_CC)
78+#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _CACHE_CC)
79+
80+#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
81+ _PAGE_GLOBAL | _PAGE_KERN | _CACHE_CC)
82+#define PAGE_KERNEL_SUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
83+ _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC)
84+#define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
85+ _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC)
86+
87+#define __P000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ)
88+#define __P001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
89+#define __P010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
90+#define __P011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
91+#define __P100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
92+#define __P101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
93+#define __P110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
94+#define __P111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
95+
96+#define __S000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ)
97+#define __S001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
98+#define __S010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE)
99+#define __S011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE)
100+#define __S100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
101+#define __S101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
102+#define __S110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE)
103+#define __S111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE)
104+
105+#ifndef __ASSEMBLY__
106+
107+#define pgprot_noncached pgprot_noncached
108+
109+static inline pgprot_t pgprot_noncached(pgprot_t _prot)
110+{
111+ unsigned long prot = pgprot_val(_prot);
112+
113+ prot = (prot & ~_CACHE_MASK) | _CACHE_SUC;
114+
115+ return __pgprot(prot);
116+}
117+
118+#define pgprot_writecombine pgprot_writecombine
119+
120+static inline pgprot_t pgprot_writecombine(pgprot_t _prot)
121+{
122+ unsigned long prot = pgprot_val(_prot);
123+
124+ prot = (prot & ~_CACHE_MASK) | _CACHE_WUC;
125+
126+ return __pgprot(prot);
127+}
128+
129+#endif /* !__ASSEMBLY__ */
130+
131+#endif /* _ASM_PGTABLE_BITS_H */
arch/loongarch/include/asm/pgtable.h+532−0
@@ -0,0 +1,532 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ *
5+ * Derived from MIPS:
6+ * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle
7+ * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc.
8+ */
9+#ifndef _ASM_PGTABLE_H
10+#define _ASM_PGTABLE_H
11+
12+#include <linux/compiler.h>
13+#include <asm/addrspace.h>
14+#include <asm/pgtable-bits.h>
15+
16+#if CONFIG_PGTABLE_LEVELS == 2
17+#include <asm-generic/pgtable-nopmd.h>
18+#elif CONFIG_PGTABLE_LEVELS == 3
19+#include <asm-generic/pgtable-nopud.h>
20+#else
21+#include <asm-generic/pgtable-nop4d.h>
22+#endif
23+
24+#define PGD_ORDER 0
25+#define PUD_ORDER 0
26+#define PMD_ORDER 0
27+#define PTE_ORDER 0
28+
29+#if CONFIG_PGTABLE_LEVELS == 2
30+#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
31+#elif CONFIG_PGTABLE_LEVELS == 3
32+#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
33+#define PMD_SIZE (1UL << PMD_SHIFT)
34+#define PMD_MASK (~(PMD_SIZE-1))
35+#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3))
36+#elif CONFIG_PGTABLE_LEVELS == 4
37+#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
38+#define PMD_SIZE (1UL << PMD_SHIFT)
39+#define PMD_MASK (~(PMD_SIZE-1))
40+#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3))
41+#define PUD_SIZE (1UL << PUD_SHIFT)
42+#define PUD_MASK (~(PUD_SIZE-1))
43+#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3))
44+#endif
45+
46+#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
47+#define PGDIR_MASK (~(PGDIR_SIZE-1))
48+
49+#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT + PGD_ORDER - 3))
50+
51+#define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) >> 3)
52+#if CONFIG_PGTABLE_LEVELS > 3
53+#define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) >> 3)
54+#endif
55+#if CONFIG_PGTABLE_LEVELS > 2
56+#define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) >> 3)
57+#endif
58+#define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) >> 3)
59+
60+#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1)
61+
62+#ifndef __ASSEMBLY__
63+
64+#include <linux/mm_types.h>
65+#include <linux/mmzone.h>
66+#include <asm/fixmap.h>
67+#include <asm/io.h>
68+
69+struct mm_struct;
70+struct vm_area_struct;
71+
72+/*
73+ * ZERO_PAGE is a global shared page that is always zero; used
74+ * for zero-mapped memory areas etc..
75+ */
76+
77+extern unsigned long empty_zero_page;
78+extern unsigned long zero_page_mask;
79+
80+#define ZERO_PAGE(vaddr) \
81+ (virt_to_page((void *)(empty_zero_page + (((unsigned long)(vaddr)) & zero_page_mask))))
82+#define __HAVE_COLOR_ZERO_PAGE
83+
84+/*
85+ * TLB refill handlers may also map the vmalloc area into xkvrange.
86+ * Avoid the first couple of pages so NULL pointer dereferences will
87+ * still reliably trap.
88+ */
89+#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
90+#define MODULES_END (MODULES_VADDR + SZ_256M)
91+
92+#define VMALLOC_START MODULES_END
93+#define VMALLOC_END \
94+ (vm_map_base + \
95+ min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE)
96+
97+#define pte_ERROR(e) \
98+ pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
99+#ifndef __PAGETABLE_PMD_FOLDED
100+#define pmd_ERROR(e) \
101+ pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
102+#endif
103+#ifndef __PAGETABLE_PUD_FOLDED
104+#define pud_ERROR(e) \
105+ pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
106+#endif
107+#define pgd_ERROR(e) \
108+ pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
109+
110+extern pte_t invalid_pte_table[PTRS_PER_PTE];
111+
112+#ifndef __PAGETABLE_PUD_FOLDED
113+
114+typedef struct { unsigned long pud; } pud_t;
115+#define pud_val(x) ((x).pud)
116+#define __pud(x) ((pud_t) { (x) })
117+
118+extern pud_t invalid_pud_table[PTRS_PER_PUD];
119+
120+/*
121+ * Empty pgd/p4d entries point to the invalid_pud_table.
122+ */
123+static inline int p4d_none(p4d_t p4d)
124+{
125+ return p4d_val(p4d) == (unsigned long)invalid_pud_table;
126+}
127+
128+static inline int p4d_bad(p4d_t p4d)
129+{
130+ return p4d_val(p4d) & ~PAGE_MASK;
131+}
132+
133+static inline int p4d_present(p4d_t p4d)
134+{
135+ return p4d_val(p4d) != (unsigned long)invalid_pud_table;
136+}
137+
138+static inline void p4d_clear(p4d_t *p4dp)
139+{
140+ p4d_val(*p4dp) = (unsigned long)invalid_pud_table;
141+}
142+
143+static inline pud_t *p4d_pgtable(p4d_t p4d)
144+{
145+ return (pud_t *)p4d_val(p4d);
146+}
147+
148+static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
149+{
150+ *p4d = p4dval;
151+}
152+
153+#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d))
154+#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT))
155+
156+#endif
157+
158+#ifndef __PAGETABLE_PMD_FOLDED
159+
160+typedef struct { unsigned long pmd; } pmd_t;
161+#define pmd_val(x) ((x).pmd)
162+#define __pmd(x) ((pmd_t) { (x) })
163+
164+extern pmd_t invalid_pmd_table[PTRS_PER_PMD];
165+
166+/*
167+ * Empty pud entries point to the invalid_pmd_table.
168+ */
169+static inline int pud_none(pud_t pud)
170+{
171+ return pud_val(pud) == (unsigned long)invalid_pmd_table;
172+}
173+
174+static inline int pud_bad(pud_t pud)
175+{
176+ return pud_val(pud) & ~PAGE_MASK;
177+}
178+
179+static inline int pud_present(pud_t pud)
180+{
181+ return pud_val(pud) != (unsigned long)invalid_pmd_table;
182+}
183+
184+static inline void pud_clear(pud_t *pudp)
185+{
186+ pud_val(*pudp) = ((unsigned long)invalid_pmd_table);
187+}
188+
189+static inline pmd_t *pud_pgtable(pud_t pud)
190+{
191+ return (pmd_t *)pud_val(pud);
192+}
193+
194+#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0)
195+
196+#define pud_phys(pud) virt_to_phys((void *)pud_val(pud))
197+#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT))
198+
199+#endif
200+
201+/*
202+ * Empty pmd entries point to the invalid_pte_table.
203+ */
204+static inline int pmd_none(pmd_t pmd)
205+{
206+ return pmd_val(pmd) == (unsigned long)invalid_pte_table;
207+}
208+
209+static inline int pmd_bad(pmd_t pmd)
210+{
211+ return (pmd_val(pmd) & ~PAGE_MASK);
212+}
213+
214+static inline int pmd_present(pmd_t pmd)
215+{
216+ if (unlikely(pmd_val(pmd) & _PAGE_HUGE))
217+ return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE));
218+
219+ return pmd_val(pmd) != (unsigned long)invalid_pte_table;
220+}
221+
222+static inline void pmd_clear(pmd_t *pmdp)
223+{
224+ pmd_val(*pmdp) = ((unsigned long)invalid_pte_table);
225+}
226+
227+#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0)
228+
229+#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))
230+
231+#ifndef CONFIG_TRANSPARENT_HUGEPAGE
232+#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
233+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
234+
235+#define pmd_page_vaddr(pmd) pmd_val(pmd)
236+
237+extern pmd_t mk_pmd(struct page *page, pgprot_t prot);
238+extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd);
239+
240+#define pte_page(x) pfn_to_page(pte_pfn(x))
241+#define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> _PFN_SHIFT))
242+#define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot))
243+#define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot))
244+
245+/*
246+ * Initialize a new pgd / pmd table with invalid pointers.
247+ */
248+extern void pgd_init(unsigned long page);
249+extern void pud_init(unsigned long page, unsigned long pagetable);
250+extern void pmd_init(unsigned long page, unsigned long pagetable);
251+
252+/*
253+ * Non-present pages: high 40 bits are offset, next 8 bits type,
254+ * low 16 bits zero.
255+ */
256+static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
257+{ pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; }
258+
259+#define __swp_type(x) (((x).val >> 16) & 0xff)
260+#define __swp_offset(x) ((x).val >> 24)
261+#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) })
262+#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
263+#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
264+#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
265+#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE })
266+
267+extern void paging_init(void);
268+
269+#define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL))
270+#define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE))
271+#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC)
272+
273+static inline void set_pte(pte_t *ptep, pte_t pteval)
274+{
275+ *ptep = pteval;
276+ if (pte_val(pteval) & _PAGE_GLOBAL) {
277+ pte_t *buddy = ptep_buddy(ptep);
278+ /*
279+ * Make sure the buddy is global too (if it's !none,
280+ * it better already be global)
281+ */
282+ if (pte_none(*buddy))
283+ pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
284+ }
285+}
286+
287+static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
288+ pte_t *ptep, pte_t pteval)
289+{
290+ set_pte(ptep, pteval);
291+}
292+
293+static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
294+{
295+ /* Preserve global status for the pair */
296+ if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL)
297+ set_pte_at(mm, addr, ptep, __pte(_PAGE_GLOBAL));
298+ else
299+ set_pte_at(mm, addr, ptep, __pte(0));
300+}
301+
302+#define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1)
303+#define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1)
304+#define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1)
305+
306+extern pgd_t swapper_pg_dir[];
307+extern pgd_t invalid_pg_dir[];
308+
309+/*
310+ * The following only work if pte_present() is true.
311+ * Undefined behaviour if not..
312+ */
313+static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
314+static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
315+static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; }
316+
317+static inline pte_t pte_mkold(pte_t pte)
318+{
319+ pte_val(pte) &= ~_PAGE_ACCESSED;
320+ return pte;
321+}
322+
323+static inline pte_t pte_mkyoung(pte_t pte)
324+{
325+ pte_val(pte) |= _PAGE_ACCESSED;
326+ return pte;
327+}
328+
329+static inline pte_t pte_mkclean(pte_t pte)
330+{
331+ pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
332+ return pte;
333+}
334+
335+static inline pte_t pte_mkdirty(pte_t pte)
336+{
337+ pte_val(pte) |= (_PAGE_DIRTY | _PAGE_MODIFIED);
338+ return pte;
339+}
340+
341+static inline pte_t pte_mkwrite(pte_t pte)
342+{
343+ pte_val(pte) |= (_PAGE_WRITE | _PAGE_DIRTY);
344+ return pte;
345+}
346+
347+static inline pte_t pte_wrprotect(pte_t pte)
348+{
349+ pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
350+ return pte;
351+}
352+
353+static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; }
354+
355+static inline pte_t pte_mkhuge(pte_t pte)
356+{
357+ pte_val(pte) |= _PAGE_HUGE;
358+ return pte;
359+}
360+
361+#if defined(CONFIG_ARCH_HAS_PTE_SPECIAL)
362+static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; }
363+static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }
364+#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
365+
366+#define pte_accessible pte_accessible
367+static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
368+{
369+ if (pte_val(a) & _PAGE_PRESENT)
370+ return true;
371+
372+ if ((pte_val(a) & _PAGE_PROTNONE) &&
373+ atomic_read(&mm->tlb_flush_pending))
374+ return true;
375+
376+ return false;
377+}
378+
379+/*
380+ * Conversion functions: convert a page and protection to a page entry,
381+ * and a page entry and page directory to the page they refer to.
382+ */
383+#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
384+
385+static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
386+{
387+ return __pte((pte_val(pte) & _PAGE_CHG_MASK) |
388+ (pgprot_val(newprot) & ~_PAGE_CHG_MASK));
389+}
390+
391+extern void __update_tlb(struct vm_area_struct *vma,
392+ unsigned long address, pte_t *ptep);
393+
394+static inline void update_mmu_cache(struct vm_area_struct *vma,
395+ unsigned long address, pte_t *ptep)
396+{
397+ __update_tlb(vma, address, ptep);
398+}
399+
400+static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
401+ unsigned long address, pmd_t *pmdp)
402+{
403+ __update_tlb(vma, address, (pte_t *)pmdp);
404+}
405+
406+#define kern_addr_valid(addr) (1)
407+
408+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
409+
410+/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/
411+#define pmdp_establish generic_pmdp_establish
412+
413+static inline int pmd_trans_huge(pmd_t pmd)
414+{
415+ return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd);
416+}
417+
418+static inline pmd_t pmd_mkhuge(pmd_t pmd)
419+{
420+ pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) |
421+ ((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT));
422+ pmd_val(pmd) |= _PAGE_HUGE;
423+
424+ return pmd;
425+}
426+
427+#define pmd_write pmd_write
428+static inline int pmd_write(pmd_t pmd)
429+{
430+ return !!(pmd_val(pmd) & _PAGE_WRITE);
431+}
432+
433+static inline pmd_t pmd_mkwrite(pmd_t pmd)
434+{
435+ pmd_val(pmd) |= (_PAGE_WRITE | _PAGE_DIRTY);
436+ return pmd;
437+}
438+
439+static inline pmd_t pmd_wrprotect(pmd_t pmd)
440+{
441+ pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
442+ return pmd;
443+}
444+
445+static inline int pmd_dirty(pmd_t pmd)
446+{
447+ return !!(pmd_val(pmd) & _PAGE_MODIFIED);
448+}
449+
450+static inline pmd_t pmd_mkclean(pmd_t pmd)
451+{
452+ pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
453+ return pmd;
454+}
455+
456+static inline pmd_t pmd_mkdirty(pmd_t pmd)
457+{
458+ pmd_val(pmd) |= (_PAGE_DIRTY | _PAGE_MODIFIED);
459+ return pmd;
460+}
461+
462+static inline int pmd_young(pmd_t pmd)
463+{
464+ return !!(pmd_val(pmd) & _PAGE_ACCESSED);
465+}
466+
467+static inline pmd_t pmd_mkold(pmd_t pmd)
468+{
469+ pmd_val(pmd) &= ~_PAGE_ACCESSED;
470+ return pmd;
471+}
472+
473+static inline pmd_t pmd_mkyoung(pmd_t pmd)
474+{
475+ pmd_val(pmd) |= _PAGE_ACCESSED;
476+ return pmd;
477+}
478+
479+static inline unsigned long pmd_pfn(pmd_t pmd)
480+{
481+ return (pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT;
482+}
483+
484+static inline struct page *pmd_page(pmd_t pmd)
485+{
486+ if (pmd_trans_huge(pmd))
487+ return pfn_to_page(pmd_pfn(pmd));
488+
489+ return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT);
490+}
491+
492+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
493+{
494+ pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) |
495+ (pgprot_val(newprot) & ~_HPAGE_CHG_MASK);
496+ return pmd;
497+}
498+
499+static inline pmd_t pmd_mkinvalid(pmd_t pmd)
500+{
501+ pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE);
502+
503+ return pmd;
504+}
505+
506+/*
507+ * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a
508+ * different prototype.
509+ */
510+#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
511+static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
512+ unsigned long address, pmd_t *pmdp)
513+{
514+ pmd_t old = *pmdp;
515+
516+ pmd_clear(pmdp);
517+
518+ return old;
519+}
520+
521+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
522+
523+/*
524+ * We provide our own get_unmapped area to cope with the virtual aliasing
525+ * constraints placed on us by the cache architecture.
526+ */
527+#define HAVE_ARCH_UNMAPPED_AREA
528+#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
529+
530+#endif /* !__ASSEMBLY__ */
531+
532+#endif /* _ASM_PGTABLE_H */
arch/loongarch/include/asm/shmparam.h+12−0
@@ -0,0 +1,12 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef _ASM_SHMPARAM_H
6+#define _ASM_SHMPARAM_H
7+
8+#define __ARCH_FORCE_SHMLBA 1
9+
10+#define SHMLBA SZ_64K /* attach addr a multiple of this */
11+
12+#endif /* _ASM_SHMPARAM_H */
arch/loongarch/include/asm/sparsemem.h+23−0
@@ -0,0 +1,23 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _LOONGARCH_SPARSEMEM_H
3+#define _LOONGARCH_SPARSEMEM_H
4+
5+#ifdef CONFIG_SPARSEMEM
6+
7+/*
8+ * SECTION_SIZE_BITS 2^N: how big each section will be
9+ * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space
10+ */
11+#define SECTION_SIZE_BITS 29 /* 2^29 = Largest Huge Page Size */
12+#define MAX_PHYSMEM_BITS 48
13+
14+#endif /* CONFIG_SPARSEMEM */
15+
16+#ifdef CONFIG_MEMORY_HOTPLUG
17+int memory_add_physaddr_to_nid(u64 addr);
18+#define memory_add_physaddr_to_nid memory_add_physaddr_to_nid
19+#endif
20+
21+#define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS)
22+
23+#endif /* _LOONGARCH_SPARSEMEM_H */
arch/loongarch/include/asm/tlb.h+180−0
@@ -0,0 +1,180 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef __ASM_TLB_H
6+#define __ASM_TLB_H
7+
8+#include <linux/mm_types.h>
9+#include <asm/cpu-features.h>
10+#include <asm/loongarch.h>
11+
12+/*
13+ * TLB Invalidate Flush
14+ */
15+static inline void tlbclr(void)
16+{
17+ __asm__ __volatile__("tlbclr");
18+}
19+
20+static inline void tlbflush(void)
21+{
22+ __asm__ __volatile__("tlbflush");
23+}
24+
25+/*
26+ * TLB R/W operations.
27+ */
28+static inline void tlb_probe(void)
29+{
30+ __asm__ __volatile__("tlbsrch");
31+}
32+
33+static inline void tlb_read(void)
34+{
35+ __asm__ __volatile__("tlbrd");
36+}
37+
38+static inline void tlb_write_indexed(void)
39+{
40+ __asm__ __volatile__("tlbwr");
41+}
42+
43+static inline void tlb_write_random(void)
44+{
45+ __asm__ __volatile__("tlbfill");
46+}
47+
48+enum invtlb_ops {
49+ /* Invalid all tlb */
50+ INVTLB_ALL = 0x0,
51+ /* Invalid current tlb */
52+ INVTLB_CURRENT_ALL = 0x1,
53+ /* Invalid all global=1 lines in current tlb */
54+ INVTLB_CURRENT_GTRUE = 0x2,
55+ /* Invalid all global=0 lines in current tlb */
56+ INVTLB_CURRENT_GFALSE = 0x3,
57+ /* Invalid global=0 and matched asid lines in current tlb */
58+ INVTLB_GFALSE_AND_ASID = 0x4,
59+ /* Invalid addr with global=0 and matched asid in current tlb */
60+ INVTLB_ADDR_GFALSE_AND_ASID = 0x5,
61+ /* Invalid addr with global=1 or matched asid in current tlb */
62+ INVTLB_ADDR_GTRUE_OR_ASID = 0x6,
63+ /* Invalid matched gid in guest tlb */
64+ INVGTLB_GID = 0x9,
65+ /* Invalid global=1, matched gid in guest tlb */
66+ INVGTLB_GID_GTRUE = 0xa,
67+ /* Invalid global=0, matched gid in guest tlb */
68+ INVGTLB_GID_GFALSE = 0xb,
69+ /* Invalid global=0, matched gid and asid in guest tlb */
70+ INVGTLB_GID_GFALSE_ASID = 0xc,
71+ /* Invalid global=0 , matched gid, asid and addr in guest tlb */
72+ INVGTLB_GID_GFALSE_ASID_ADDR = 0xd,
73+ /* Invalid global=1 , matched gid, asid and addr in guest tlb */
74+ INVGTLB_GID_GTRUE_ASID_ADDR = 0xe,
75+ /* Invalid all gid gva-->gpa guest tlb */
76+ INVGTLB_ALLGID_GVA_TO_GPA = 0x10,
77+ /* Invalid all gid gpa-->hpa tlb */
78+ INVTLB_ALLGID_GPA_TO_HPA = 0x11,
79+ /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */
80+ INVTLB_ALLGID = 0x12,
81+ /* Invalid matched gid gva-->gpa guest tlb */
82+ INVGTLB_GID_GVA_TO_GPA = 0x13,
83+ /* Invalid matched gid gpa-->hpa tlb */
84+ INVTLB_GID_GPA_TO_HPA = 0x14,
85+ /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */
86+ INVTLB_GID_ALL = 0x15,
87+ /* Invalid matched gid and addr gpa-->hpa tlb */
88+ INVTLB_GID_ADDR = 0x16,
89+};
90+
91+/*
92+ * invtlb op info addr
93+ * (0x1 << 26) | (0x24 << 20) | (0x13 << 15) |
94+ * (addr << 10) | (info << 5) | op
95+ */
96+static inline void invtlb(u32 op, u32 info, u64 addr)
97+{
98+ __asm__ __volatile__(
99+ "parse_r addr,%0\n\t"
100+ "parse_r info,%1\n\t"
101+ ".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t"
102+ :
103+ : "r"(addr), "r"(info), "i"(op)
104+ :
105+ );
106+}
107+
108+static inline void invtlb_addr(u32 op, u32 info, u64 addr)
109+{
110+ __asm__ __volatile__(
111+ "parse_r addr,%0\n\t"
112+ ".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t"
113+ :
114+ : "r"(addr), "i"(op)
115+ :
116+ );
117+}
118+
119+static inline void invtlb_info(u32 op, u32 info, u64 addr)
120+{
121+ __asm__ __volatile__(
122+ "parse_r info,%0\n\t"
123+ ".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t"
124+ :
125+ : "r"(info), "i"(op)
126+ :
127+ );
128+}
129+
130+static inline void invtlb_all(u32 op, u32 info, u64 addr)
131+{
132+ __asm__ __volatile__(
133+ ".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t"
134+ :
135+ : "i"(op)
136+ :
137+ );
138+}
139+
140+/*
141+ * LoongArch doesn't need any special per-pte or per-vma handling, except
142+ * we need to flush cache for area to be unmapped.
143+ */
144+#define tlb_start_vma(tlb, vma) \
145+ do { \
146+ if (!(tlb)->fullmm) \
147+ flush_cache_range(vma, vma->vm_start, vma->vm_end); \
148+ } while (0)
149+#define tlb_end_vma(tlb, vma) do { } while (0)
150+#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
151+
152+static void tlb_flush(struct mmu_gather *tlb);
153+
154+#define tlb_flush tlb_flush
155+#include <asm-generic/tlb.h>
156+
157+static inline void tlb_flush(struct mmu_gather *tlb)
158+{
159+ struct vm_area_struct vma;
160+
161+ vma.vm_mm = tlb->mm;
162+ vma.vm_flags = 0;
163+ if (tlb->fullmm) {
164+ flush_tlb_mm(tlb->mm);
165+ return;
166+ }
167+
168+ flush_tlb_range(&vma, tlb->start, tlb->end);
169+}
170+
171+extern void handle_tlb_load(void);
172+extern void handle_tlb_store(void);
173+extern void handle_tlb_modify(void);
174+extern void handle_tlb_refill(void);
175+extern void handle_tlb_protect(void);
176+
177+extern void dump_tlb_all(void);
178+extern void dump_tlb_regs(void);
179+
180+#endif /* __ASM_TLB_H */
arch/loongarch/include/asm/tlbflush.h+35−0
@@ -0,0 +1,35 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#ifndef __ASM_TLBFLUSH_H
6+#define __ASM_TLBFLUSH_H
7+
8+#include <linux/mm.h>
9+
10+/*
11+ * TLB flushing:
12+ *
13+ * - flush_tlb_all() flushes all processes TLB entries
14+ * - flush_tlb_mm(mm) flushes the specified mm context TLB entries
15+ * - flush_tlb_page(vma, vmaddr) flushes one page
16+ * - flush_tlb_range(vma, start, end) flushes a range of pages
17+ * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
18+ */
19+extern void local_flush_tlb_all(void);
20+extern void local_flush_tlb_user(void);
21+extern void local_flush_tlb_kernel(void);
22+extern void local_flush_tlb_mm(struct mm_struct *mm);
23+extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
24+extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end);
25+extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
26+extern void local_flush_tlb_one(unsigned long vaddr);
27+
28+#define flush_tlb_all() local_flush_tlb_all()
29+#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
30+#define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end)
31+#define flush_tlb_kernel_range(vmaddr, end) local_flush_tlb_kernel_range(vmaddr, end)
32+#define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page)
33+#define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr)
34+
35+#endif /* __ASM_TLBFLUSH_H */
arch/loongarch/include/asm/vmalloc.h+4−0
@@ -0,0 +1,4 @@
1+#ifndef _ASM_LOONGARCH_VMALLOC_H
2+#define _ASM_LOONGARCH_VMALLOC_H
3+
4+#endif /* _ASM_LOONGARCH_VMALLOC_H */
arch/loongarch/mm/cache.c+141−0
@@ -0,0 +1,141 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ *
5+ * Derived from MIPS:
6+ * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7+ * Copyright (C) 2007 MIPS Technologies, Inc.
8+ */
9+#include <linux/export.h>
10+#include <linux/fcntl.h>
11+#include <linux/fs.h>
12+#include <linux/highmem.h>
13+#include <linux/kernel.h>
14+#include <linux/linkage.h>
15+#include <linux/mm.h>
16+#include <linux/sched.h>
17+#include <linux/syscalls.h>
18+
19+#include <asm/cacheflush.h>
20+#include <asm/cpu.h>
21+#include <asm/cpu-features.h>
22+#include <asm/dma.h>
23+#include <asm/loongarch.h>
24+#include <asm/processor.h>
25+#include <asm/setup.h>
26+
27+/*
28+ * LoongArch maintains ICache/DCache coherency by hardware,
29+ * we just need "ibar" to avoid instruction hazard here.
30+ */
31+void local_flush_icache_range(unsigned long start, unsigned long end)
32+{
33+ asm volatile ("\tibar 0\n"::);
34+}
35+EXPORT_SYMBOL(local_flush_icache_range);
36+
37+void cache_error_setup(void)
38+{
39+ extern char __weak except_vec_cex;
40+ set_merr_handler(0x0, &except_vec_cex, 0x80);
41+}
42+
43+static unsigned long icache_size __read_mostly;
44+static unsigned long dcache_size __read_mostly;
45+static unsigned long vcache_size __read_mostly;
46+static unsigned long scache_size __read_mostly;
47+
48+static char *way_string[] = { NULL, "direct mapped", "2-way",
49+ "3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
50+ "9-way", "10-way", "11-way", "12-way",
51+ "13-way", "14-way", "15-way", "16-way",
52+};
53+
54+static void probe_pcache(void)
55+{
56+ struct cpuinfo_loongarch *c = &current_cpu_data;
57+ unsigned int lsize, sets, ways;
58+ unsigned int config;
59+
60+ config = read_cpucfg(LOONGARCH_CPUCFG17);
61+ lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE);
62+ sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS);
63+ ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1;
64+
65+ c->icache.linesz = lsize;
66+ c->icache.sets = sets;
67+ c->icache.ways = ways;
68+ icache_size = sets * ways * lsize;
69+ c->icache.waysize = icache_size / c->icache.ways;
70+
71+ config = read_cpucfg(LOONGARCH_CPUCFG18);
72+ lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE);
73+ sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS);
74+ ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1;
75+
76+ c->dcache.linesz = lsize;
77+ c->dcache.sets = sets;
78+ c->dcache.ways = ways;
79+ dcache_size = sets * ways * lsize;
80+ c->dcache.waysize = dcache_size / c->dcache.ways;
81+
82+ c->options |= LOONGARCH_CPU_PREFETCH;
83+
84+ pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
85+ icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz);
86+
87+ pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
88+ dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz);
89+}
90+
91+static void probe_vcache(void)
92+{
93+ struct cpuinfo_loongarch *c = &current_cpu_data;
94+ unsigned int lsize, sets, ways;
95+ unsigned int config;
96+
97+ config = read_cpucfg(LOONGARCH_CPUCFG19);
98+ lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE);
99+ sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS);
100+ ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1;
101+
102+ c->vcache.linesz = lsize;
103+ c->vcache.sets = sets;
104+ c->vcache.ways = ways;
105+ vcache_size = lsize * sets * ways;
106+ c->vcache.waysize = vcache_size / c->vcache.ways;
107+
108+ pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
109+ vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
110+}
111+
112+static void probe_scache(void)
113+{
114+ struct cpuinfo_loongarch *c = &current_cpu_data;
115+ unsigned int lsize, sets, ways;
116+ unsigned int config;
117+
118+ config = read_cpucfg(LOONGARCH_CPUCFG20);
119+ lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE);
120+ sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS);
121+ ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1;
122+
123+ c->scache.linesz = lsize;
124+ c->scache.sets = sets;
125+ c->scache.ways = ways;
126+ /* 4 cores. scaches are shared */
127+ scache_size = lsize * sets * ways;
128+ c->scache.waysize = scache_size / c->scache.ways;
129+
130+ pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
131+ scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
132+}
133+
134+void cpu_cache_init(void)
135+{
136+ probe_pcache();
137+ probe_vcache();
138+ probe_scache();
139+
140+ shm_align_mask = PAGE_SIZE - 1;
141+}
arch/loongarch/mm/extable.c+22−0
@@ -0,0 +1,22 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/extable.h>
6+#include <linux/spinlock.h>
7+#include <asm/branch.h>
8+#include <linux/uaccess.h>
9+
10+int fixup_exception(struct pt_regs *regs)
11+{
12+ const struct exception_table_entry *fixup;
13+
14+ fixup = search_exception_tables(exception_era(regs));
15+ if (fixup) {
16+ regs->csr_era = fixup->fixup;
17+
18+ return 1;
19+ }
20+
21+ return 0;
22+}
arch/loongarch/mm/fault.c+261−0
@@ -0,0 +1,261 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ *
5+ * Derived from MIPS:
6+ * Copyright (C) 1995 - 2000 by Ralf Baechle
7+ */
8+#include <linux/context_tracking.h>
9+#include <linux/signal.h>
10+#include <linux/sched.h>
11+#include <linux/interrupt.h>
12+#include <linux/kernel.h>
13+#include <linux/entry-common.h>
14+#include <linux/errno.h>
15+#include <linux/string.h>
16+#include <linux/types.h>
17+#include <linux/ptrace.h>
18+#include <linux/ratelimit.h>
19+#include <linux/mman.h>
20+#include <linux/mm.h>
21+#include <linux/smp.h>
22+#include <linux/kdebug.h>
23+#include <linux/kprobes.h>
24+#include <linux/perf_event.h>
25+#include <linux/uaccess.h>
26+
27+#include <asm/branch.h>
28+#include <asm/mmu_context.h>
29+#include <asm/ptrace.h>
30+
31+int show_unhandled_signals = 1;
32+
33+static void __kprobes no_context(struct pt_regs *regs, unsigned long address)
34+{
35+ const int field = sizeof(unsigned long) * 2;
36+
37+ /* Are we prepared to handle this kernel fault? */
38+ if (fixup_exception(regs))
39+ return;
40+
41+ /*
42+ * Oops. The kernel tried to access some bad page. We'll have to
43+ * terminate things with extreme prejudice.
44+ */
45+ bust_spinlocks(1);
46+
47+ pr_alert("CPU %d Unable to handle kernel paging request at "
48+ "virtual address %0*lx, era == %0*lx, ra == %0*lx\n",
49+ raw_smp_processor_id(), field, address, field, regs->csr_era,
50+ field, regs->regs[1]);
51+ die("Oops", regs);
52+}
53+
54+static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address)
55+{
56+ /*
57+ * We ran out of memory, call the OOM killer, and return the userspace
58+ * (which will retry the fault, or kill us if we got oom-killed).
59+ */
60+ if (!user_mode(regs)) {
61+ no_context(regs, address);
62+ return;
63+ }
64+ pagefault_out_of_memory();
65+}
66+
67+static void __kprobes do_sigbus(struct pt_regs *regs,
68+ unsigned long write, unsigned long address, int si_code)
69+{
70+ /* Kernel mode? Handle exceptions or die */
71+ if (!user_mode(regs)) {
72+ no_context(regs, address);
73+ return;
74+ }
75+
76+ /*
77+ * Send a sigbus, regardless of whether we were in kernel
78+ * or user mode.
79+ */
80+ current->thread.csr_badvaddr = address;
81+ current->thread.trap_nr = read_csr_excode();
82+ force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
83+}
84+
85+static void __kprobes do_sigsegv(struct pt_regs *regs,
86+ unsigned long write, unsigned long address, int si_code)
87+{
88+ const int field = sizeof(unsigned long) * 2;
89+ static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
90+
91+ /* Kernel mode? Handle exceptions or die */
92+ if (!user_mode(regs)) {
93+ no_context(regs, address);
94+ return;
95+ }
96+
97+ /* User mode accesses just cause a SIGSEGV */
98+ current->thread.csr_badvaddr = address;
99+ if (!write)
100+ current->thread.error_code = 1;
101+ else
102+ current->thread.error_code = 2;
103+ current->thread.trap_nr = read_csr_excode();
104+
105+ if (show_unhandled_signals &&
106+ unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) {
107+ pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n",
108+ current->comm,
109+ write ? "write access to" : "read access from",
110+ field, address);
111+ pr_info("era = %0*lx in", field,
112+ (unsigned long) regs->csr_era);
113+ print_vma_addr(KERN_CONT " ", regs->csr_era);
114+ pr_cont("\n");
115+ pr_info("ra = %0*lx in", field,
116+ (unsigned long) regs->regs[1]);
117+ print_vma_addr(KERN_CONT " ", regs->regs[1]);
118+ pr_cont("\n");
119+ }
120+ force_sig_fault(SIGSEGV, si_code, (void __user *)address);
121+}
122+
123+/*
124+ * This routine handles page faults. It determines the address,
125+ * and the problem, and then passes it off to one of the appropriate
126+ * routines.
127+ */
128+static void __kprobes __do_page_fault(struct pt_regs *regs,
129+ unsigned long write, unsigned long address)
130+{
131+ int si_code = SEGV_MAPERR;
132+ unsigned int flags = FAULT_FLAG_DEFAULT;
133+ struct task_struct *tsk = current;
134+ struct mm_struct *mm = tsk->mm;
135+ struct vm_area_struct *vma = NULL;
136+ vm_fault_t fault;
137+
138+ /*
139+ * We fault-in kernel-space virtual memory on-demand. The
140+ * 'reference' page table is init_mm.pgd.
141+ *
142+ * NOTE! We MUST NOT take any locks for this case. We may
143+ * be in an interrupt or a critical region, and should
144+ * only copy the information from the master page table,
145+ * nothing more.
146+ */
147+ if (address & __UA_LIMIT) {
148+ if (!user_mode(regs))
149+ no_context(regs, address);
150+ else
151+ do_sigsegv(regs, write, address, si_code);
152+ return;
153+ }
154+
155+ /*
156+ * If we're in an interrupt or have no user
157+ * context, we must not take the fault..
158+ */
159+ if (faulthandler_disabled() || !mm) {
160+ do_sigsegv(regs, write, address, si_code);
161+ return;
162+ }
163+
164+ if (user_mode(regs))
165+ flags |= FAULT_FLAG_USER;
166+
167+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
168+retry:
169+ mmap_read_lock(mm);
170+ vma = find_vma(mm, address);
171+ if (!vma)
172+ goto bad_area;
173+ if (vma->vm_start <= address)
174+ goto good_area;
175+ if (!(vma->vm_flags & VM_GROWSDOWN))
176+ goto bad_area;
177+ if (!expand_stack(vma, address))
178+ goto good_area;
179+/*
180+ * Something tried to access memory that isn't in our memory map..
181+ * Fix it, but check if it's kernel or user first..
182+ */
183+bad_area:
184+ mmap_read_unlock(mm);
185+ do_sigsegv(regs, write, address, si_code);
186+ return;
187+
188+/*
189+ * Ok, we have a good vm_area for this memory access, so
190+ * we can handle it..
191+ */
192+good_area:
193+ si_code = SEGV_ACCERR;
194+
195+ if (write) {
196+ flags |= FAULT_FLAG_WRITE;
197+ if (!(vma->vm_flags & VM_WRITE))
198+ goto bad_area;
199+ } else {
200+ if (!(vma->vm_flags & VM_READ) && address != exception_era(regs))
201+ goto bad_area;
202+ if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs))
203+ goto bad_area;
204+ }
205+
206+ /*
207+ * If for any reason at all we couldn't handle the fault,
208+ * make sure we exit gracefully rather than endlessly redo
209+ * the fault.
210+ */
211+ fault = handle_mm_fault(vma, address, flags, regs);
212+
213+ if (fault_signal_pending(fault, regs)) {
214+ if (!user_mode(regs))
215+ no_context(regs, address);
216+ return;
217+ }
218+
219+ if (unlikely(fault & VM_FAULT_RETRY)) {
220+ flags |= FAULT_FLAG_TRIED;
221+
222+ /*
223+ * No need to mmap_read_unlock(mm) as we would
224+ * have already released it in __lock_page_or_retry
225+ * in mm/filemap.c.
226+ */
227+ goto retry;
228+ }
229+ if (unlikely(fault & VM_FAULT_ERROR)) {
230+ mmap_read_unlock(mm);
231+ if (fault & VM_FAULT_OOM) {
232+ do_out_of_memory(regs, address);
233+ return;
234+ } else if (fault & VM_FAULT_SIGSEGV) {
235+ do_sigsegv(regs, write, address, si_code);
236+ return;
237+ } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
238+ do_sigbus(regs, write, address, si_code);
239+ return;
240+ }
241+ BUG();
242+ }
243+
244+ mmap_read_unlock(mm);
245+}
246+
247+asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
248+ unsigned long write, unsigned long address)
249+{
250+ irqentry_state_t state = irqentry_enter(regs);
251+
252+ /* Enable interrupt if enabled in parent context */
253+ if (likely(regs->csr_prmd & CSR_PRMD_PIE))
254+ local_irq_enable();
255+
256+ __do_page_fault(regs, write, address);
257+
258+ local_irq_disable();
259+
260+ irqentry_exit(regs, state);
261+}
arch/loongarch/mm/hugetlbpage.c+87−0
@@ -0,0 +1,87 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+
6+#include <linux/fs.h>
7+#include <linux/mm.h>
8+#include <linux/hugetlb.h>
9+#include <linux/pagemap.h>
10+#include <linux/err.h>
11+#include <linux/sysctl.h>
12+#include <asm/mman.h>
13+#include <asm/tlb.h>
14+#include <asm/tlbflush.h>
15+
16+pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
17+ unsigned long addr, unsigned long sz)
18+{
19+ pgd_t *pgd;
20+ p4d_t *p4d;
21+ pud_t *pud;
22+ pte_t *pte = NULL;
23+
24+ pgd = pgd_offset(mm, addr);
25+ p4d = p4d_alloc(mm, pgd, addr);
26+ pud = pud_alloc(mm, p4d, addr);
27+ if (pud)
28+ pte = (pte_t *)pmd_alloc(mm, pud, addr);
29+
30+ return pte;
31+}
32+
33+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
34+ unsigned long sz)
35+{
36+ pgd_t *pgd;
37+ p4d_t *p4d;
38+ pud_t *pud;
39+ pmd_t *pmd = NULL;
40+
41+ pgd = pgd_offset(mm, addr);
42+ if (pgd_present(*pgd)) {
43+ p4d = p4d_offset(pgd, addr);
44+ if (p4d_present(*p4d)) {
45+ pud = pud_offset(p4d, addr);
46+ if (pud_present(*pud))
47+ pmd = pmd_offset(pud, addr);
48+ }
49+ }
50+ return (pte_t *) pmd;
51+}
52+
53+/*
54+ * This function checks for proper alignment of input addr and len parameters.
55+ */
56+int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
57+{
58+ if (len & ~HPAGE_MASK)
59+ return -EINVAL;
60+ if (addr & ~HPAGE_MASK)
61+ return -EINVAL;
62+ return 0;
63+}
64+
65+int pmd_huge(pmd_t pmd)
66+{
67+ return (pmd_val(pmd) & _PAGE_HUGE) != 0;
68+}
69+
70+int pud_huge(pud_t pud)
71+{
72+ return (pud_val(pud) & _PAGE_HUGE) != 0;
73+}
74+
75+uint64_t pmd_to_entrylo(unsigned long pmd_val)
76+{
77+ uint64_t val;
78+ /* PMD as PTE. Must be huge page */
79+ if (!pmd_huge(__pmd(pmd_val)))
80+ panic("%s", __func__);
81+
82+ val = pmd_val ^ _PAGE_HUGE;
83+ val |= ((val & _PAGE_HGLOBAL) >>
84+ (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT));
85+
86+ return val;
87+}
arch/loongarch/mm/init.c+165−0
@@ -0,0 +1,165 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/init.h>
6+#include <linux/export.h>
7+#include <linux/signal.h>
8+#include <linux/sched.h>
9+#include <linux/smp.h>
10+#include <linux/kernel.h>
11+#include <linux/errno.h>
12+#include <linux/string.h>
13+#include <linux/types.h>
14+#include <linux/pagemap.h>
15+#include <linux/memblock.h>
16+#include <linux/memremap.h>
17+#include <linux/mm.h>
18+#include <linux/mman.h>
19+#include <linux/highmem.h>
20+#include <linux/swap.h>
21+#include <linux/proc_fs.h>
22+#include <linux/pfn.h>
23+#include <linux/hardirq.h>
24+#include <linux/gfp.h>
25+#include <linux/initrd.h>
26+#include <linux/mmzone.h>
27+
28+#include <asm/asm-offsets.h>
29+#include <asm/bootinfo.h>
30+#include <asm/cpu.h>
31+#include <asm/dma.h>
32+#include <asm/mmu_context.h>
33+#include <asm/sections.h>
34+#include <asm/pgtable.h>
35+#include <asm/pgalloc.h>
36+#include <asm/tlb.h>
37+
38+/*
39+ * We have up to 8 empty zeroed pages so we can map one of the right colour
40+ * when needed. Since page is never written to after the initialization we
41+ * don't have to care about aliases on other CPUs.
42+ */
43+unsigned long empty_zero_page, zero_page_mask;
44+EXPORT_SYMBOL_GPL(empty_zero_page);
45+EXPORT_SYMBOL(zero_page_mask);
46+
47+void setup_zero_pages(void)
48+{
49+ unsigned int order, i;
50+ struct page *page;
51+
52+ order = 0;
53+
54+ empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
55+ if (!empty_zero_page)
56+ panic("Oh boy, that early out of memory?");
57+
58+ page = virt_to_page((void *)empty_zero_page);
59+ split_page(page, order);
60+ for (i = 0; i < (1 << order); i++, page++)
61+ mark_page_reserved(page);
62+
63+ zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
64+}
65+
66+void copy_user_highpage(struct page *to, struct page *from,
67+ unsigned long vaddr, struct vm_area_struct *vma)
68+{
69+ void *vfrom, *vto;
70+
71+ vto = kmap_atomic(to);
72+ vfrom = kmap_atomic(from);
73+ copy_page(vto, vfrom);
74+ kunmap_atomic(vfrom);
75+ kunmap_atomic(vto);
76+ /* Make sure this page is cleared on other CPU's too before using it */
77+ smp_wmb();
78+}
79+
80+int __ref page_is_ram(unsigned long pfn)
81+{
82+ unsigned long addr = PFN_PHYS(pfn);
83+
84+ return memblock_is_memory(addr) && !memblock_is_reserved(addr);
85+}
86+
87+void __init paging_init(void)
88+{
89+ unsigned long max_zone_pfns[MAX_NR_ZONES];
90+
91+#ifdef CONFIG_ZONE_DMA
92+ max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
93+#endif
94+#ifdef CONFIG_ZONE_DMA32
95+ max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
96+#endif
97+ max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
98+
99+ free_area_init(max_zone_pfns);
100+}
101+
102+void __init mem_init(void)
103+{
104+ max_mapnr = max_low_pfn;
105+ high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
106+
107+ memblock_free_all();
108+ setup_zero_pages(); /* Setup zeroed pages. */
109+}
110+
111+void __ref free_initmem(void)
112+{
113+ free_initmem_default(POISON_FREE_INITMEM);
114+}
115+
116+#ifdef CONFIG_MEMORY_HOTPLUG
117+int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params)
118+{
119+ unsigned long start_pfn = start >> PAGE_SHIFT;
120+ unsigned long nr_pages = size >> PAGE_SHIFT;
121+ int ret;
122+
123+ ret = __add_pages(nid, start_pfn, nr_pages, params);
124+
125+ if (ret)
126+ pr_warn("%s: Problem encountered in __add_pages() as ret=%d\n",
127+ __func__, ret);
128+
129+ return ret;
130+}
131+
132+#ifdef CONFIG_MEMORY_HOTREMOVE
133+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
134+{
135+ unsigned long start_pfn = start >> PAGE_SHIFT;
136+ unsigned long nr_pages = size >> PAGE_SHIFT;
137+ struct page *page = pfn_to_page(start_pfn);
138+
139+ /* With altmap the first mapped page is offset from @start */
140+ if (altmap)
141+ page += vmem_altmap_offset(altmap);
142+ __remove_pages(start_pfn, nr_pages, altmap);
143+}
144+#endif
145+#endif
146+
147+/*
148+ * Align swapper_pg_dir in to 64K, allows its address to be loaded
149+ * with a single LUI instruction in the TLB handlers. If we used
150+ * __aligned(64K), its size would get rounded up to the alignment
151+ * size, and waste space. So we place it in its own section and align
152+ * it in the linker script.
153+ */
154+pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
155+
156+pgd_t invalid_pg_dir[_PTRS_PER_PGD] __page_aligned_bss;
157+#ifndef __PAGETABLE_PUD_FOLDED
158+pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
159+#endif
160+#ifndef __PAGETABLE_PMD_FOLDED
161+pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
162+EXPORT_SYMBOL_GPL(invalid_pmd_table);
163+#endif
164+pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
165+EXPORT_SYMBOL(invalid_pte_table);
arch/loongarch/mm/ioremap.c+27−0
@@ -0,0 +1,27 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+
6+#include <asm/io.h>
7+
8+void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size)
9+{
10+ return ((void __iomem *)TO_CACHE(phys_addr));
11+}
12+
13+void __init early_iounmap(void __iomem *addr, unsigned long size)
14+{
15+
16+}
17+
18+void *early_memremap_ro(resource_size_t phys_addr, unsigned long size)
19+{
20+ return early_memremap(phys_addr, size);
21+}
22+
23+void *early_memremap_prot(resource_size_t phys_addr, unsigned long size,
24+ unsigned long prot_val)
25+{
26+ return early_memremap(phys_addr, size);
27+}
arch/loongarch/mm/maccess.c+10−0
@@ -0,0 +1,10 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+
3+#include <linux/uaccess.h>
4+#include <linux/kernel.h>
5+
6+bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
7+{
8+ /* highest bit set means kernel space */
9+ return (unsigned long)unsafe_src >> (BITS_PER_LONG - 1);
10+}
arch/loongarch/mm/mmap.c+125−0
@@ -0,0 +1,125 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/compiler.h>
6+#include <linux/elf-randomize.h>
7+#include <linux/errno.h>
8+#include <linux/mm.h>
9+#include <linux/mman.h>
10+#include <linux/export.h>
11+#include <linux/personality.h>
12+#include <linux/random.h>
13+#include <linux/sched/signal.h>
14+#include <linux/sched/mm.h>
15+
16+unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */
17+EXPORT_SYMBOL(shm_align_mask);
18+
19+#define COLOUR_ALIGN(addr, pgoff) \
20+ ((((addr) + shm_align_mask) & ~shm_align_mask) + \
21+ (((pgoff) << PAGE_SHIFT) & shm_align_mask))
22+
23+enum mmap_allocation_direction {UP, DOWN};
24+
25+static unsigned long arch_get_unmapped_area_common(struct file *filp,
26+ unsigned long addr0, unsigned long len, unsigned long pgoff,
27+ unsigned long flags, enum mmap_allocation_direction dir)
28+{
29+ struct mm_struct *mm = current->mm;
30+ struct vm_area_struct *vma;
31+ unsigned long addr = addr0;
32+ int do_color_align;
33+ struct vm_unmapped_area_info info;
34+
35+ if (unlikely(len > TASK_SIZE))
36+ return -ENOMEM;
37+
38+ if (flags & MAP_FIXED) {
39+ /* Even MAP_FIXED mappings must reside within TASK_SIZE */
40+ if (TASK_SIZE - len < addr)
41+ return -EINVAL;
42+
43+ /*
44+ * We do not accept a shared mapping if it would violate
45+ * cache aliasing constraints.
46+ */
47+ if ((flags & MAP_SHARED) &&
48+ ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
49+ return -EINVAL;
50+ return addr;
51+ }
52+
53+ do_color_align = 0;
54+ if (filp || (flags & MAP_SHARED))
55+ do_color_align = 1;
56+
57+ /* requesting a specific address */
58+ if (addr) {
59+ if (do_color_align)
60+ addr = COLOUR_ALIGN(addr, pgoff);
61+ else
62+ addr = PAGE_ALIGN(addr);
63+
64+ vma = find_vma(mm, addr);
65+ if (TASK_SIZE - len >= addr &&
66+ (!vma || addr + len <= vm_start_gap(vma)))
67+ return addr;
68+ }
69+
70+ info.length = len;
71+ info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
72+ info.align_offset = pgoff << PAGE_SHIFT;
73+
74+ if (dir == DOWN) {
75+ info.flags = VM_UNMAPPED_AREA_TOPDOWN;
76+ info.low_limit = PAGE_SIZE;
77+ info.high_limit = mm->mmap_base;
78+ addr = vm_unmapped_area(&info);
79+
80+ if (!(addr & ~PAGE_MASK))
81+ return addr;
82+
83+ /*
84+ * A failed mmap() very likely causes application failure,
85+ * so fall back to the bottom-up function here. This scenario
86+ * can happen with large stack limits and large mmap()
87+ * allocations.
88+ */
89+ }
90+
91+ info.flags = 0;
92+ info.low_limit = mm->mmap_base;
93+ info.high_limit = TASK_SIZE;
94+ return vm_unmapped_area(&info);
95+}
96+
97+unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0,
98+ unsigned long len, unsigned long pgoff, unsigned long flags)
99+{
100+ return arch_get_unmapped_area_common(filp,
101+ addr0, len, pgoff, flags, UP);
102+}
103+
104+/*
105+ * There is no need to export this but sched.h declares the function as
106+ * extern so making it static here results in an error.
107+ */
108+unsigned long arch_get_unmapped_area_topdown(struct file *filp,
109+ unsigned long addr0, unsigned long len, unsigned long pgoff,
110+ unsigned long flags)
111+{
112+ return arch_get_unmapped_area_common(filp,
113+ addr0, len, pgoff, flags, DOWN);
114+}
115+
116+int __virt_addr_valid(volatile void *kaddr)
117+{
118+ unsigned long vaddr = (unsigned long)kaddr;
119+
120+ if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base))
121+ return 0;
122+
123+ return pfn_valid(PFN_DOWN(virt_to_phys(kaddr)));
124+}
125+EXPORT_SYMBOL_GPL(__virt_addr_valid);
arch/loongarch/mm/page.S+84−0
@@ -0,0 +1,84 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/linkage.h>
6+#include <asm/asm.h>
7+#include <asm/export.h>
8+#include <asm/page.h>
9+#include <asm/regdef.h>
10+
11+ .align 5
12+SYM_FUNC_START(clear_page)
13+ lu12i.w t0, 1 << (PAGE_SHIFT - 12)
14+ add.d t0, t0, a0
15+1:
16+ st.d zero, a0, 0
17+ st.d zero, a0, 8
18+ st.d zero, a0, 16
19+ st.d zero, a0, 24
20+ st.d zero, a0, 32
21+ st.d zero, a0, 40
22+ st.d zero, a0, 48
23+ st.d zero, a0, 56
24+ addi.d a0, a0, 128
25+ st.d zero, a0, -64
26+ st.d zero, a0, -56
27+ st.d zero, a0, -48
28+ st.d zero, a0, -40
29+ st.d zero, a0, -32
30+ st.d zero, a0, -24
31+ st.d zero, a0, -16
32+ st.d zero, a0, -8
33+ bne t0, a0, 1b
34+
35+ jirl $r0, ra, 0
36+SYM_FUNC_END(clear_page)
37+EXPORT_SYMBOL(clear_page)
38+
39+.align 5
40+SYM_FUNC_START(copy_page)
41+ lu12i.w t8, 1 << (PAGE_SHIFT - 12)
42+ add.d t8, t8, a0
43+1:
44+ ld.d t0, a1, 0
45+ ld.d t1, a1, 8
46+ ld.d t2, a1, 16
47+ ld.d t3, a1, 24
48+ ld.d t4, a1, 32
49+ ld.d t5, a1, 40
50+ ld.d t6, a1, 48
51+ ld.d t7, a1, 56
52+
53+ st.d t0, a0, 0
54+ st.d t1, a0, 8
55+ ld.d t0, a1, 64
56+ ld.d t1, a1, 72
57+ st.d t2, a0, 16
58+ st.d t3, a0, 24
59+ ld.d t2, a1, 80
60+ ld.d t3, a1, 88
61+ st.d t4, a0, 32
62+ st.d t5, a0, 40
63+ ld.d t4, a1, 96
64+ ld.d t5, a1, 104
65+ st.d t6, a0, 48
66+ st.d t7, a0, 56
67+ ld.d t6, a1, 112
68+ ld.d t7, a1, 120
69+ addi.d a0, a0, 128
70+ addi.d a1, a1, 128
71+
72+ st.d t0, a0, -64
73+ st.d t1, a0, -56
74+ st.d t2, a0, -48
75+ st.d t3, a0, -40
76+ st.d t4, a0, -32
77+ st.d t5, a0, -24
78+ st.d t6, a0, -16
79+ st.d t7, a0, -8
80+
81+ bne t8, a0, 1b
82+ jirl $r0, ra, 0
83+SYM_FUNC_END(copy_page)
84+EXPORT_SYMBOL(copy_page)
arch/loongarch/mm/pgtable.c+130−0
@@ -0,0 +1,130 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/init.h>
6+#include <linux/export.h>
7+#include <linux/mm.h>
8+#include <asm/pgalloc.h>
9+#include <asm/pgtable.h>
10+#include <asm/tlbflush.h>
11+
12+pgd_t *pgd_alloc(struct mm_struct *mm)
13+{
14+ pgd_t *ret, *init;
15+
16+ ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER);
17+ if (ret) {
18+ init = pgd_offset(&init_mm, 0UL);
19+ pgd_init((unsigned long)ret);
20+ memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
21+ (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
22+ }
23+
24+ return ret;
25+}
26+EXPORT_SYMBOL_GPL(pgd_alloc);
27+
28+void pgd_init(unsigned long page)
29+{
30+ unsigned long *p, *end;
31+ unsigned long entry;
32+
33+#if !defined(__PAGETABLE_PUD_FOLDED)
34+ entry = (unsigned long)invalid_pud_table;
35+#elif !defined(__PAGETABLE_PMD_FOLDED)
36+ entry = (unsigned long)invalid_pmd_table;
37+#else
38+ entry = (unsigned long)invalid_pte_table;
39+#endif
40+
41+ p = (unsigned long *) page;
42+ end = p + PTRS_PER_PGD;
43+
44+ do {
45+ p[0] = entry;
46+ p[1] = entry;
47+ p[2] = entry;
48+ p[3] = entry;
49+ p[4] = entry;
50+ p += 8;
51+ p[-3] = entry;
52+ p[-2] = entry;
53+ p[-1] = entry;
54+ } while (p != end);
55+}
56+EXPORT_SYMBOL_GPL(pgd_init);
57+
58+#ifndef __PAGETABLE_PMD_FOLDED
59+void pmd_init(unsigned long addr, unsigned long pagetable)
60+{
61+ unsigned long *p, *end;
62+
63+ p = (unsigned long *) addr;
64+ end = p + PTRS_PER_PMD;
65+
66+ do {
67+ p[0] = pagetable;
68+ p[1] = pagetable;
69+ p[2] = pagetable;
70+ p[3] = pagetable;
71+ p[4] = pagetable;
72+ p += 8;
73+ p[-3] = pagetable;
74+ p[-2] = pagetable;
75+ p[-1] = pagetable;
76+ } while (p != end);
77+}
78+EXPORT_SYMBOL_GPL(pmd_init);
79+#endif
80+
81+#ifndef __PAGETABLE_PUD_FOLDED
82+void pud_init(unsigned long addr, unsigned long pagetable)
83+{
84+ unsigned long *p, *end;
85+
86+ p = (unsigned long *)addr;
87+ end = p + PTRS_PER_PUD;
88+
89+ do {
90+ p[0] = pagetable;
91+ p[1] = pagetable;
92+ p[2] = pagetable;
93+ p[3] = pagetable;
94+ p[4] = pagetable;
95+ p += 8;
96+ p[-3] = pagetable;
97+ p[-2] = pagetable;
98+ p[-1] = pagetable;
99+ } while (p != end);
100+}
101+#endif
102+
103+pmd_t mk_pmd(struct page *page, pgprot_t prot)
104+{
105+ pmd_t pmd;
106+
107+ pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
108+
109+ return pmd;
110+}
111+
112+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
113+ pmd_t *pmdp, pmd_t pmd)
114+{
115+ *pmdp = pmd;
116+ flush_tlb_all();
117+}
118+
119+void __init pagetable_init(void)
120+{
121+ /* Initialize the entire pgd. */
122+ pgd_init((unsigned long)swapper_pg_dir);
123+ pgd_init((unsigned long)invalid_pg_dir);
124+#ifndef __PAGETABLE_PUD_FOLDED
125+ pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table);
126+#endif
127+#ifndef __PAGETABLE_PMD_FOLDED
128+ pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
129+#endif
130+}
arch/loongarch/mm/tlb.c+282−0
@@ -0,0 +1,282 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <linux/init.h>
6+#include <linux/sched.h>
7+#include <linux/smp.h>
8+#include <linux/mm.h>
9+#include <linux/hugetlb.h>
10+#include <linux/export.h>
11+
12+#include <asm/cpu.h>
13+#include <asm/bootinfo.h>
14+#include <asm/mmu_context.h>
15+#include <asm/pgtable.h>
16+#include <asm/tlb.h>
17+
18+void local_flush_tlb_all(void)
19+{
20+ invtlb_all(INVTLB_CURRENT_ALL, 0, 0);
21+}
22+EXPORT_SYMBOL(local_flush_tlb_all);
23+
24+void local_flush_tlb_user(void)
25+{
26+ invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0);
27+}
28+EXPORT_SYMBOL(local_flush_tlb_user);
29+
30+void local_flush_tlb_kernel(void)
31+{
32+ invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0);
33+}
34+EXPORT_SYMBOL(local_flush_tlb_kernel);
35+
36+/*
37+ * All entries common to a mm share an asid. To effectively flush
38+ * these entries, we just bump the asid.
39+ */
40+void local_flush_tlb_mm(struct mm_struct *mm)
41+{
42+ int cpu;
43+
44+ preempt_disable();
45+
46+ cpu = smp_processor_id();
47+
48+ if (asid_valid(mm, cpu))
49+ drop_mmu_context(mm, cpu);
50+ else
51+ cpumask_clear_cpu(cpu, mm_cpumask(mm));
52+
53+ preempt_enable();
54+}
55+
56+void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
57+ unsigned long end)
58+{
59+ struct mm_struct *mm = vma->vm_mm;
60+ int cpu = smp_processor_id();
61+
62+ if (asid_valid(mm, cpu)) {
63+ unsigned long size, flags;
64+
65+ local_irq_save(flags);
66+ start = round_down(start, PAGE_SIZE << 1);
67+ end = round_up(end, PAGE_SIZE << 1);
68+ size = (end - start) >> (PAGE_SHIFT + 1);
69+ if (size <= (current_cpu_data.tlbsizestlbsets ?
70+ current_cpu_data.tlbsize / 8 :
71+ current_cpu_data.tlbsize / 2)) {
72+ int asid = cpu_asid(cpu, mm);
73+
74+ while (start < end) {
75+ invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start);
76+ start += (PAGE_SIZE << 1);
77+ }
78+ } else {
79+ drop_mmu_context(mm, cpu);
80+ }
81+ local_irq_restore(flags);
82+ } else {
83+ cpumask_clear_cpu(cpu, mm_cpumask(mm));
84+ }
85+}
86+
87+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
88+{
89+ unsigned long size, flags;
90+
91+ local_irq_save(flags);
92+ size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
93+ size = (size + 1) >> 1;
94+ if (size <= (current_cpu_data.tlbsizestlbsets ?
95+ current_cpu_data.tlbsize / 8 :
96+ current_cpu_data.tlbsize / 2)) {
97+
98+ start &= (PAGE_MASK << 1);
99+ end += ((PAGE_SIZE << 1) - 1);
100+ end &= (PAGE_MASK << 1);
101+
102+ while (start < end) {
103+ invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start);
104+ start += (PAGE_SIZE << 1);
105+ }
106+ } else {
107+ local_flush_tlb_kernel();
108+ }
109+ local_irq_restore(flags);
110+}
111+
112+void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
113+{
114+ int cpu = smp_processor_id();
115+
116+ if (asid_valid(vma->vm_mm, cpu)) {
117+ int newpid;
118+
119+ newpid = cpu_asid(cpu, vma->vm_mm);
120+ page &= (PAGE_MASK << 1);
121+ invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page);
122+ } else {
123+ cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm));
124+ }
125+}
126+
127+/*
128+ * This one is only used for pages with the global bit set so we don't care
129+ * much about the ASID.
130+ */
131+void local_flush_tlb_one(unsigned long page)
132+{
133+ page &= (PAGE_MASK << 1);
134+ invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page);
135+}
136+
137+static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
138+{
139+#ifdef CONFIG_HUGETLB_PAGE
140+ int idx;
141+ unsigned long lo;
142+ unsigned long flags;
143+
144+ local_irq_save(flags);
145+
146+ address &= (PAGE_MASK << 1);
147+ write_csr_entryhi(address);
148+ tlb_probe();
149+ idx = read_csr_tlbidx();
150+ write_csr_pagesize(PS_HUGE_SIZE);
151+ lo = pmd_to_entrylo(pte_val(*ptep));
152+ write_csr_entrylo0(lo);
153+ write_csr_entrylo1(lo + (HPAGE_SIZE >> 1));
154+
155+ if (idx < 0)
156+ tlb_write_random();
157+ else
158+ tlb_write_indexed();
159+ write_csr_pagesize(PS_DEFAULT_SIZE);
160+
161+ local_irq_restore(flags);
162+#endif
163+}
164+
165+void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
166+{
167+ int idx;
168+ unsigned long flags;
169+
170+ /*
171+ * Handle debugger faulting in for debugee.
172+ */
173+ if (current->active_mm != vma->vm_mm)
174+ return;
175+
176+ if (pte_val(*ptep) & _PAGE_HUGE) {
177+ __update_hugetlb(vma, address, ptep);
178+ return;
179+ }
180+
181+ local_irq_save(flags);
182+
183+ if ((unsigned long)ptep & sizeof(pte_t))
184+ ptep--;
185+
186+ address &= (PAGE_MASK << 1);
187+ write_csr_entryhi(address);
188+ tlb_probe();
189+ idx = read_csr_tlbidx();
190+ write_csr_pagesize(PS_DEFAULT_SIZE);
191+ write_csr_entrylo0(pte_val(*ptep++));
192+ write_csr_entrylo1(pte_val(*ptep));
193+ if (idx < 0)
194+ tlb_write_random();
195+ else
196+ tlb_write_indexed();
197+
198+ local_irq_restore(flags);
199+}
200+
201+static void setup_ptwalker(void)
202+{
203+ unsigned long pwctl0, pwctl1;
204+ unsigned long pgd_i = 0, pgd_w = 0;
205+ unsigned long pud_i = 0, pud_w = 0;
206+ unsigned long pmd_i = 0, pmd_w = 0;
207+ unsigned long pte_i = 0, pte_w = 0;
208+
209+ pgd_i = PGDIR_SHIFT;
210+ pgd_w = PAGE_SHIFT - 3;
211+#if CONFIG_PGTABLE_LEVELS > 3
212+ pud_i = PUD_SHIFT;
213+ pud_w = PAGE_SHIFT - 3;
214+#endif
215+#if CONFIG_PGTABLE_LEVELS > 2
216+ pmd_i = PMD_SHIFT;
217+ pmd_w = PAGE_SHIFT - 3;
218+#endif
219+ pte_i = PAGE_SHIFT;
220+ pte_w = PAGE_SHIFT - 3;
221+
222+ pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
223+ pwctl1 = pgd_i | pgd_w << 6;
224+
225+ csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
226+ csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
227+ csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
228+ csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL);
229+ csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID);
230+}
231+
232+static void output_pgtable_bits_defines(void)
233+{
234+#define pr_define(fmt, ...) \
235+ pr_debug("#define " fmt, ##__VA_ARGS__)
236+
237+ pr_debug("#include <asm/asm.h>\n");
238+ pr_debug("#include <asm/regdef.h>\n");
239+ pr_debug("\n");
240+
241+ pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
242+ pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
243+ pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
244+ pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
245+ pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
246+ pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
247+ pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
248+ pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
249+ pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
250+ pr_debug("\n");
251+}
252+
253+void setup_tlb_handler(void)
254+{
255+ static int run_once = 0;
256+
257+ setup_ptwalker();
258+ output_pgtable_bits_defines();
259+
260+ /* The tlb handlers are generated only once */
261+ if (!run_once) {
262+ memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
263+ local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
264+ set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
265+ set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
266+ set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
267+ set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
268+ set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
269+ set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
270+ set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
271+ run_once++;
272+ }
273+}
274+
275+void tlb_init(void)
276+{
277+ write_csr_pagesize(PS_DEFAULT_SIZE);
278+ write_csr_stlbpgsize(PS_DEFAULT_SIZE);
279+ write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE);
280+ setup_tlb_handler();
281+ local_flush_tlb_all();
282+}
arch/loongarch/mm/tlbex.S+477−0
@@ -0,0 +1,477 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4+ */
5+#include <asm/asm.h>
6+#include <asm/export.h>
7+#include <asm/loongarch.h>
8+#include <asm/page.h>
9+#include <asm/pgtable.h>
10+#include <asm/regdef.h>
11+#include <asm/stackframe.h>
12+
13+ .macro tlb_do_page_fault, write
14+ SYM_FUNC_START(tlb_do_page_fault_\write)
15+ SAVE_ALL
16+ csrrd a2, LOONGARCH_CSR_BADV
17+ move a0, sp
18+ REG_S a2, sp, PT_BVADDR
19+ li.w a1, \write
20+ la.abs t0, do_page_fault
21+ jirl ra, t0, 0
22+ RESTORE_ALL_AND_RET
23+ SYM_FUNC_END(tlb_do_page_fault_\write)
24+ .endm
25+
26+ tlb_do_page_fault 0
27+ tlb_do_page_fault 1
28+
29+SYM_FUNC_START(handle_tlb_protect)
30+ BACKUP_T0T1
31+ SAVE_ALL
32+ move a0, sp
33+ move a1, zero
34+ csrrd a2, LOONGARCH_CSR_BADV
35+ REG_S a2, sp, PT_BVADDR
36+ la.abs t0, do_page_fault
37+ jirl ra, t0, 0
38+ RESTORE_ALL_AND_RET
39+SYM_FUNC_END(handle_tlb_protect)
40+
41+SYM_FUNC_START(handle_tlb_load)
42+ csrwr t0, EXCEPTION_KS0
43+ csrwr t1, EXCEPTION_KS1
44+ csrwr ra, EXCEPTION_KS2
45+
46+ /*
47+ * The vmalloc handling is not in the hotpath.
48+ */
49+ csrrd t0, LOONGARCH_CSR_BADV
50+ blt t0, $r0, vmalloc_load
51+ csrrd t1, LOONGARCH_CSR_PGDL
52+
53+vmalloc_done_load:
54+ /* Get PGD offset in bytes */
55+ srli.d t0, t0, PGDIR_SHIFT
56+ andi t0, t0, (PTRS_PER_PGD - 1)
57+ slli.d t0, t0, 3
58+ add.d t1, t1, t0
59+#if CONFIG_PGTABLE_LEVELS > 3
60+ csrrd t0, LOONGARCH_CSR_BADV
61+ ld.d t1, t1, 0
62+ srli.d t0, t0, PUD_SHIFT
63+ andi t0, t0, (PTRS_PER_PUD - 1)
64+ slli.d t0, t0, 3
65+ add.d t1, t1, t0
66+#endif
67+#if CONFIG_PGTABLE_LEVELS > 2
68+ csrrd t0, LOONGARCH_CSR_BADV
69+ ld.d t1, t1, 0
70+ srli.d t0, t0, PMD_SHIFT
71+ andi t0, t0, (PTRS_PER_PMD - 1)
72+ slli.d t0, t0, 3
73+ add.d t1, t1, t0
74+#endif
75+ ld.d ra, t1, 0
76+
77+ /*
78+ * For huge tlb entries, pmde doesn't contain an address but
79+ * instead contains the tlb pte. Check the PAGE_HUGE bit and
80+ * see if we need to jump to huge tlb processing.
81+ */
82+ andi t0, ra, _PAGE_HUGE
83+ bne t0, $r0, tlb_huge_update_load
84+
85+ csrrd t0, LOONGARCH_CSR_BADV
86+ srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
87+ andi t0, t0, (PTRS_PER_PTE - 1)
88+ slli.d t0, t0, _PTE_T_LOG2
89+ add.d t1, ra, t0
90+
91+ ld.d t0, t1, 0
92+ tlbsrch
93+
94+ srli.d ra, t0, _PAGE_PRESENT_SHIFT
95+ andi ra, ra, 1
96+ beq ra, $r0, nopage_tlb_load
97+
98+ ori t0, t0, _PAGE_VALID
99+ st.d t0, t1, 0
100+ ori t1, t1, 8
101+ xori t1, t1, 8
102+ ld.d t0, t1, 0
103+ ld.d t1, t1, 8
104+ csrwr t0, LOONGARCH_CSR_TLBELO0
105+ csrwr t1, LOONGARCH_CSR_TLBELO1
106+ tlbwr
107+leave_load:
108+ csrrd t0, EXCEPTION_KS0
109+ csrrd t1, EXCEPTION_KS1
110+ csrrd ra, EXCEPTION_KS2
111+ ertn
112+#ifdef CONFIG_64BIT
113+vmalloc_load:
114+ la.abs t1, swapper_pg_dir
115+ b vmalloc_done_load
116+#endif
117+
118+ /*
119+ * This is the entry point when build_tlbchange_handler_head
120+ * spots a huge page.
121+ */
122+tlb_huge_update_load:
123+ ld.d t0, t1, 0
124+ srli.d ra, t0, _PAGE_PRESENT_SHIFT
125+ andi ra, ra, 1
126+ beq ra, $r0, nopage_tlb_load
127+ tlbsrch
128+
129+ ori t0, t0, _PAGE_VALID
130+ st.d t0, t1, 0
131+ addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
132+ addi.d ra, t1, 0
133+ csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
134+ tlbwr
135+
136+ csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX
137+
138+ /*
139+ * A huge PTE describes an area the size of the
140+ * configured huge page size. This is twice the
141+ * of the large TLB entry size we intend to use.
142+ * A TLB entry half the size of the configured
143+ * huge page size is configured into entrylo0
144+ * and entrylo1 to cover the contiguous huge PTE
145+ * address space.
146+ */
147+ /* Huge page: Move Global bit */
148+ xori t0, t0, _PAGE_HUGE
149+ lu12i.w t1, _PAGE_HGLOBAL >> 12
150+ and t1, t0, t1
151+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
152+ or t0, t0, t1
153+
154+ addi.d ra, t0, 0
155+ csrwr t0, LOONGARCH_CSR_TLBELO0
156+ addi.d t0, ra, 0
157+
158+ /* Convert to entrylo1 */
159+ addi.d t1, $r0, 1
160+ slli.d t1, t1, (HPAGE_SHIFT - 1)
161+ add.d t0, t0, t1
162+ csrwr t0, LOONGARCH_CSR_TLBELO1
163+
164+ /* Set huge page tlb entry size */
165+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
166+ addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
167+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
168+
169+ tlbfill
170+
171+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
172+ addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
173+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
174+
175+nopage_tlb_load:
176+ csrrd ra, EXCEPTION_KS2
177+ la.abs t0, tlb_do_page_fault_0
178+ jirl $r0, t0, 0
179+SYM_FUNC_END(handle_tlb_load)
180+
181+SYM_FUNC_START(handle_tlb_store)
182+ csrwr t0, EXCEPTION_KS0
183+ csrwr t1, EXCEPTION_KS1
184+ csrwr ra, EXCEPTION_KS2
185+
186+ /*
187+ * The vmalloc handling is not in the hotpath.
188+ */
189+ csrrd t0, LOONGARCH_CSR_BADV
190+ blt t0, $r0, vmalloc_store
191+ csrrd t1, LOONGARCH_CSR_PGDL
192+
193+vmalloc_done_store:
194+ /* Get PGD offset in bytes */
195+ srli.d t0, t0, PGDIR_SHIFT
196+ andi t0, t0, (PTRS_PER_PGD - 1)
197+ slli.d t0, t0, 3
198+ add.d t1, t1, t0
199+
200+#if CONFIG_PGTABLE_LEVELS > 3
201+ csrrd t0, LOONGARCH_CSR_BADV
202+ ld.d t1, t1, 0
203+ srli.d t0, t0, PUD_SHIFT
204+ andi t0, t0, (PTRS_PER_PUD - 1)
205+ slli.d t0, t0, 3
206+ add.d t1, t1, t0
207+#endif
208+#if CONFIG_PGTABLE_LEVELS > 2
209+ csrrd t0, LOONGARCH_CSR_BADV
210+ ld.d t1, t1, 0
211+ srli.d t0, t0, PMD_SHIFT
212+ andi t0, t0, (PTRS_PER_PMD - 1)
213+ slli.d t0, t0, 3
214+ add.d t1, t1, t0
215+#endif
216+ ld.d ra, t1, 0
217+
218+ /*
219+ * For huge tlb entries, pmde doesn't contain an address but
220+ * instead contains the tlb pte. Check the PAGE_HUGE bit and
221+ * see if we need to jump to huge tlb processing.
222+ */
223+ andi t0, ra, _PAGE_HUGE
224+ bne t0, $r0, tlb_huge_update_store
225+
226+ csrrd t0, LOONGARCH_CSR_BADV
227+ srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
228+ andi t0, t0, (PTRS_PER_PTE - 1)
229+ slli.d t0, t0, _PTE_T_LOG2
230+ add.d t1, ra, t0
231+
232+ ld.d t0, t1, 0
233+ tlbsrch
234+
235+ srli.d ra, t0, _PAGE_PRESENT_SHIFT
236+ andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
237+ xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
238+ bne ra, $r0, nopage_tlb_store
239+
240+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
241+ st.d t0, t1, 0
242+
243+ ori t1, t1, 8
244+ xori t1, t1, 8
245+ ld.d t0, t1, 0
246+ ld.d t1, t1, 8
247+ csrwr t0, LOONGARCH_CSR_TLBELO0
248+ csrwr t1, LOONGARCH_CSR_TLBELO1
249+ tlbwr
250+leave_store:
251+ csrrd t0, EXCEPTION_KS0
252+ csrrd t1, EXCEPTION_KS1
253+ csrrd ra, EXCEPTION_KS2
254+ ertn
255+#ifdef CONFIG_64BIT
256+vmalloc_store:
257+ la.abs t1, swapper_pg_dir
258+ b vmalloc_done_store
259+#endif
260+
261+ /*
262+ * This is the entry point when build_tlbchange_handler_head
263+ * spots a huge page.
264+ */
265+tlb_huge_update_store:
266+ ld.d t0, t1, 0
267+ srli.d ra, t0, _PAGE_PRESENT_SHIFT
268+ andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
269+ xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
270+ bne ra, $r0, nopage_tlb_store
271+
272+ tlbsrch
273+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
274+
275+ st.d t0, t1, 0
276+ addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
277+ addi.d ra, t1, 0
278+ csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
279+ tlbwr
280+
281+ csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX
282+ /*
283+ * A huge PTE describes an area the size of the
284+ * configured huge page size. This is twice the
285+ * of the large TLB entry size we intend to use.
286+ * A TLB entry half the size of the configured
287+ * huge page size is configured into entrylo0
288+ * and entrylo1 to cover the contiguous huge PTE
289+ * address space.
290+ */
291+ /* Huge page: Move Global bit */
292+ xori t0, t0, _PAGE_HUGE
293+ lu12i.w t1, _PAGE_HGLOBAL >> 12
294+ and t1, t0, t1
295+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
296+ or t0, t0, t1
297+
298+ addi.d ra, t0, 0
299+ csrwr t0, LOONGARCH_CSR_TLBELO0
300+ addi.d t0, ra, 0
301+
302+ /* Convert to entrylo1 */
303+ addi.d t1, $r0, 1
304+ slli.d t1, t1, (HPAGE_SHIFT - 1)
305+ add.d t0, t0, t1
306+ csrwr t0, LOONGARCH_CSR_TLBELO1
307+
308+ /* Set huge page tlb entry size */
309+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
310+ addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
311+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
312+
313+ tlbfill
314+
315+ /* Reset default page size */
316+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
317+ addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
318+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
319+
320+nopage_tlb_store:
321+ csrrd ra, EXCEPTION_KS2
322+ la.abs t0, tlb_do_page_fault_1
323+ jirl $r0, t0, 0
324+SYM_FUNC_END(handle_tlb_store)
325+
326+SYM_FUNC_START(handle_tlb_modify)
327+ csrwr t0, EXCEPTION_KS0
328+ csrwr t1, EXCEPTION_KS1
329+ csrwr ra, EXCEPTION_KS2
330+
331+ /*
332+ * The vmalloc handling is not in the hotpath.
333+ */
334+ csrrd t0, LOONGARCH_CSR_BADV
335+ blt t0, $r0, vmalloc_modify
336+ csrrd t1, LOONGARCH_CSR_PGDL
337+
338+vmalloc_done_modify:
339+ /* Get PGD offset in bytes */
340+ srli.d t0, t0, PGDIR_SHIFT
341+ andi t0, t0, (PTRS_PER_PGD - 1)
342+ slli.d t0, t0, 3
343+ add.d t1, t1, t0
344+#if CONFIG_PGTABLE_LEVELS > 3
345+ csrrd t0, LOONGARCH_CSR_BADV
346+ ld.d t1, t1, 0
347+ srli.d t0, t0, PUD_SHIFT
348+ andi t0, t0, (PTRS_PER_PUD - 1)
349+ slli.d t0, t0, 3
350+ add.d t1, t1, t0
351+#endif
352+#if CONFIG_PGTABLE_LEVELS > 2
353+ csrrd t0, LOONGARCH_CSR_BADV
354+ ld.d t1, t1, 0
355+ srli.d t0, t0, PMD_SHIFT
356+ andi t0, t0, (PTRS_PER_PMD - 1)
357+ slli.d t0, t0, 3
358+ add.d t1, t1, t0
359+#endif
360+ ld.d ra, t1, 0
361+
362+ /*
363+ * For huge tlb entries, pmde doesn't contain an address but
364+ * instead contains the tlb pte. Check the PAGE_HUGE bit and
365+ * see if we need to jump to huge tlb processing.
366+ */
367+ andi t0, ra, _PAGE_HUGE
368+ bne t0, $r0, tlb_huge_update_modify
369+
370+ csrrd t0, LOONGARCH_CSR_BADV
371+ srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
372+ andi t0, t0, (PTRS_PER_PTE - 1)
373+ slli.d t0, t0, _PTE_T_LOG2
374+ add.d t1, ra, t0
375+
376+ ld.d t0, t1, 0
377+ tlbsrch
378+
379+ srli.d ra, t0, _PAGE_WRITE_SHIFT
380+ andi ra, ra, 1
381+ beq ra, $r0, nopage_tlb_modify
382+
383+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
384+ st.d t0, t1, 0
385+ ori t1, t1, 8
386+ xori t1, t1, 8
387+ ld.d t0, t1, 0
388+ ld.d t1, t1, 8
389+ csrwr t0, LOONGARCH_CSR_TLBELO0
390+ csrwr t1, LOONGARCH_CSR_TLBELO1
391+ tlbwr
392+leave_modify:
393+ csrrd t0, EXCEPTION_KS0
394+ csrrd t1, EXCEPTION_KS1
395+ csrrd ra, EXCEPTION_KS2
396+ ertn
397+#ifdef CONFIG_64BIT
398+vmalloc_modify:
399+ la.abs t1, swapper_pg_dir
400+ b vmalloc_done_modify
401+#endif
402+
403+ /*
404+ * This is the entry point when
405+ * build_tlbchange_handler_head spots a huge page.
406+ */
407+tlb_huge_update_modify:
408+ ld.d t0, t1, 0
409+
410+ srli.d ra, t0, _PAGE_WRITE_SHIFT
411+ andi ra, ra, 1
412+ beq ra, $r0, nopage_tlb_modify
413+
414+ tlbsrch
415+ ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
416+
417+ st.d t0, t1, 0
418+ /*
419+ * A huge PTE describes an area the size of the
420+ * configured huge page size. This is twice the
421+ * of the large TLB entry size we intend to use.
422+ * A TLB entry half the size of the configured
423+ * huge page size is configured into entrylo0
424+ * and entrylo1 to cover the contiguous huge PTE
425+ * address space.
426+ */
427+ /* Huge page: Move Global bit */
428+ xori t0, t0, _PAGE_HUGE
429+ lu12i.w t1, _PAGE_HGLOBAL >> 12
430+ and t1, t0, t1
431+ srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
432+ or t0, t0, t1
433+
434+ addi.d ra, t0, 0
435+ csrwr t0, LOONGARCH_CSR_TLBELO0
436+ addi.d t0, ra, 0
437+
438+ /* Convert to entrylo1 */
439+ addi.d t1, $r0, 1
440+ slli.d t1, t1, (HPAGE_SHIFT - 1)
441+ add.d t0, t0, t1
442+ csrwr t0, LOONGARCH_CSR_TLBELO1
443+
444+ /* Set huge page tlb entry size */
445+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
446+ addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
447+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
448+
449+ tlbwr
450+
451+ /* Reset default page size */
452+ addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
453+ addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
454+ csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
455+
456+nopage_tlb_modify:
457+ csrrd ra, EXCEPTION_KS2
458+ la.abs t0, tlb_do_page_fault_1
459+ jirl $r0, t0, 0
460+SYM_FUNC_END(handle_tlb_modify)
461+
462+SYM_FUNC_START(handle_tlb_refill)
463+ csrwr t0, LOONGARCH_CSR_TLBRSAVE
464+ csrrd t0, LOONGARCH_CSR_PGD
465+ lddir t0, t0, 3
466+#if CONFIG_PGTABLE_LEVELS > 3
467+ lddir t0, t0, 2
468+#endif
469+#if CONFIG_PGTABLE_LEVELS > 2
470+ lddir t0, t0, 1
471+#endif
472+ ldpte t0, 0
473+ ldpte t0, 1
474+ tlbfill
475+ csrrd t0, LOONGARCH_CSR_TLBRSAVE
476+ ertn
477+SYM_FUNC_END(handle_tlb_refill)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t