History

LoongArch: Add memory management

Add memory management support for LoongArch, including: cache and tlb management, page fault handling and ioremap/mmap support.

Huacai Chen committed 4 years ago

Commit 09cfefb1 parent 803b0fc26 files · +3172 −0Browse files

Unified · Split

26 files
  1. arch/loongarch/include/asm/cache.h +13 −0
  2. arch/loongarch/include/asm/cacheflush.h +80 −0
  3. arch/loongarch/include/asm/cacheops.h +37 −0
  4. arch/loongarch/include/asm/fixmap.h +13 −0
  5. arch/loongarch/include/asm/hugetlb.h +83 −0
  6. arch/loongarch/include/asm/page.h +115 −0
  7. arch/loongarch/include/asm/pgalloc.h +103 −0
  8. arch/loongarch/include/asm/pgtable-bits.h +131 −0
  9. arch/loongarch/include/asm/pgtable.h +532 −0
  10. arch/loongarch/include/asm/shmparam.h +12 −0
  11. arch/loongarch/include/asm/sparsemem.h +23 −0
  12. arch/loongarch/include/asm/tlb.h +180 −0
  13. arch/loongarch/include/asm/tlbflush.h +35 −0
  14. arch/loongarch/include/asm/vmalloc.h +4 −0
  15. arch/loongarch/mm/cache.c +141 −0
  16. arch/loongarch/mm/extable.c +22 −0
  17. arch/loongarch/mm/fault.c +261 −0
  18. arch/loongarch/mm/hugetlbpage.c +87 −0
  19. arch/loongarch/mm/init.c +165 −0
  20. arch/loongarch/mm/ioremap.c +27 −0
  21. arch/loongarch/mm/maccess.c +10 −0
  22. arch/loongarch/mm/mmap.c +125 −0
  23. arch/loongarch/mm/page.S +84 −0
  24. arch/loongarch/mm/pgtable.c +130 −0
  25. arch/loongarch/mm/tlb.c +282 −0
  26. arch/loongarch/mm/tlbex.S +477 −0
arch/loongarch/include/asm/cache.h+13−0
@@ -0,0 +1,13 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_CACHE_H
6#define _ASM_CACHE_H
7
8#define L1_CACHE_SHIFT CONFIG_L1_CACHE_SHIFT
9#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
10
11#define __read_mostly __section(".data..read_mostly")
12
13#endif /* _ASM_CACHE_H */
arch/loongarch/include/asm/cacheflush.h+80−0
@@ -0,0 +1,80 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_CACHEFLUSH_H
6#define _ASM_CACHEFLUSH_H
7
8#include <linux/mm.h>
9#include <asm/cpu-features.h>
10#include <asm/cacheops.h>
11
12extern void local_flush_icache_range(unsigned long start, unsigned long end);
13
14#define flush_icache_range local_flush_icache_range
15#define flush_icache_user_range local_flush_icache_range
16
17#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0
18
19#define flush_cache_all() do { } while (0)
20#define flush_cache_mm(mm) do { } while (0)
21#define flush_cache_dup_mm(mm) do { } while (0)
22#define flush_cache_range(vma, start, end) do { } while (0)
23#define flush_cache_page(vma, vmaddr, pfn) do { } while (0)
24#define flush_cache_vmap(start, end) do { } while (0)
25#define flush_cache_vunmap(start, end) do { } while (0)
26#define flush_icache_page(vma, page) do { } while (0)
27#define flush_icache_user_page(vma, page, addr, len) do { } while (0)
28#define flush_dcache_page(page) do { } while (0)
29#define flush_dcache_mmap_lock(mapping) do { } while (0)
30#define flush_dcache_mmap_unlock(mapping) do { } while (0)
31
32#define cache_op(op, addr) \
33 __asm__ __volatile__( \
34 " cacop %0, %1 \n" \
35 : \
36 : "i" (op), "ZC" (*(unsigned char *)(addr)))
37
38static inline void flush_icache_line_indexed(unsigned long addr)
39{
40 cache_op(Index_Invalidate_I, addr);
41}
42
43static inline void flush_dcache_line_indexed(unsigned long addr)
44{
45 cache_op(Index_Writeback_Inv_D, addr);
46}
47
48static inline void flush_vcache_line_indexed(unsigned long addr)
49{
50 cache_op(Index_Writeback_Inv_V, addr);
51}
52
53static inline void flush_scache_line_indexed(unsigned long addr)
54{
55 cache_op(Index_Writeback_Inv_S, addr);
56}
57
58static inline void flush_icache_line(unsigned long addr)
59{
60 cache_op(Hit_Invalidate_I, addr);
61}
62
63static inline void flush_dcache_line(unsigned long addr)
64{
65 cache_op(Hit_Writeback_Inv_D, addr);
66}
67
68static inline void flush_vcache_line(unsigned long addr)
69{
70 cache_op(Hit_Writeback_Inv_V, addr);
71}
72
73static inline void flush_scache_line(unsigned long addr)
74{
75 cache_op(Hit_Writeback_Inv_S, addr);
76}
77
78#include <asm-generic/cacheflush.h>
79
80#endif /* _ASM_CACHEFLUSH_H */
arch/loongarch/include/asm/cacheops.h+37−0
@@ -0,0 +1,37 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Cache operations for the cache instruction.
4 *
5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6 */
7#ifndef __ASM_CACHEOPS_H
8#define __ASM_CACHEOPS_H
9
10/*
11 * Most cache ops are split into a 2 bit field identifying the cache, and a 3
12 * bit field identifying the cache operation.
13 */
14#define CacheOp_Cache 0x03
15#define CacheOp_Op 0x1c
16
17#define Cache_I 0x00
18#define Cache_D 0x01
19#define Cache_V 0x02
20#define Cache_S 0x03
21
22#define Index_Invalidate 0x08
23#define Index_Writeback_Inv 0x08
24#define Hit_Invalidate 0x10
25#define Hit_Writeback_Inv 0x10
26#define CacheOp_User_Defined 0x18
27
28#define Index_Invalidate_I (Cache_I | Index_Invalidate)
29#define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv)
30#define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv)
31#define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv)
32#define Hit_Invalidate_I (Cache_I | Hit_Invalidate)
33#define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv)
34#define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv)
35#define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv)
36
37#endif /* __ASM_CACHEOPS_H */
arch/loongarch/include/asm/fixmap.h+13−0
@@ -0,0 +1,13 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * fixmap.h: compile-time virtual memory allocation
4 *
5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6 */
7
8#ifndef _ASM_FIXMAP_H
9#define _ASM_FIXMAP_H
10
11#define NR_FIX_BTMAPS 64
12
13#endif
arch/loongarch/include/asm/hugetlb.h+83−0
@@ -0,0 +1,83 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#ifndef __ASM_HUGETLB_H
7#define __ASM_HUGETLB_H
8
9#include <asm/page.h>
10
11uint64_t pmd_to_entrylo(unsigned long pmd_val);
12
13#define __HAVE_ARCH_PREPARE_HUGEPAGE_RANGE
14static inline int prepare_hugepage_range(struct file *file,
15 unsigned long addr,
16 unsigned long len)
17{
18 unsigned long task_size = STACK_TOP;
19 struct hstate *h = hstate_file(file);
20
21 if (len & ~huge_page_mask(h))
22 return -EINVAL;
23 if (addr & ~huge_page_mask(h))
24 return -EINVAL;
25 if (len > task_size)
26 return -ENOMEM;
27 if (task_size - len < addr)
28 return -EINVAL;
29 return 0;
30}
31
32#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
33static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
34 unsigned long addr, pte_t *ptep)
35{
36 pte_t clear;
37 pte_t pte = *ptep;
38
39 pte_val(clear) = (unsigned long)invalid_pte_table;
40 set_pte_at(mm, addr, ptep, clear);
41 return pte;
42}
43
44#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
45static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
46 unsigned long addr, pte_t *ptep)
47{
48 pte_t pte;
49
50 pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
51 flush_tlb_page(vma, addr);
52 return pte;
53}
54
55#define __HAVE_ARCH_HUGE_PTE_NONE
56static inline int huge_pte_none(pte_t pte)
57{
58 unsigned long val = pte_val(pte) & ~_PAGE_GLOBAL;
59 return !val || (val == (unsigned long)invalid_pte_table);
60}
61
62#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
63static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
64 unsigned long addr,
65 pte_t *ptep, pte_t pte,
66 int dirty)
67{
68 int changed = !pte_same(*ptep, pte);
69
70 if (changed) {
71 set_pte_at(vma->vm_mm, addr, ptep, pte);
72 /*
73 * There could be some standard sized pages in there,
74 * get them all.
75 */
76 flush_tlb_range(vma, addr, addr + HPAGE_SIZE);
77 }
78 return changed;
79}
80
81#include <asm-generic/hugetlb.h>
82
83#endif /* __ASM_HUGETLB_H */
arch/loongarch/include/asm/page.h+115−0
@@ -0,0 +1,115 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_PAGE_H
6#define _ASM_PAGE_H
7
8#include <linux/const.h>
9
10/*
11 * PAGE_SHIFT determines the page size
12 */
13#ifdef CONFIG_PAGE_SIZE_4KB
14#define PAGE_SHIFT 12
15#endif
16#ifdef CONFIG_PAGE_SIZE_16KB
17#define PAGE_SHIFT 14
18#endif
19#ifdef CONFIG_PAGE_SIZE_64KB
20#define PAGE_SHIFT 16
21#endif
22#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
23#define PAGE_MASK (~(PAGE_SIZE - 1))
24
25#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3)
26#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
27#define HPAGE_MASK (~(HPAGE_SIZE - 1))
28#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
29
30#ifndef __ASSEMBLY__
31
32#include <linux/kernel.h>
33#include <linux/pfn.h>
34
35#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT))
36
37/*
38 * It's normally defined only for FLATMEM config but it's
39 * used in our early mem init code for all memory models.
40 * So always define it.
41 */
42#define ARCH_PFN_OFFSET PFN_UP(PHYS_OFFSET)
43
44extern void clear_page(void *page);
45extern void copy_page(void *to, void *from);
46
47#define clear_user_page(page, vaddr, pg) clear_page(page)
48#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
49
50extern unsigned long shm_align_mask;
51
52struct page;
53struct vm_area_struct;
54void copy_user_highpage(struct page *to, struct page *from,
55 unsigned long vaddr, struct vm_area_struct *vma);
56
57#define __HAVE_ARCH_COPY_USER_HIGHPAGE
58
59typedef struct { unsigned long pte; } pte_t;
60#define pte_val(x) ((x).pte)
61#define __pte(x) ((pte_t) { (x) })
62typedef struct page *pgtable_t;
63
64typedef struct { unsigned long pgd; } pgd_t;
65#define pgd_val(x) ((x).pgd)
66#define __pgd(x) ((pgd_t) { (x) })
67
68/*
69 * Manipulate page protection bits
70 */
71typedef struct { unsigned long pgprot; } pgprot_t;
72#define pgprot_val(x) ((x).pgprot)
73#define __pgprot(x) ((pgprot_t) { (x) })
74#define pte_pgprot(x) __pgprot(pte_val(x) & ~_PFN_MASK)
75
76#define ptep_buddy(x) ((pte_t *)((unsigned long)(x) ^ sizeof(pte_t)))
77
78/*
79 * __pa()/__va() should be used only during mem init.
80 */
81#define __pa(x) PHYSADDR(x)
82#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET))
83
84#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT)
85
86#ifdef CONFIG_FLATMEM
87
88static inline int pfn_valid(unsigned long pfn)
89{
90 /* avoid <linux/mm.h> include hell */
91 extern unsigned long max_mapnr;
92 unsigned long pfn_offset = ARCH_PFN_OFFSET;
93
94 return pfn >= pfn_offset && pfn < max_mapnr;
95}
96
97#endif
98
99#define virt_to_pfn(kaddr) PFN_DOWN(virt_to_phys((void *)(kaddr)))
100#define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr))
101
102extern int __virt_addr_valid(volatile void *kaddr);
103#define virt_addr_valid(kaddr) __virt_addr_valid((volatile void *)(kaddr))
104
105#define VM_DATA_DEFAULT_FLAGS \
106 (VM_READ | VM_WRITE | \
107 ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) | \
108 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
109
110#include <asm-generic/memory_model.h>
111#include <asm-generic/getorder.h>
112
113#endif /* !__ASSEMBLY__ */
114
115#endif /* _ASM_PAGE_H */
arch/loongarch/include/asm/pgalloc.h+103−0
@@ -0,0 +1,103 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_PGALLOC_H
6#define _ASM_PGALLOC_H
7
8#include <linux/mm.h>
9#include <linux/sched.h>
10
11#define __HAVE_ARCH_PMD_ALLOC_ONE
12#define __HAVE_ARCH_PUD_ALLOC_ONE
13#include <asm-generic/pgalloc.h>
14
15static inline void pmd_populate_kernel(struct mm_struct *mm,
16 pmd_t *pmd, pte_t *pte)
17{
18 set_pmd(pmd, __pmd((unsigned long)pte));
19}
20
21static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t pte)
22{
23 set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
24}
25
26#ifndef __PAGETABLE_PMD_FOLDED
27
28static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
29{
30 set_pud(pud, __pud((unsigned long)pmd));
31}
32#endif
33
34#ifndef __PAGETABLE_PUD_FOLDED
35
36static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
37{
38 set_p4d(p4d, __p4d((unsigned long)pud));
39}
40
41#endif /* __PAGETABLE_PUD_FOLDED */
42
43extern void pagetable_init(void);
44
45/*
46 * Initialize a new pmd table with invalid pointers.
47 */
48extern void pmd_init(unsigned long page, unsigned long pagetable);
49
50/*
51 * Initialize a new pgd / pmd table with invalid pointers.
52 */
53extern void pgd_init(unsigned long page);
54extern pgd_t *pgd_alloc(struct mm_struct *mm);
55
56#define __pte_free_tlb(tlb, pte, address) \
57do { \
58 pgtable_pte_page_dtor(pte); \
59 tlb_remove_page((tlb), pte); \
60} while (0)
61
62#ifndef __PAGETABLE_PMD_FOLDED
63
64static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
65{
66 pmd_t *pmd;
67 struct page *pg;
68
69 pg = alloc_pages(GFP_KERNEL_ACCOUNT, PMD_ORDER);
70 if (!pg)
71 return NULL;
72
73 if (!pgtable_pmd_page_ctor(pg)) {
74 __free_pages(pg, PMD_ORDER);
75 return NULL;
76 }
77
78 pmd = (pmd_t *)page_address(pg);
79 pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table);
80 return pmd;
81}
82
83#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
84
85#endif
86
87#ifndef __PAGETABLE_PUD_FOLDED
88
89static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
90{
91 pud_t *pud;
92
93 pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_ORDER);
94 if (pud)
95 pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
96 return pud;
97}
98
99#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x)
100
101#endif /* __PAGETABLE_PUD_FOLDED */
102
103#endif /* _ASM_PGALLOC_H */
arch/loongarch/include/asm/pgtable-bits.h+131−0
@@ -0,0 +1,131 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_PGTABLE_BITS_H
6#define _ASM_PGTABLE_BITS_H
7
8/* Page table bits */
9#define _PAGE_VALID_SHIFT 0
10#define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */
11#define _PAGE_DIRTY_SHIFT 1
12#define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */
13#define _CACHE_SHIFT 4 /* 4~5, two bits */
14#define _PAGE_GLOBAL_SHIFT 6
15#define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */
16#define _PAGE_PRESENT_SHIFT 7
17#define _PAGE_WRITE_SHIFT 8
18#define _PAGE_MODIFIED_SHIFT 9
19#define _PAGE_PROTNONE_SHIFT 10
20#define _PAGE_SPECIAL_SHIFT 11
21#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */
22#define _PAGE_PFN_SHIFT 12
23#define _PAGE_PFN_END_SHIFT 48
24#define _PAGE_NO_READ_SHIFT 61
25#define _PAGE_NO_EXEC_SHIFT 62
26#define _PAGE_RPLV_SHIFT 63
27
28/* Used by software */
29#define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT)
30#define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT)
31#define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT)
32#define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT)
33#define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT)
34#define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT)
35
36/* Used by TLB hardware (placed in EntryLo*) */
37#define _PAGE_VALID (_ULCAST_(1) << _PAGE_VALID_SHIFT)
38#define _PAGE_DIRTY (_ULCAST_(1) << _PAGE_DIRTY_SHIFT)
39#define _PAGE_PLV (_ULCAST_(3) << _PAGE_PLV_SHIFT)
40#define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT)
41#define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT)
42#define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT)
43#define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT)
44#define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT)
45#define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT)
46#define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT)
47#define _PFN_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT)
48
49#define _PAGE_USER (PLV_USER << _PAGE_PLV_SHIFT)
50#define _PAGE_KERN (PLV_KERN << _PAGE_PLV_SHIFT)
51
52#define _PFN_MASK (~((_ULCAST_(1) << (_PFN_SHIFT)) - 1) & \
53 ((_ULCAST_(1) << (_PAGE_PFN_END_SHIFT)) - 1))
54
55/*
56 * Cache attributes
57 */
58#ifndef _CACHE_SUC
59#define _CACHE_SUC (0<<_CACHE_SHIFT) /* Strong-ordered UnCached */
60#endif
61#ifndef _CACHE_CC
62#define _CACHE_CC (1<<_CACHE_SHIFT) /* Coherent Cached */
63#endif
64#ifndef _CACHE_WUC
65#define _CACHE_WUC (2<<_CACHE_SHIFT) /* Weak-ordered UnCached */
66#endif
67
68#define __READABLE (_PAGE_VALID)
69#define __WRITEABLE (_PAGE_DIRTY | _PAGE_WRITE)
70
71#define _PAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV)
72#define _HPAGE_CHG_MASK (_PAGE_MODIFIED | _PAGE_SPECIAL | _PFN_MASK | _CACHE_MASK | _PAGE_PLV | _PAGE_HUGE)
73
74#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_NO_READ | \
75 _PAGE_USER | _CACHE_CC)
76#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_WRITE | \
77 _PAGE_USER | _CACHE_CC)
78#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | _CACHE_CC)
79
80#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
81 _PAGE_GLOBAL | _PAGE_KERN | _CACHE_CC)
82#define PAGE_KERNEL_SUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
83 _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC)
84#define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \
85 _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC)
86
87#define __P000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ)
88#define __P001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
89#define __P010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
90#define __P011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
91#define __P100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
92#define __P101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
93#define __P110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
94#define __P111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
95
96#define __S000 __pgprot(_CACHE_CC | _PAGE_USER | _PAGE_PROTNONE | _PAGE_NO_EXEC | _PAGE_NO_READ)
97#define __S001 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC)
98#define __S010 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE)
99#define __S011 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE)
100#define __S100 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
101#define __S101 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT)
102#define __S110 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE)
103#define __S111 __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_WRITE)
104
105#ifndef __ASSEMBLY__
106
107#define pgprot_noncached pgprot_noncached
108
109static inline pgprot_t pgprot_noncached(pgprot_t _prot)
110{
111 unsigned long prot = pgprot_val(_prot);
112
113 prot = (prot & ~_CACHE_MASK) | _CACHE_SUC;
114
115 return __pgprot(prot);
116}
117
118#define pgprot_writecombine pgprot_writecombine
119
120static inline pgprot_t pgprot_writecombine(pgprot_t _prot)
121{
122 unsigned long prot = pgprot_val(_prot);
123
124 prot = (prot & ~_CACHE_MASK) | _CACHE_WUC;
125
126 return __pgprot(prot);
127}
128
129#endif /* !__ASSEMBLY__ */
130
131#endif /* _ASM_PGTABLE_BITS_H */
arch/loongarch/include/asm/pgtable.h+532−0
@@ -0,0 +1,532 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle
7 * Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_PGTABLE_H
10#define _ASM_PGTABLE_H
11
12#include <linux/compiler.h>
13#include <asm/addrspace.h>
14#include <asm/pgtable-bits.h>
15
16#if CONFIG_PGTABLE_LEVELS == 2
17#include <asm-generic/pgtable-nopmd.h>
18#elif CONFIG_PGTABLE_LEVELS == 3
19#include <asm-generic/pgtable-nopud.h>
20#else
21#include <asm-generic/pgtable-nop4d.h>
22#endif
23
24#define PGD_ORDER 0
25#define PUD_ORDER 0
26#define PMD_ORDER 0
27#define PTE_ORDER 0
28
29#if CONFIG_PGTABLE_LEVELS == 2
30#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
31#elif CONFIG_PGTABLE_LEVELS == 3
32#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
33#define PMD_SIZE (1UL << PMD_SHIFT)
34#define PMD_MASK (~(PMD_SIZE-1))
35#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3))
36#elif CONFIG_PGTABLE_LEVELS == 4
37#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT + PTE_ORDER - 3))
38#define PMD_SIZE (1UL << PMD_SHIFT)
39#define PMD_MASK (~(PMD_SIZE-1))
40#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_ORDER - 3))
41#define PUD_SIZE (1UL << PUD_SHIFT)
42#define PUD_MASK (~(PUD_SIZE-1))
43#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_ORDER - 3))
44#endif
45
46#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
47#define PGDIR_MASK (~(PGDIR_SIZE-1))
48
49#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT + PGD_ORDER - 3))
50
51#define PTRS_PER_PGD ((PAGE_SIZE << PGD_ORDER) >> 3)
52#if CONFIG_PGTABLE_LEVELS > 3
53#define PTRS_PER_PUD ((PAGE_SIZE << PUD_ORDER) >> 3)
54#endif
55#if CONFIG_PGTABLE_LEVELS > 2
56#define PTRS_PER_PMD ((PAGE_SIZE << PMD_ORDER) >> 3)
57#endif
58#define PTRS_PER_PTE ((PAGE_SIZE << PTE_ORDER) >> 3)
59
60#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1)
61
62#ifndef __ASSEMBLY__
63
64#include <linux/mm_types.h>
65#include <linux/mmzone.h>
66#include <asm/fixmap.h>
67#include <asm/io.h>
68
69struct mm_struct;
70struct vm_area_struct;
71
72/*
73 * ZERO_PAGE is a global shared page that is always zero; used
74 * for zero-mapped memory areas etc..
75 */
76
77extern unsigned long empty_zero_page;
78extern unsigned long zero_page_mask;
79
80#define ZERO_PAGE(vaddr) \
81 (virt_to_page((void *)(empty_zero_page + (((unsigned long)(vaddr)) & zero_page_mask))))
82#define __HAVE_COLOR_ZERO_PAGE
83
84/*
85 * TLB refill handlers may also map the vmalloc area into xkvrange.
86 * Avoid the first couple of pages so NULL pointer dereferences will
87 * still reliably trap.
88 */
89#define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
90#define MODULES_END (MODULES_VADDR + SZ_256M)
91
92#define VMALLOC_START MODULES_END
93#define VMALLOC_END \
94 (vm_map_base + \
95 min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, (1UL << cpu_vabits)) - PMD_SIZE)
96
97#define pte_ERROR(e) \
98 pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
99#ifndef __PAGETABLE_PMD_FOLDED
100#define pmd_ERROR(e) \
101 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
102#endif
103#ifndef __PAGETABLE_PUD_FOLDED
104#define pud_ERROR(e) \
105 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
106#endif
107#define pgd_ERROR(e) \
108 pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
109
110extern pte_t invalid_pte_table[PTRS_PER_PTE];
111
112#ifndef __PAGETABLE_PUD_FOLDED
113
114typedef struct { unsigned long pud; } pud_t;
115#define pud_val(x) ((x).pud)
116#define __pud(x) ((pud_t) { (x) })
117
118extern pud_t invalid_pud_table[PTRS_PER_PUD];
119
120/*
121 * Empty pgd/p4d entries point to the invalid_pud_table.
122 */
123static inline int p4d_none(p4d_t p4d)
124{
125 return p4d_val(p4d) == (unsigned long)invalid_pud_table;
126}
127
128static inline int p4d_bad(p4d_t p4d)
129{
130 return p4d_val(p4d) & ~PAGE_MASK;
131}
132
133static inline int p4d_present(p4d_t p4d)
134{
135 return p4d_val(p4d) != (unsigned long)invalid_pud_table;
136}
137
138static inline void p4d_clear(p4d_t *p4dp)
139{
140 p4d_val(*p4dp) = (unsigned long)invalid_pud_table;
141}
142
143static inline pud_t *p4d_pgtable(p4d_t p4d)
144{
145 return (pud_t *)p4d_val(p4d);
146}
147
148static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
149{
150 *p4d = p4dval;
151}
152
153#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d))
154#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT))
155
156#endif
157
158#ifndef __PAGETABLE_PMD_FOLDED
159
160typedef struct { unsigned long pmd; } pmd_t;
161#define pmd_val(x) ((x).pmd)
162#define __pmd(x) ((pmd_t) { (x) })
163
164extern pmd_t invalid_pmd_table[PTRS_PER_PMD];
165
166/*
167 * Empty pud entries point to the invalid_pmd_table.
168 */
169static inline int pud_none(pud_t pud)
170{
171 return pud_val(pud) == (unsigned long)invalid_pmd_table;
172}
173
174static inline int pud_bad(pud_t pud)
175{
176 return pud_val(pud) & ~PAGE_MASK;
177}
178
179static inline int pud_present(pud_t pud)
180{
181 return pud_val(pud) != (unsigned long)invalid_pmd_table;
182}
183
184static inline void pud_clear(pud_t *pudp)
185{
186 pud_val(*pudp) = ((unsigned long)invalid_pmd_table);
187}
188
189static inline pmd_t *pud_pgtable(pud_t pud)
190{
191 return (pmd_t *)pud_val(pud);
192}
193
194#define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0)
195
196#define pud_phys(pud) virt_to_phys((void *)pud_val(pud))
197#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT))
198
199#endif
200
201/*
202 * Empty pmd entries point to the invalid_pte_table.
203 */
204static inline int pmd_none(pmd_t pmd)
205{
206 return pmd_val(pmd) == (unsigned long)invalid_pte_table;
207}
208
209static inline int pmd_bad(pmd_t pmd)
210{
211 return (pmd_val(pmd) & ~PAGE_MASK);
212}
213
214static inline int pmd_present(pmd_t pmd)
215{
216 if (unlikely(pmd_val(pmd) & _PAGE_HUGE))
217 return !!(pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE));
218
219 return pmd_val(pmd) != (unsigned long)invalid_pte_table;
220}
221
222static inline void pmd_clear(pmd_t *pmdp)
223{
224 pmd_val(*pmdp) = ((unsigned long)invalid_pte_table);
225}
226
227#define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0)
228
229#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd))
230
231#ifndef CONFIG_TRANSPARENT_HUGEPAGE
232#define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT))
233#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
234
235#define pmd_page_vaddr(pmd) pmd_val(pmd)
236
237extern pmd_t mk_pmd(struct page *page, pgprot_t prot);
238extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd);
239
240#define pte_page(x) pfn_to_page(pte_pfn(x))
241#define pte_pfn(x) ((unsigned long)(((x).pte & _PFN_MASK) >> _PFN_SHIFT))
242#define pfn_pte(pfn, prot) __pte(((pfn) << _PFN_SHIFT) | pgprot_val(prot))
243#define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot))
244
245/*
246 * Initialize a new pgd / pmd table with invalid pointers.
247 */
248extern void pgd_init(unsigned long page);
249extern void pud_init(unsigned long page, unsigned long pagetable);
250extern void pmd_init(unsigned long page, unsigned long pagetable);
251
252/*
253 * Non-present pages: high 40 bits are offset, next 8 bits type,
254 * low 16 bits zero.
255 */
256static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
257{ pte_t pte; pte_val(pte) = (type << 16) | (offset << 24); return pte; }
258
259#define __swp_type(x) (((x).val >> 16) & 0xff)
260#define __swp_offset(x) ((x).val >> 24)
261#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) })
262#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
263#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
264#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
265#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE })
266
267extern void paging_init(void);
268
269#define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL))
270#define pte_present(pte) (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROTNONE))
271#define pte_no_exec(pte) (pte_val(pte) & _PAGE_NO_EXEC)
272
273static inline void set_pte(pte_t *ptep, pte_t pteval)
274{
275 *ptep = pteval;
276 if (pte_val(pteval) & _PAGE_GLOBAL) {
277 pte_t *buddy = ptep_buddy(ptep);
278 /*
279 * Make sure the buddy is global too (if it's !none,
280 * it better already be global)
281 */
282 if (pte_none(*buddy))
283 pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
284 }
285}
286
287static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
288 pte_t *ptep, pte_t pteval)
289{
290 set_pte(ptep, pteval);
291}
292
293static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
294{
295 /* Preserve global status for the pair */
296 if (pte_val(*ptep_buddy(ptep)) & _PAGE_GLOBAL)
297 set_pte_at(mm, addr, ptep, __pte(_PAGE_GLOBAL));
298 else
299 set_pte_at(mm, addr, ptep, __pte(0));
300}
301
302#define PGD_T_LOG2 (__builtin_ffs(sizeof(pgd_t)) - 1)
303#define PMD_T_LOG2 (__builtin_ffs(sizeof(pmd_t)) - 1)
304#define PTE_T_LOG2 (__builtin_ffs(sizeof(pte_t)) - 1)
305
306extern pgd_t swapper_pg_dir[];
307extern pgd_t invalid_pg_dir[];
308
309/*
310 * The following only work if pte_present() is true.
311 * Undefined behaviour if not..
312 */
313static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITE; }
314static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
315static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_MODIFIED; }
316
317static inline pte_t pte_mkold(pte_t pte)
318{
319 pte_val(pte) &= ~_PAGE_ACCESSED;
320 return pte;
321}
322
323static inline pte_t pte_mkyoung(pte_t pte)
324{
325 pte_val(pte) |= _PAGE_ACCESSED;
326 return pte;
327}
328
329static inline pte_t pte_mkclean(pte_t pte)
330{
331 pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
332 return pte;
333}
334
335static inline pte_t pte_mkdirty(pte_t pte)
336{
337 pte_val(pte) |= (_PAGE_DIRTY | _PAGE_MODIFIED);
338 return pte;
339}
340
341static inline pte_t pte_mkwrite(pte_t pte)
342{
343 pte_val(pte) |= (_PAGE_WRITE | _PAGE_DIRTY);
344 return pte;
345}
346
347static inline pte_t pte_wrprotect(pte_t pte)
348{
349 pte_val(pte) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
350 return pte;
351}
352
353static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; }
354
355static inline pte_t pte_mkhuge(pte_t pte)
356{
357 pte_val(pte) |= _PAGE_HUGE;
358 return pte;
359}
360
361#if defined(CONFIG_ARCH_HAS_PTE_SPECIAL)
362static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; }
363static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }
364#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
365
366#define pte_accessible pte_accessible
367static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
368{
369 if (pte_val(a) & _PAGE_PRESENT)
370 return true;
371
372 if ((pte_val(a) & _PAGE_PROTNONE) &&
373 atomic_read(&mm->tlb_flush_pending))
374 return true;
375
376 return false;
377}
378
379/*
380 * Conversion functions: convert a page and protection to a page entry,
381 * and a page entry and page directory to the page they refer to.
382 */
383#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
384
385static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
386{
387 return __pte((pte_val(pte) & _PAGE_CHG_MASK) |
388 (pgprot_val(newprot) & ~_PAGE_CHG_MASK));
389}
390
391extern void __update_tlb(struct vm_area_struct *vma,
392 unsigned long address, pte_t *ptep);
393
394static inline void update_mmu_cache(struct vm_area_struct *vma,
395 unsigned long address, pte_t *ptep)
396{
397 __update_tlb(vma, address, ptep);
398}
399
400static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
401 unsigned long address, pmd_t *pmdp)
402{
403 __update_tlb(vma, address, (pte_t *)pmdp);
404}
405
406#define kern_addr_valid(addr) (1)
407
408#ifdef CONFIG_TRANSPARENT_HUGEPAGE
409
410/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/
411#define pmdp_establish generic_pmdp_establish
412
413static inline int pmd_trans_huge(pmd_t pmd)
414{
415 return !!(pmd_val(pmd) & _PAGE_HUGE) && pmd_present(pmd);
416}
417
418static inline pmd_t pmd_mkhuge(pmd_t pmd)
419{
420 pmd_val(pmd) = (pmd_val(pmd) & ~(_PAGE_GLOBAL)) |
421 ((pmd_val(pmd) & _PAGE_GLOBAL) << (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT));
422 pmd_val(pmd) |= _PAGE_HUGE;
423
424 return pmd;
425}
426
427#define pmd_write pmd_write
428static inline int pmd_write(pmd_t pmd)
429{
430 return !!(pmd_val(pmd) & _PAGE_WRITE);
431}
432
433static inline pmd_t pmd_mkwrite(pmd_t pmd)
434{
435 pmd_val(pmd) |= (_PAGE_WRITE | _PAGE_DIRTY);
436 return pmd;
437}
438
439static inline pmd_t pmd_wrprotect(pmd_t pmd)
440{
441 pmd_val(pmd) &= ~(_PAGE_WRITE | _PAGE_DIRTY);
442 return pmd;
443}
444
445static inline int pmd_dirty(pmd_t pmd)
446{
447 return !!(pmd_val(pmd) & _PAGE_MODIFIED);
448}
449
450static inline pmd_t pmd_mkclean(pmd_t pmd)
451{
452 pmd_val(pmd) &= ~(_PAGE_DIRTY | _PAGE_MODIFIED);
453 return pmd;
454}
455
456static inline pmd_t pmd_mkdirty(pmd_t pmd)
457{
458 pmd_val(pmd) |= (_PAGE_DIRTY | _PAGE_MODIFIED);
459 return pmd;
460}
461
462static inline int pmd_young(pmd_t pmd)
463{
464 return !!(pmd_val(pmd) & _PAGE_ACCESSED);
465}
466
467static inline pmd_t pmd_mkold(pmd_t pmd)
468{
469 pmd_val(pmd) &= ~_PAGE_ACCESSED;
470 return pmd;
471}
472
473static inline pmd_t pmd_mkyoung(pmd_t pmd)
474{
475 pmd_val(pmd) |= _PAGE_ACCESSED;
476 return pmd;
477}
478
479static inline unsigned long pmd_pfn(pmd_t pmd)
480{
481 return (pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT;
482}
483
484static inline struct page *pmd_page(pmd_t pmd)
485{
486 if (pmd_trans_huge(pmd))
487 return pfn_to_page(pmd_pfn(pmd));
488
489 return pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT);
490}
491
492static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
493{
494 pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) |
495 (pgprot_val(newprot) & ~_HPAGE_CHG_MASK);
496 return pmd;
497}
498
499static inline pmd_t pmd_mkinvalid(pmd_t pmd)
500{
501 pmd_val(pmd) &= ~(_PAGE_PRESENT | _PAGE_VALID | _PAGE_DIRTY | _PAGE_PROTNONE);
502
503 return pmd;
504}
505
506/*
507 * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a
508 * different prototype.
509 */
510#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
511static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
512 unsigned long address, pmd_t *pmdp)
513{
514 pmd_t old = *pmdp;
515
516 pmd_clear(pmdp);
517
518 return old;
519}
520
521#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
522
523/*
524 * We provide our own get_unmapped area to cope with the virtual aliasing
525 * constraints placed on us by the cache architecture.
526 */
527#define HAVE_ARCH_UNMAPPED_AREA
528#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
529
530#endif /* !__ASSEMBLY__ */
531
532#endif /* _ASM_PGTABLE_H */
arch/loongarch/include/asm/shmparam.h+12−0
@@ -0,0 +1,12 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef _ASM_SHMPARAM_H
6#define _ASM_SHMPARAM_H
7
8#define __ARCH_FORCE_SHMLBA 1
9
10#define SHMLBA SZ_64K /* attach addr a multiple of this */
11
12#endif /* _ASM_SHMPARAM_H */
arch/loongarch/include/asm/sparsemem.h+23−0
@@ -0,0 +1,23 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LOONGARCH_SPARSEMEM_H
3#define _LOONGARCH_SPARSEMEM_H
4
5#ifdef CONFIG_SPARSEMEM
6
7/*
8 * SECTION_SIZE_BITS 2^N: how big each section will be
9 * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space
10 */
11#define SECTION_SIZE_BITS 29 /* 2^29 = Largest Huge Page Size */
12#define MAX_PHYSMEM_BITS 48
13
14#endif /* CONFIG_SPARSEMEM */
15
16#ifdef CONFIG_MEMORY_HOTPLUG
17int memory_add_physaddr_to_nid(u64 addr);
18#define memory_add_physaddr_to_nid memory_add_physaddr_to_nid
19#endif
20
21#define INIT_MEMBLOCK_RESERVED_REGIONS (INIT_MEMBLOCK_REGIONS + NR_CPUS)
22
23#endif /* _LOONGARCH_SPARSEMEM_H */
arch/loongarch/include/asm/tlb.h+180−0
@@ -0,0 +1,180 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef __ASM_TLB_H
6#define __ASM_TLB_H
7
8#include <linux/mm_types.h>
9#include <asm/cpu-features.h>
10#include <asm/loongarch.h>
11
12/*
13 * TLB Invalidate Flush
14 */
15static inline void tlbclr(void)
16{
17 __asm__ __volatile__("tlbclr");
18}
19
20static inline void tlbflush(void)
21{
22 __asm__ __volatile__("tlbflush");
23}
24
25/*
26 * TLB R/W operations.
27 */
28static inline void tlb_probe(void)
29{
30 __asm__ __volatile__("tlbsrch");
31}
32
33static inline void tlb_read(void)
34{
35 __asm__ __volatile__("tlbrd");
36}
37
38static inline void tlb_write_indexed(void)
39{
40 __asm__ __volatile__("tlbwr");
41}
42
43static inline void tlb_write_random(void)
44{
45 __asm__ __volatile__("tlbfill");
46}
47
48enum invtlb_ops {
49 /* Invalid all tlb */
50 INVTLB_ALL = 0x0,
51 /* Invalid current tlb */
52 INVTLB_CURRENT_ALL = 0x1,
53 /* Invalid all global=1 lines in current tlb */
54 INVTLB_CURRENT_GTRUE = 0x2,
55 /* Invalid all global=0 lines in current tlb */
56 INVTLB_CURRENT_GFALSE = 0x3,
57 /* Invalid global=0 and matched asid lines in current tlb */
58 INVTLB_GFALSE_AND_ASID = 0x4,
59 /* Invalid addr with global=0 and matched asid in current tlb */
60 INVTLB_ADDR_GFALSE_AND_ASID = 0x5,
61 /* Invalid addr with global=1 or matched asid in current tlb */
62 INVTLB_ADDR_GTRUE_OR_ASID = 0x6,
63 /* Invalid matched gid in guest tlb */
64 INVGTLB_GID = 0x9,
65 /* Invalid global=1, matched gid in guest tlb */
66 INVGTLB_GID_GTRUE = 0xa,
67 /* Invalid global=0, matched gid in guest tlb */
68 INVGTLB_GID_GFALSE = 0xb,
69 /* Invalid global=0, matched gid and asid in guest tlb */
70 INVGTLB_GID_GFALSE_ASID = 0xc,
71 /* Invalid global=0 , matched gid, asid and addr in guest tlb */
72 INVGTLB_GID_GFALSE_ASID_ADDR = 0xd,
73 /* Invalid global=1 , matched gid, asid and addr in guest tlb */
74 INVGTLB_GID_GTRUE_ASID_ADDR = 0xe,
75 /* Invalid all gid gva-->gpa guest tlb */
76 INVGTLB_ALLGID_GVA_TO_GPA = 0x10,
77 /* Invalid all gid gpa-->hpa tlb */
78 INVTLB_ALLGID_GPA_TO_HPA = 0x11,
79 /* Invalid all gid tlb, including gva-->gpa and gpa-->hpa */
80 INVTLB_ALLGID = 0x12,
81 /* Invalid matched gid gva-->gpa guest tlb */
82 INVGTLB_GID_GVA_TO_GPA = 0x13,
83 /* Invalid matched gid gpa-->hpa tlb */
84 INVTLB_GID_GPA_TO_HPA = 0x14,
85 /* Invalid matched gid tlb,including gva-->gpa and gpa-->hpa */
86 INVTLB_GID_ALL = 0x15,
87 /* Invalid matched gid and addr gpa-->hpa tlb */
88 INVTLB_GID_ADDR = 0x16,
89};
90
91/*
92 * invtlb op info addr
93 * (0x1 << 26) | (0x24 << 20) | (0x13 << 15) |
94 * (addr << 10) | (info << 5) | op
95 */
96static inline void invtlb(u32 op, u32 info, u64 addr)
97{
98 __asm__ __volatile__(
99 "parse_r addr,%0\n\t"
100 "parse_r info,%1\n\t"
101 ".word ((0x6498000) | (addr << 10) | (info << 5) | %2)\n\t"
102 :
103 : "r"(addr), "r"(info), "i"(op)
104 :
105 );
106}
107
108static inline void invtlb_addr(u32 op, u32 info, u64 addr)
109{
110 __asm__ __volatile__(
111 "parse_r addr,%0\n\t"
112 ".word ((0x6498000) | (addr << 10) | (0 << 5) | %1)\n\t"
113 :
114 : "r"(addr), "i"(op)
115 :
116 );
117}
118
119static inline void invtlb_info(u32 op, u32 info, u64 addr)
120{
121 __asm__ __volatile__(
122 "parse_r info,%0\n\t"
123 ".word ((0x6498000) | (0 << 10) | (info << 5) | %1)\n\t"
124 :
125 : "r"(info), "i"(op)
126 :
127 );
128}
129
130static inline void invtlb_all(u32 op, u32 info, u64 addr)
131{
132 __asm__ __volatile__(
133 ".word ((0x6498000) | (0 << 10) | (0 << 5) | %0)\n\t"
134 :
135 : "i"(op)
136 :
137 );
138}
139
140/*
141 * LoongArch doesn't need any special per-pte or per-vma handling, except
142 * we need to flush cache for area to be unmapped.
143 */
144#define tlb_start_vma(tlb, vma) \
145 do { \
146 if (!(tlb)->fullmm) \
147 flush_cache_range(vma, vma->vm_start, vma->vm_end); \
148 } while (0)
149#define tlb_end_vma(tlb, vma) do { } while (0)
150#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
151
152static void tlb_flush(struct mmu_gather *tlb);
153
154#define tlb_flush tlb_flush
155#include <asm-generic/tlb.h>
156
157static inline void tlb_flush(struct mmu_gather *tlb)
158{
159 struct vm_area_struct vma;
160
161 vma.vm_mm = tlb->mm;
162 vma.vm_flags = 0;
163 if (tlb->fullmm) {
164 flush_tlb_mm(tlb->mm);
165 return;
166 }
167
168 flush_tlb_range(&vma, tlb->start, tlb->end);
169}
170
171extern void handle_tlb_load(void);
172extern void handle_tlb_store(void);
173extern void handle_tlb_modify(void);
174extern void handle_tlb_refill(void);
175extern void handle_tlb_protect(void);
176
177extern void dump_tlb_all(void);
178extern void dump_tlb_regs(void);
179
180#endif /* __ASM_TLB_H */
arch/loongarch/include/asm/tlbflush.h+35−0
@@ -0,0 +1,35 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#ifndef __ASM_TLBFLUSH_H
6#define __ASM_TLBFLUSH_H
7
8#include <linux/mm.h>
9
10/*
11 * TLB flushing:
12 *
13 * - flush_tlb_all() flushes all processes TLB entries
14 * - flush_tlb_mm(mm) flushes the specified mm context TLB entries
15 * - flush_tlb_page(vma, vmaddr) flushes one page
16 * - flush_tlb_range(vma, start, end) flushes a range of pages
17 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
18 */
19extern void local_flush_tlb_all(void);
20extern void local_flush_tlb_user(void);
21extern void local_flush_tlb_kernel(void);
22extern void local_flush_tlb_mm(struct mm_struct *mm);
23extern void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
24extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end);
25extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
26extern void local_flush_tlb_one(unsigned long vaddr);
27
28#define flush_tlb_all() local_flush_tlb_all()
29#define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
30#define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end)
31#define flush_tlb_kernel_range(vmaddr, end) local_flush_tlb_kernel_range(vmaddr, end)
32#define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page)
33#define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr)
34
35#endif /* __ASM_TLBFLUSH_H */
arch/loongarch/include/asm/vmalloc.h+4−0
@@ -0,0 +1,4 @@
1#ifndef _ASM_LOONGARCH_VMALLOC_H
2#define _ASM_LOONGARCH_VMALLOC_H
3
4#endif /* _ASM_LOONGARCH_VMALLOC_H */
arch/loongarch/mm/cache.c+141−0
@@ -0,0 +1,141 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2007 MIPS Technologies, Inc.
8 */
9#include <linux/export.h>
10#include <linux/fcntl.h>
11#include <linux/fs.h>
12#include <linux/highmem.h>
13#include <linux/kernel.h>
14#include <linux/linkage.h>
15#include <linux/mm.h>
16#include <linux/sched.h>
17#include <linux/syscalls.h>
18
19#include <asm/cacheflush.h>
20#include <asm/cpu.h>
21#include <asm/cpu-features.h>
22#include <asm/dma.h>
23#include <asm/loongarch.h>
24#include <asm/processor.h>
25#include <asm/setup.h>
26
27/*
28 * LoongArch maintains ICache/DCache coherency by hardware,
29 * we just need "ibar" to avoid instruction hazard here.
30 */
31void local_flush_icache_range(unsigned long start, unsigned long end)
32{
33 asm volatile ("\tibar 0\n"::);
34}
35EXPORT_SYMBOL(local_flush_icache_range);
36
37void cache_error_setup(void)
38{
39 extern char __weak except_vec_cex;
40 set_merr_handler(0x0, &except_vec_cex, 0x80);
41}
42
43static unsigned long icache_size __read_mostly;
44static unsigned long dcache_size __read_mostly;
45static unsigned long vcache_size __read_mostly;
46static unsigned long scache_size __read_mostly;
47
48static char *way_string[] = { NULL, "direct mapped", "2-way",
49 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
50 "9-way", "10-way", "11-way", "12-way",
51 "13-way", "14-way", "15-way", "16-way",
52};
53
54static void probe_pcache(void)
55{
56 struct cpuinfo_loongarch *c = &current_cpu_data;
57 unsigned int lsize, sets, ways;
58 unsigned int config;
59
60 config = read_cpucfg(LOONGARCH_CPUCFG17);
61 lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE);
62 sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS);
63 ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1;
64
65 c->icache.linesz = lsize;
66 c->icache.sets = sets;
67 c->icache.ways = ways;
68 icache_size = sets * ways * lsize;
69 c->icache.waysize = icache_size / c->icache.ways;
70
71 config = read_cpucfg(LOONGARCH_CPUCFG18);
72 lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE);
73 sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS);
74 ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1;
75
76 c->dcache.linesz = lsize;
77 c->dcache.sets = sets;
78 c->dcache.ways = ways;
79 dcache_size = sets * ways * lsize;
80 c->dcache.waysize = dcache_size / c->dcache.ways;
81
82 c->options |= LOONGARCH_CPU_PREFETCH;
83
84 pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
85 icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz);
86
87 pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
88 dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz);
89}
90
91static void probe_vcache(void)
92{
93 struct cpuinfo_loongarch *c = &current_cpu_data;
94 unsigned int lsize, sets, ways;
95 unsigned int config;
96
97 config = read_cpucfg(LOONGARCH_CPUCFG19);
98 lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE);
99 sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS);
100 ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1;
101
102 c->vcache.linesz = lsize;
103 c->vcache.sets = sets;
104 c->vcache.ways = ways;
105 vcache_size = lsize * sets * ways;
106 c->vcache.waysize = vcache_size / c->vcache.ways;
107
108 pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
109 vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
110}
111
112static void probe_scache(void)
113{
114 struct cpuinfo_loongarch *c = &current_cpu_data;
115 unsigned int lsize, sets, ways;
116 unsigned int config;
117
118 config = read_cpucfg(LOONGARCH_CPUCFG20);
119 lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE);
120 sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS);
121 ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1;
122
123 c->scache.linesz = lsize;
124 c->scache.sets = sets;
125 c->scache.ways = ways;
126 /* 4 cores. scaches are shared */
127 scache_size = lsize * sets * ways;
128 c->scache.waysize = scache_size / c->scache.ways;
129
130 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
131 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
132}
133
134void cpu_cache_init(void)
135{
136 probe_pcache();
137 probe_vcache();
138 probe_scache();
139
140 shm_align_mask = PAGE_SIZE - 1;
141}
arch/loongarch/mm/extable.c+22−0
@@ -0,0 +1,22 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/extable.h>
6#include <linux/spinlock.h>
7#include <asm/branch.h>
8#include <linux/uaccess.h>
9
10int fixup_exception(struct pt_regs *regs)
11{
12 const struct exception_table_entry *fixup;
13
14 fixup = search_exception_tables(exception_era(regs));
15 if (fixup) {
16 regs->csr_era = fixup->fixup;
17
18 return 1;
19 }
20
21 return 0;
22}
arch/loongarch/mm/fault.c+261−0
@@ -0,0 +1,261 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 1995 - 2000 by Ralf Baechle
7 */
8#include <linux/context_tracking.h>
9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/entry-common.h>
14#include <linux/errno.h>
15#include <linux/string.h>
16#include <linux/types.h>
17#include <linux/ptrace.h>
18#include <linux/ratelimit.h>
19#include <linux/mman.h>
20#include <linux/mm.h>
21#include <linux/smp.h>
22#include <linux/kdebug.h>
23#include <linux/kprobes.h>
24#include <linux/perf_event.h>
25#include <linux/uaccess.h>
26
27#include <asm/branch.h>
28#include <asm/mmu_context.h>
29#include <asm/ptrace.h>
30
31int show_unhandled_signals = 1;
32
33static void __kprobes no_context(struct pt_regs *regs, unsigned long address)
34{
35 const int field = sizeof(unsigned long) * 2;
36
37 /* Are we prepared to handle this kernel fault? */
38 if (fixup_exception(regs))
39 return;
40
41 /*
42 * Oops. The kernel tried to access some bad page. We'll have to
43 * terminate things with extreme prejudice.
44 */
45 bust_spinlocks(1);
46
47 pr_alert("CPU %d Unable to handle kernel paging request at "
48 "virtual address %0*lx, era == %0*lx, ra == %0*lx\n",
49 raw_smp_processor_id(), field, address, field, regs->csr_era,
50 field, regs->regs[1]);
51 die("Oops", regs);
52}
53
54static void __kprobes do_out_of_memory(struct pt_regs *regs, unsigned long address)
55{
56 /*
57 * We ran out of memory, call the OOM killer, and return the userspace
58 * (which will retry the fault, or kill us if we got oom-killed).
59 */
60 if (!user_mode(regs)) {
61 no_context(regs, address);
62 return;
63 }
64 pagefault_out_of_memory();
65}
66
67static void __kprobes do_sigbus(struct pt_regs *regs,
68 unsigned long write, unsigned long address, int si_code)
69{
70 /* Kernel mode? Handle exceptions or die */
71 if (!user_mode(regs)) {
72 no_context(regs, address);
73 return;
74 }
75
76 /*
77 * Send a sigbus, regardless of whether we were in kernel
78 * or user mode.
79 */
80 current->thread.csr_badvaddr = address;
81 current->thread.trap_nr = read_csr_excode();
82 force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
83}
84
85static void __kprobes do_sigsegv(struct pt_regs *regs,
86 unsigned long write, unsigned long address, int si_code)
87{
88 const int field = sizeof(unsigned long) * 2;
89 static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
90
91 /* Kernel mode? Handle exceptions or die */
92 if (!user_mode(regs)) {
93 no_context(regs, address);
94 return;
95 }
96
97 /* User mode accesses just cause a SIGSEGV */
98 current->thread.csr_badvaddr = address;
99 if (!write)
100 current->thread.error_code = 1;
101 else
102 current->thread.error_code = 2;
103 current->thread.trap_nr = read_csr_excode();
104
105 if (show_unhandled_signals &&
106 unhandled_signal(current, SIGSEGV) && __ratelimit(&ratelimit_state)) {
107 pr_info("do_page_fault(): sending SIGSEGV to %s for invalid %s %0*lx\n",
108 current->comm,
109 write ? "write access to" : "read access from",
110 field, address);
111 pr_info("era = %0*lx in", field,
112 (unsigned long) regs->csr_era);
113 print_vma_addr(KERN_CONT " ", regs->csr_era);
114 pr_cont("\n");
115 pr_info("ra = %0*lx in", field,
116 (unsigned long) regs->regs[1]);
117 print_vma_addr(KERN_CONT " ", regs->regs[1]);
118 pr_cont("\n");
119 }
120 force_sig_fault(SIGSEGV, si_code, (void __user *)address);
121}
122
123/*
124 * This routine handles page faults. It determines the address,
125 * and the problem, and then passes it off to one of the appropriate
126 * routines.
127 */
128static void __kprobes __do_page_fault(struct pt_regs *regs,
129 unsigned long write, unsigned long address)
130{
131 int si_code = SEGV_MAPERR;
132 unsigned int flags = FAULT_FLAG_DEFAULT;
133 struct task_struct *tsk = current;
134 struct mm_struct *mm = tsk->mm;
135 struct vm_area_struct *vma = NULL;
136 vm_fault_t fault;
137
138 /*
139 * We fault-in kernel-space virtual memory on-demand. The
140 * 'reference' page table is init_mm.pgd.
141 *
142 * NOTE! We MUST NOT take any locks for this case. We may
143 * be in an interrupt or a critical region, and should
144 * only copy the information from the master page table,
145 * nothing more.
146 */
147 if (address & __UA_LIMIT) {
148 if (!user_mode(regs))
149 no_context(regs, address);
150 else
151 do_sigsegv(regs, write, address, si_code);
152 return;
153 }
154
155 /*
156 * If we're in an interrupt or have no user
157 * context, we must not take the fault..
158 */
159 if (faulthandler_disabled() || !mm) {
160 do_sigsegv(regs, write, address, si_code);
161 return;
162 }
163
164 if (user_mode(regs))
165 flags |= FAULT_FLAG_USER;
166
167 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
168retry:
169 mmap_read_lock(mm);
170 vma = find_vma(mm, address);
171 if (!vma)
172 goto bad_area;
173 if (vma->vm_start <= address)
174 goto good_area;
175 if (!(vma->vm_flags & VM_GROWSDOWN))
176 goto bad_area;
177 if (!expand_stack(vma, address))
178 goto good_area;
179/*
180 * Something tried to access memory that isn't in our memory map..
181 * Fix it, but check if it's kernel or user first..
182 */
183bad_area:
184 mmap_read_unlock(mm);
185 do_sigsegv(regs, write, address, si_code);
186 return;
187
188/*
189 * Ok, we have a good vm_area for this memory access, so
190 * we can handle it..
191 */
192good_area:
193 si_code = SEGV_ACCERR;
194
195 if (write) {
196 flags |= FAULT_FLAG_WRITE;
197 if (!(vma->vm_flags & VM_WRITE))
198 goto bad_area;
199 } else {
200 if (!(vma->vm_flags & VM_READ) && address != exception_era(regs))
201 goto bad_area;
202 if (!(vma->vm_flags & VM_EXEC) && address == exception_era(regs))
203 goto bad_area;
204 }
205
206 /*
207 * If for any reason at all we couldn't handle the fault,
208 * make sure we exit gracefully rather than endlessly redo
209 * the fault.
210 */
211 fault = handle_mm_fault(vma, address, flags, regs);
212
213 if (fault_signal_pending(fault, regs)) {
214 if (!user_mode(regs))
215 no_context(regs, address);
216 return;
217 }
218
219 if (unlikely(fault & VM_FAULT_RETRY)) {
220 flags |= FAULT_FLAG_TRIED;
221
222 /*
223 * No need to mmap_read_unlock(mm) as we would
224 * have already released it in __lock_page_or_retry
225 * in mm/filemap.c.
226 */
227 goto retry;
228 }
229 if (unlikely(fault & VM_FAULT_ERROR)) {
230 mmap_read_unlock(mm);
231 if (fault & VM_FAULT_OOM) {
232 do_out_of_memory(regs, address);
233 return;
234 } else if (fault & VM_FAULT_SIGSEGV) {
235 do_sigsegv(regs, write, address, si_code);
236 return;
237 } else if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON|VM_FAULT_HWPOISON_LARGE)) {
238 do_sigbus(regs, write, address, si_code);
239 return;
240 }
241 BUG();
242 }
243
244 mmap_read_unlock(mm);
245}
246
247asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
248 unsigned long write, unsigned long address)
249{
250 irqentry_state_t state = irqentry_enter(regs);
251
252 /* Enable interrupt if enabled in parent context */
253 if (likely(regs->csr_prmd & CSR_PRMD_PIE))
254 local_irq_enable();
255
256 __do_page_fault(regs, write, address);
257
258 local_irq_disable();
259
260 irqentry_exit(regs, state);
261}
arch/loongarch/mm/hugetlbpage.c+87−0
@@ -0,0 +1,87 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#include <linux/fs.h>
7#include <linux/mm.h>
8#include <linux/hugetlb.h>
9#include <linux/pagemap.h>
10#include <linux/err.h>
11#include <linux/sysctl.h>
12#include <asm/mman.h>
13#include <asm/tlb.h>
14#include <asm/tlbflush.h>
15
16pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
17 unsigned long addr, unsigned long sz)
18{
19 pgd_t *pgd;
20 p4d_t *p4d;
21 pud_t *pud;
22 pte_t *pte = NULL;
23
24 pgd = pgd_offset(mm, addr);
25 p4d = p4d_alloc(mm, pgd, addr);
26 pud = pud_alloc(mm, p4d, addr);
27 if (pud)
28 pte = (pte_t *)pmd_alloc(mm, pud, addr);
29
30 return pte;
31}
32
33pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
34 unsigned long sz)
35{
36 pgd_t *pgd;
37 p4d_t *p4d;
38 pud_t *pud;
39 pmd_t *pmd = NULL;
40
41 pgd = pgd_offset(mm, addr);
42 if (pgd_present(*pgd)) {
43 p4d = p4d_offset(pgd, addr);
44 if (p4d_present(*p4d)) {
45 pud = pud_offset(p4d, addr);
46 if (pud_present(*pud))
47 pmd = pmd_offset(pud, addr);
48 }
49 }
50 return (pte_t *) pmd;
51}
52
53/*
54 * This function checks for proper alignment of input addr and len parameters.
55 */
56int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
57{
58 if (len & ~HPAGE_MASK)
59 return -EINVAL;
60 if (addr & ~HPAGE_MASK)
61 return -EINVAL;
62 return 0;
63}
64
65int pmd_huge(pmd_t pmd)
66{
67 return (pmd_val(pmd) & _PAGE_HUGE) != 0;
68}
69
70int pud_huge(pud_t pud)
71{
72 return (pud_val(pud) & _PAGE_HUGE) != 0;
73}
74
75uint64_t pmd_to_entrylo(unsigned long pmd_val)
76{
77 uint64_t val;
78 /* PMD as PTE. Must be huge page */
79 if (!pmd_huge(__pmd(pmd_val)))
80 panic("%s", __func__);
81
82 val = pmd_val ^ _PAGE_HUGE;
83 val |= ((val & _PAGE_HGLOBAL) >>
84 (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT));
85
86 return val;
87}
arch/loongarch/mm/init.c+165−0
@@ -0,0 +1,165 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/init.h>
6#include <linux/export.h>
7#include <linux/signal.h>
8#include <linux/sched.h>
9#include <linux/smp.h>
10#include <linux/kernel.h>
11#include <linux/errno.h>
12#include <linux/string.h>
13#include <linux/types.h>
14#include <linux/pagemap.h>
15#include <linux/memblock.h>
16#include <linux/memremap.h>
17#include <linux/mm.h>
18#include <linux/mman.h>
19#include <linux/highmem.h>
20#include <linux/swap.h>
21#include <linux/proc_fs.h>
22#include <linux/pfn.h>
23#include <linux/hardirq.h>
24#include <linux/gfp.h>
25#include <linux/initrd.h>
26#include <linux/mmzone.h>
27
28#include <asm/asm-offsets.h>
29#include <asm/bootinfo.h>
30#include <asm/cpu.h>
31#include <asm/dma.h>
32#include <asm/mmu_context.h>
33#include <asm/sections.h>
34#include <asm/pgtable.h>
35#include <asm/pgalloc.h>
36#include <asm/tlb.h>
37
38/*
39 * We have up to 8 empty zeroed pages so we can map one of the right colour
40 * when needed. Since page is never written to after the initialization we
41 * don't have to care about aliases on other CPUs.
42 */
43unsigned long empty_zero_page, zero_page_mask;
44EXPORT_SYMBOL_GPL(empty_zero_page);
45EXPORT_SYMBOL(zero_page_mask);
46
47void setup_zero_pages(void)
48{
49 unsigned int order, i;
50 struct page *page;
51
52 order = 0;
53
54 empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
55 if (!empty_zero_page)
56 panic("Oh boy, that early out of memory?");
57
58 page = virt_to_page((void *)empty_zero_page);
59 split_page(page, order);
60 for (i = 0; i < (1 << order); i++, page++)
61 mark_page_reserved(page);
62
63 zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
64}
65
66void copy_user_highpage(struct page *to, struct page *from,
67 unsigned long vaddr, struct vm_area_struct *vma)
68{
69 void *vfrom, *vto;
70
71 vto = kmap_atomic(to);
72 vfrom = kmap_atomic(from);
73 copy_page(vto, vfrom);
74 kunmap_atomic(vfrom);
75 kunmap_atomic(vto);
76 /* Make sure this page is cleared on other CPU's too before using it */
77 smp_wmb();
78}
79
80int __ref page_is_ram(unsigned long pfn)
81{
82 unsigned long addr = PFN_PHYS(pfn);
83
84 return memblock_is_memory(addr) && !memblock_is_reserved(addr);
85}
86
87void __init paging_init(void)
88{
89 unsigned long max_zone_pfns[MAX_NR_ZONES];
90
91#ifdef CONFIG_ZONE_DMA
92 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
93#endif
94#ifdef CONFIG_ZONE_DMA32
95 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
96#endif
97 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
98
99 free_area_init(max_zone_pfns);
100}
101
102void __init mem_init(void)
103{
104 max_mapnr = max_low_pfn;
105 high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
106
107 memblock_free_all();
108 setup_zero_pages(); /* Setup zeroed pages. */
109}
110
111void __ref free_initmem(void)
112{
113 free_initmem_default(POISON_FREE_INITMEM);
114}
115
116#ifdef CONFIG_MEMORY_HOTPLUG
117int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params)
118{
119 unsigned long start_pfn = start >> PAGE_SHIFT;
120 unsigned long nr_pages = size >> PAGE_SHIFT;
121 int ret;
122
123 ret = __add_pages(nid, start_pfn, nr_pages, params);
124
125 if (ret)
126 pr_warn("%s: Problem encountered in __add_pages() as ret=%d\n",
127 __func__, ret);
128
129 return ret;
130}
131
132#ifdef CONFIG_MEMORY_HOTREMOVE
133void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
134{
135 unsigned long start_pfn = start >> PAGE_SHIFT;
136 unsigned long nr_pages = size >> PAGE_SHIFT;
137 struct page *page = pfn_to_page(start_pfn);
138
139 /* With altmap the first mapped page is offset from @start */
140 if (altmap)
141 page += vmem_altmap_offset(altmap);
142 __remove_pages(start_pfn, nr_pages, altmap);
143}
144#endif
145#endif
146
147/*
148 * Align swapper_pg_dir in to 64K, allows its address to be loaded
149 * with a single LUI instruction in the TLB handlers. If we used
150 * __aligned(64K), its size would get rounded up to the alignment
151 * size, and waste space. So we place it in its own section and align
152 * it in the linker script.
153 */
154pgd_t swapper_pg_dir[_PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
155
156pgd_t invalid_pg_dir[_PTRS_PER_PGD] __page_aligned_bss;
157#ifndef __PAGETABLE_PUD_FOLDED
158pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
159#endif
160#ifndef __PAGETABLE_PMD_FOLDED
161pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
162EXPORT_SYMBOL_GPL(invalid_pmd_table);
163#endif
164pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
165EXPORT_SYMBOL(invalid_pte_table);
arch/loongarch/mm/ioremap.c+27−0
@@ -0,0 +1,27 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5
6#include <asm/io.h>
7
8void __init __iomem *early_ioremap(u64 phys_addr, unsigned long size)
9{
10 return ((void __iomem *)TO_CACHE(phys_addr));
11}
12
13void __init early_iounmap(void __iomem *addr, unsigned long size)
14{
15
16}
17
18void *early_memremap_ro(resource_size_t phys_addr, unsigned long size)
19{
20 return early_memremap(phys_addr, size);
21}
22
23void *early_memremap_prot(resource_size_t phys_addr, unsigned long size,
24 unsigned long prot_val)
25{
26 return early_memremap(phys_addr, size);
27}
arch/loongarch/mm/maccess.c+10−0
@@ -0,0 +1,10 @@
1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/uaccess.h>
4#include <linux/kernel.h>
5
6bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
7{
8 /* highest bit set means kernel space */
9 return (unsigned long)unsafe_src >> (BITS_PER_LONG - 1);
10}
arch/loongarch/mm/mmap.c+125−0
@@ -0,0 +1,125 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/compiler.h>
6#include <linux/elf-randomize.h>
7#include <linux/errno.h>
8#include <linux/mm.h>
9#include <linux/mman.h>
10#include <linux/export.h>
11#include <linux/personality.h>
12#include <linux/random.h>
13#include <linux/sched/signal.h>
14#include <linux/sched/mm.h>
15
16unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */
17EXPORT_SYMBOL(shm_align_mask);
18
19#define COLOUR_ALIGN(addr, pgoff) \
20 ((((addr) + shm_align_mask) & ~shm_align_mask) + \
21 (((pgoff) << PAGE_SHIFT) & shm_align_mask))
22
23enum mmap_allocation_direction {UP, DOWN};
24
25static unsigned long arch_get_unmapped_area_common(struct file *filp,
26 unsigned long addr0, unsigned long len, unsigned long pgoff,
27 unsigned long flags, enum mmap_allocation_direction dir)
28{
29 struct mm_struct *mm = current->mm;
30 struct vm_area_struct *vma;
31 unsigned long addr = addr0;
32 int do_color_align;
33 struct vm_unmapped_area_info info;
34
35 if (unlikely(len > TASK_SIZE))
36 return -ENOMEM;
37
38 if (flags & MAP_FIXED) {
39 /* Even MAP_FIXED mappings must reside within TASK_SIZE */
40 if (TASK_SIZE - len < addr)
41 return -EINVAL;
42
43 /*
44 * We do not accept a shared mapping if it would violate
45 * cache aliasing constraints.
46 */
47 if ((flags & MAP_SHARED) &&
48 ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask))
49 return -EINVAL;
50 return addr;
51 }
52
53 do_color_align = 0;
54 if (filp || (flags & MAP_SHARED))
55 do_color_align = 1;
56
57 /* requesting a specific address */
58 if (addr) {
59 if (do_color_align)
60 addr = COLOUR_ALIGN(addr, pgoff);
61 else
62 addr = PAGE_ALIGN(addr);
63
64 vma = find_vma(mm, addr);
65 if (TASK_SIZE - len >= addr &&
66 (!vma || addr + len <= vm_start_gap(vma)))
67 return addr;
68 }
69
70 info.length = len;
71 info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
72 info.align_offset = pgoff << PAGE_SHIFT;
73
74 if (dir == DOWN) {
75 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
76 info.low_limit = PAGE_SIZE;
77 info.high_limit = mm->mmap_base;
78 addr = vm_unmapped_area(&info);
79
80 if (!(addr & ~PAGE_MASK))
81 return addr;
82
83 /*
84 * A failed mmap() very likely causes application failure,
85 * so fall back to the bottom-up function here. This scenario
86 * can happen with large stack limits and large mmap()
87 * allocations.
88 */
89 }
90
91 info.flags = 0;
92 info.low_limit = mm->mmap_base;
93 info.high_limit = TASK_SIZE;
94 return vm_unmapped_area(&info);
95}
96
97unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr0,
98 unsigned long len, unsigned long pgoff, unsigned long flags)
99{
100 return arch_get_unmapped_area_common(filp,
101 addr0, len, pgoff, flags, UP);
102}
103
104/*
105 * There is no need to export this but sched.h declares the function as
106 * extern so making it static here results in an error.
107 */
108unsigned long arch_get_unmapped_area_topdown(struct file *filp,
109 unsigned long addr0, unsigned long len, unsigned long pgoff,
110 unsigned long flags)
111{
112 return arch_get_unmapped_area_common(filp,
113 addr0, len, pgoff, flags, DOWN);
114}
115
116int __virt_addr_valid(volatile void *kaddr)
117{
118 unsigned long vaddr = (unsigned long)kaddr;
119
120 if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base))
121 return 0;
122
123 return pfn_valid(PFN_DOWN(virt_to_phys(kaddr)));
124}
125EXPORT_SYMBOL_GPL(__virt_addr_valid);
arch/loongarch/mm/page.S+84−0
@@ -0,0 +1,84 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/linkage.h>
6#include <asm/asm.h>
7#include <asm/export.h>
8#include <asm/page.h>
9#include <asm/regdef.h>
10
11 .align 5
12SYM_FUNC_START(clear_page)
13 lu12i.w t0, 1 << (PAGE_SHIFT - 12)
14 add.d t0, t0, a0
151:
16 st.d zero, a0, 0
17 st.d zero, a0, 8
18 st.d zero, a0, 16
19 st.d zero, a0, 24
20 st.d zero, a0, 32
21 st.d zero, a0, 40
22 st.d zero, a0, 48
23 st.d zero, a0, 56
24 addi.d a0, a0, 128
25 st.d zero, a0, -64
26 st.d zero, a0, -56
27 st.d zero, a0, -48
28 st.d zero, a0, -40
29 st.d zero, a0, -32
30 st.d zero, a0, -24
31 st.d zero, a0, -16
32 st.d zero, a0, -8
33 bne t0, a0, 1b
34
35 jirl $r0, ra, 0
36SYM_FUNC_END(clear_page)
37EXPORT_SYMBOL(clear_page)
38
39.align 5
40SYM_FUNC_START(copy_page)
41 lu12i.w t8, 1 << (PAGE_SHIFT - 12)
42 add.d t8, t8, a0
431:
44 ld.d t0, a1, 0
45 ld.d t1, a1, 8
46 ld.d t2, a1, 16
47 ld.d t3, a1, 24
48 ld.d t4, a1, 32
49 ld.d t5, a1, 40
50 ld.d t6, a1, 48
51 ld.d t7, a1, 56
52
53 st.d t0, a0, 0
54 st.d t1, a0, 8
55 ld.d t0, a1, 64
56 ld.d t1, a1, 72
57 st.d t2, a0, 16
58 st.d t3, a0, 24
59 ld.d t2, a1, 80
60 ld.d t3, a1, 88
61 st.d t4, a0, 32
62 st.d t5, a0, 40
63 ld.d t4, a1, 96
64 ld.d t5, a1, 104
65 st.d t6, a0, 48
66 st.d t7, a0, 56
67 ld.d t6, a1, 112
68 ld.d t7, a1, 120
69 addi.d a0, a0, 128
70 addi.d a1, a1, 128
71
72 st.d t0, a0, -64
73 st.d t1, a0, -56
74 st.d t2, a0, -48
75 st.d t3, a0, -40
76 st.d t4, a0, -32
77 st.d t5, a0, -24
78 st.d t6, a0, -16
79 st.d t7, a0, -8
80
81 bne t8, a0, 1b
82 jirl $r0, ra, 0
83SYM_FUNC_END(copy_page)
84EXPORT_SYMBOL(copy_page)
arch/loongarch/mm/pgtable.c+130−0
@@ -0,0 +1,130 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/init.h>
6#include <linux/export.h>
7#include <linux/mm.h>
8#include <asm/pgalloc.h>
9#include <asm/pgtable.h>
10#include <asm/tlbflush.h>
11
12pgd_t *pgd_alloc(struct mm_struct *mm)
13{
14 pgd_t *ret, *init;
15
16 ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER);
17 if (ret) {
18 init = pgd_offset(&init_mm, 0UL);
19 pgd_init((unsigned long)ret);
20 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
21 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
22 }
23
24 return ret;
25}
26EXPORT_SYMBOL_GPL(pgd_alloc);
27
28void pgd_init(unsigned long page)
29{
30 unsigned long *p, *end;
31 unsigned long entry;
32
33#if !defined(__PAGETABLE_PUD_FOLDED)
34 entry = (unsigned long)invalid_pud_table;
35#elif !defined(__PAGETABLE_PMD_FOLDED)
36 entry = (unsigned long)invalid_pmd_table;
37#else
38 entry = (unsigned long)invalid_pte_table;
39#endif
40
41 p = (unsigned long *) page;
42 end = p + PTRS_PER_PGD;
43
44 do {
45 p[0] = entry;
46 p[1] = entry;
47 p[2] = entry;
48 p[3] = entry;
49 p[4] = entry;
50 p += 8;
51 p[-3] = entry;
52 p[-2] = entry;
53 p[-1] = entry;
54 } while (p != end);
55}
56EXPORT_SYMBOL_GPL(pgd_init);
57
58#ifndef __PAGETABLE_PMD_FOLDED
59void pmd_init(unsigned long addr, unsigned long pagetable)
60{
61 unsigned long *p, *end;
62
63 p = (unsigned long *) addr;
64 end = p + PTRS_PER_PMD;
65
66 do {
67 p[0] = pagetable;
68 p[1] = pagetable;
69 p[2] = pagetable;
70 p[3] = pagetable;
71 p[4] = pagetable;
72 p += 8;
73 p[-3] = pagetable;
74 p[-2] = pagetable;
75 p[-1] = pagetable;
76 } while (p != end);
77}
78EXPORT_SYMBOL_GPL(pmd_init);
79#endif
80
81#ifndef __PAGETABLE_PUD_FOLDED
82void pud_init(unsigned long addr, unsigned long pagetable)
83{
84 unsigned long *p, *end;
85
86 p = (unsigned long *)addr;
87 end = p + PTRS_PER_PUD;
88
89 do {
90 p[0] = pagetable;
91 p[1] = pagetable;
92 p[2] = pagetable;
93 p[3] = pagetable;
94 p[4] = pagetable;
95 p += 8;
96 p[-3] = pagetable;
97 p[-2] = pagetable;
98 p[-1] = pagetable;
99 } while (p != end);
100}
101#endif
102
103pmd_t mk_pmd(struct page *page, pgprot_t prot)
104{
105 pmd_t pmd;
106
107 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
108
109 return pmd;
110}
111
112void set_pmd_at(struct mm_struct *mm, unsigned long addr,
113 pmd_t *pmdp, pmd_t pmd)
114{
115 *pmdp = pmd;
116 flush_tlb_all();
117}
118
119void __init pagetable_init(void)
120{
121 /* Initialize the entire pgd. */
122 pgd_init((unsigned long)swapper_pg_dir);
123 pgd_init((unsigned long)invalid_pg_dir);
124#ifndef __PAGETABLE_PUD_FOLDED
125 pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table);
126#endif
127#ifndef __PAGETABLE_PMD_FOLDED
128 pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
129#endif
130}
arch/loongarch/mm/tlb.c+282−0
@@ -0,0 +1,282 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/init.h>
6#include <linux/sched.h>
7#include <linux/smp.h>
8#include <linux/mm.h>
9#include <linux/hugetlb.h>
10#include <linux/export.h>
11
12#include <asm/cpu.h>
13#include <asm/bootinfo.h>
14#include <asm/mmu_context.h>
15#include <asm/pgtable.h>
16#include <asm/tlb.h>
17
18void local_flush_tlb_all(void)
19{
20 invtlb_all(INVTLB_CURRENT_ALL, 0, 0);
21}
22EXPORT_SYMBOL(local_flush_tlb_all);
23
24void local_flush_tlb_user(void)
25{
26 invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0);
27}
28EXPORT_SYMBOL(local_flush_tlb_user);
29
30void local_flush_tlb_kernel(void)
31{
32 invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0);
33}
34EXPORT_SYMBOL(local_flush_tlb_kernel);
35
36/*
37 * All entries common to a mm share an asid. To effectively flush
38 * these entries, we just bump the asid.
39 */
40void local_flush_tlb_mm(struct mm_struct *mm)
41{
42 int cpu;
43
44 preempt_disable();
45
46 cpu = smp_processor_id();
47
48 if (asid_valid(mm, cpu))
49 drop_mmu_context(mm, cpu);
50 else
51 cpumask_clear_cpu(cpu, mm_cpumask(mm));
52
53 preempt_enable();
54}
55
56void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
57 unsigned long end)
58{
59 struct mm_struct *mm = vma->vm_mm;
60 int cpu = smp_processor_id();
61
62 if (asid_valid(mm, cpu)) {
63 unsigned long size, flags;
64
65 local_irq_save(flags);
66 start = round_down(start, PAGE_SIZE << 1);
67 end = round_up(end, PAGE_SIZE << 1);
68 size = (end - start) >> (PAGE_SHIFT + 1);
69 if (size <= (current_cpu_data.tlbsizestlbsets ?
70 current_cpu_data.tlbsize / 8 :
71 current_cpu_data.tlbsize / 2)) {
72 int asid = cpu_asid(cpu, mm);
73
74 while (start < end) {
75 invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start);
76 start += (PAGE_SIZE << 1);
77 }
78 } else {
79 drop_mmu_context(mm, cpu);
80 }
81 local_irq_restore(flags);
82 } else {
83 cpumask_clear_cpu(cpu, mm_cpumask(mm));
84 }
85}
86
87void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
88{
89 unsigned long size, flags;
90
91 local_irq_save(flags);
92 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
93 size = (size + 1) >> 1;
94 if (size <= (current_cpu_data.tlbsizestlbsets ?
95 current_cpu_data.tlbsize / 8 :
96 current_cpu_data.tlbsize / 2)) {
97
98 start &= (PAGE_MASK << 1);
99 end += ((PAGE_SIZE << 1) - 1);
100 end &= (PAGE_MASK << 1);
101
102 while (start < end) {
103 invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start);
104 start += (PAGE_SIZE << 1);
105 }
106 } else {
107 local_flush_tlb_kernel();
108 }
109 local_irq_restore(flags);
110}
111
112void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
113{
114 int cpu = smp_processor_id();
115
116 if (asid_valid(vma->vm_mm, cpu)) {
117 int newpid;
118
119 newpid = cpu_asid(cpu, vma->vm_mm);
120 page &= (PAGE_MASK << 1);
121 invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page);
122 } else {
123 cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm));
124 }
125}
126
127/*
128 * This one is only used for pages with the global bit set so we don't care
129 * much about the ASID.
130 */
131void local_flush_tlb_one(unsigned long page)
132{
133 page &= (PAGE_MASK << 1);
134 invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page);
135}
136
137static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
138{
139#ifdef CONFIG_HUGETLB_PAGE
140 int idx;
141 unsigned long lo;
142 unsigned long flags;
143
144 local_irq_save(flags);
145
146 address &= (PAGE_MASK << 1);
147 write_csr_entryhi(address);
148 tlb_probe();
149 idx = read_csr_tlbidx();
150 write_csr_pagesize(PS_HUGE_SIZE);
151 lo = pmd_to_entrylo(pte_val(*ptep));
152 write_csr_entrylo0(lo);
153 write_csr_entrylo1(lo + (HPAGE_SIZE >> 1));
154
155 if (idx < 0)
156 tlb_write_random();
157 else
158 tlb_write_indexed();
159 write_csr_pagesize(PS_DEFAULT_SIZE);
160
161 local_irq_restore(flags);
162#endif
163}
164
165void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
166{
167 int idx;
168 unsigned long flags;
169
170 /*
171 * Handle debugger faulting in for debugee.
172 */
173 if (current->active_mm != vma->vm_mm)
174 return;
175
176 if (pte_val(*ptep) & _PAGE_HUGE) {
177 __update_hugetlb(vma, address, ptep);
178 return;
179 }
180
181 local_irq_save(flags);
182
183 if ((unsigned long)ptep & sizeof(pte_t))
184 ptep--;
185
186 address &= (PAGE_MASK << 1);
187 write_csr_entryhi(address);
188 tlb_probe();
189 idx = read_csr_tlbidx();
190 write_csr_pagesize(PS_DEFAULT_SIZE);
191 write_csr_entrylo0(pte_val(*ptep++));
192 write_csr_entrylo1(pte_val(*ptep));
193 if (idx < 0)
194 tlb_write_random();
195 else
196 tlb_write_indexed();
197
198 local_irq_restore(flags);
199}
200
201static void setup_ptwalker(void)
202{
203 unsigned long pwctl0, pwctl1;
204 unsigned long pgd_i = 0, pgd_w = 0;
205 unsigned long pud_i = 0, pud_w = 0;
206 unsigned long pmd_i = 0, pmd_w = 0;
207 unsigned long pte_i = 0, pte_w = 0;
208
209 pgd_i = PGDIR_SHIFT;
210 pgd_w = PAGE_SHIFT - 3;
211#if CONFIG_PGTABLE_LEVELS > 3
212 pud_i = PUD_SHIFT;
213 pud_w = PAGE_SHIFT - 3;
214#endif
215#if CONFIG_PGTABLE_LEVELS > 2
216 pmd_i = PMD_SHIFT;
217 pmd_w = PAGE_SHIFT - 3;
218#endif
219 pte_i = PAGE_SHIFT;
220 pte_w = PAGE_SHIFT - 3;
221
222 pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
223 pwctl1 = pgd_i | pgd_w << 6;
224
225 csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
226 csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
227 csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
228 csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL);
229 csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID);
230}
231
232static void output_pgtable_bits_defines(void)
233{
234#define pr_define(fmt, ...) \
235 pr_debug("#define " fmt, ##__VA_ARGS__)
236
237 pr_debug("#include <asm/asm.h>\n");
238 pr_debug("#include <asm/regdef.h>\n");
239 pr_debug("\n");
240
241 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
242 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
243 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
244 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
245 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
246 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
247 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
248 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
249 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
250 pr_debug("\n");
251}
252
253void setup_tlb_handler(void)
254{
255 static int run_once = 0;
256
257 setup_ptwalker();
258 output_pgtable_bits_defines();
259
260 /* The tlb handlers are generated only once */
261 if (!run_once) {
262 memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
263 local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
264 set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
265 set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
266 set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
267 set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
268 set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
269 set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
270 set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
271 run_once++;
272 }
273}
274
275void tlb_init(void)
276{
277 write_csr_pagesize(PS_DEFAULT_SIZE);
278 write_csr_stlbpgsize(PS_DEFAULT_SIZE);
279 write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE);
280 setup_tlb_handler();
281 local_flush_tlb_all();
282}
arch/loongarch/mm/tlbex.S+477−0
@@ -0,0 +1,477 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <asm/asm.h>
6#include <asm/export.h>
7#include <asm/loongarch.h>
8#include <asm/page.h>
9#include <asm/pgtable.h>
10#include <asm/regdef.h>
11#include <asm/stackframe.h>
12
13 .macro tlb_do_page_fault, write
14 SYM_FUNC_START(tlb_do_page_fault_\write)
15 SAVE_ALL
16 csrrd a2, LOONGARCH_CSR_BADV
17 move a0, sp
18 REG_S a2, sp, PT_BVADDR
19 li.w a1, \write
20 la.abs t0, do_page_fault
21 jirl ra, t0, 0
22 RESTORE_ALL_AND_RET
23 SYM_FUNC_END(tlb_do_page_fault_\write)
24 .endm
25
26 tlb_do_page_fault 0
27 tlb_do_page_fault 1
28
29SYM_FUNC_START(handle_tlb_protect)
30 BACKUP_T0T1
31 SAVE_ALL
32 move a0, sp
33 move a1, zero
34 csrrd a2, LOONGARCH_CSR_BADV
35 REG_S a2, sp, PT_BVADDR
36 la.abs t0, do_page_fault
37 jirl ra, t0, 0
38 RESTORE_ALL_AND_RET
39SYM_FUNC_END(handle_tlb_protect)
40
41SYM_FUNC_START(handle_tlb_load)
42 csrwr t0, EXCEPTION_KS0
43 csrwr t1, EXCEPTION_KS1
44 csrwr ra, EXCEPTION_KS2
45
46 /*
47 * The vmalloc handling is not in the hotpath.
48 */
49 csrrd t0, LOONGARCH_CSR_BADV
50 blt t0, $r0, vmalloc_load
51 csrrd t1, LOONGARCH_CSR_PGDL
52
53vmalloc_done_load:
54 /* Get PGD offset in bytes */
55 srli.d t0, t0, PGDIR_SHIFT
56 andi t0, t0, (PTRS_PER_PGD - 1)
57 slli.d t0, t0, 3
58 add.d t1, t1, t0
59#if CONFIG_PGTABLE_LEVELS > 3
60 csrrd t0, LOONGARCH_CSR_BADV
61 ld.d t1, t1, 0
62 srli.d t0, t0, PUD_SHIFT
63 andi t0, t0, (PTRS_PER_PUD - 1)
64 slli.d t0, t0, 3
65 add.d t1, t1, t0
66#endif
67#if CONFIG_PGTABLE_LEVELS > 2
68 csrrd t0, LOONGARCH_CSR_BADV
69 ld.d t1, t1, 0
70 srli.d t0, t0, PMD_SHIFT
71 andi t0, t0, (PTRS_PER_PMD - 1)
72 slli.d t0, t0, 3
73 add.d t1, t1, t0
74#endif
75 ld.d ra, t1, 0
76
77 /*
78 * For huge tlb entries, pmde doesn't contain an address but
79 * instead contains the tlb pte. Check the PAGE_HUGE bit and
80 * see if we need to jump to huge tlb processing.
81 */
82 andi t0, ra, _PAGE_HUGE
83 bne t0, $r0, tlb_huge_update_load
84
85 csrrd t0, LOONGARCH_CSR_BADV
86 srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
87 andi t0, t0, (PTRS_PER_PTE - 1)
88 slli.d t0, t0, _PTE_T_LOG2
89 add.d t1, ra, t0
90
91 ld.d t0, t1, 0
92 tlbsrch
93
94 srli.d ra, t0, _PAGE_PRESENT_SHIFT
95 andi ra, ra, 1
96 beq ra, $r0, nopage_tlb_load
97
98 ori t0, t0, _PAGE_VALID
99 st.d t0, t1, 0
100 ori t1, t1, 8
101 xori t1, t1, 8
102 ld.d t0, t1, 0
103 ld.d t1, t1, 8
104 csrwr t0, LOONGARCH_CSR_TLBELO0
105 csrwr t1, LOONGARCH_CSR_TLBELO1
106 tlbwr
107leave_load:
108 csrrd t0, EXCEPTION_KS0
109 csrrd t1, EXCEPTION_KS1
110 csrrd ra, EXCEPTION_KS2
111 ertn
112#ifdef CONFIG_64BIT
113vmalloc_load:
114 la.abs t1, swapper_pg_dir
115 b vmalloc_done_load
116#endif
117
118 /*
119 * This is the entry point when build_tlbchange_handler_head
120 * spots a huge page.
121 */
122tlb_huge_update_load:
123 ld.d t0, t1, 0
124 srli.d ra, t0, _PAGE_PRESENT_SHIFT
125 andi ra, ra, 1
126 beq ra, $r0, nopage_tlb_load
127 tlbsrch
128
129 ori t0, t0, _PAGE_VALID
130 st.d t0, t1, 0
131 addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
132 addi.d ra, t1, 0
133 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
134 tlbwr
135
136 csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX
137
138 /*
139 * A huge PTE describes an area the size of the
140 * configured huge page size. This is twice the
141 * of the large TLB entry size we intend to use.
142 * A TLB entry half the size of the configured
143 * huge page size is configured into entrylo0
144 * and entrylo1 to cover the contiguous huge PTE
145 * address space.
146 */
147 /* Huge page: Move Global bit */
148 xori t0, t0, _PAGE_HUGE
149 lu12i.w t1, _PAGE_HGLOBAL >> 12
150 and t1, t0, t1
151 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
152 or t0, t0, t1
153
154 addi.d ra, t0, 0
155 csrwr t0, LOONGARCH_CSR_TLBELO0
156 addi.d t0, ra, 0
157
158 /* Convert to entrylo1 */
159 addi.d t1, $r0, 1
160 slli.d t1, t1, (HPAGE_SHIFT - 1)
161 add.d t0, t0, t1
162 csrwr t0, LOONGARCH_CSR_TLBELO1
163
164 /* Set huge page tlb entry size */
165 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
166 addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
167 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
168
169 tlbfill
170
171 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
172 addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
173 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
174
175nopage_tlb_load:
176 csrrd ra, EXCEPTION_KS2
177 la.abs t0, tlb_do_page_fault_0
178 jirl $r0, t0, 0
179SYM_FUNC_END(handle_tlb_load)
180
181SYM_FUNC_START(handle_tlb_store)
182 csrwr t0, EXCEPTION_KS0
183 csrwr t1, EXCEPTION_KS1
184 csrwr ra, EXCEPTION_KS2
185
186 /*
187 * The vmalloc handling is not in the hotpath.
188 */
189 csrrd t0, LOONGARCH_CSR_BADV
190 blt t0, $r0, vmalloc_store
191 csrrd t1, LOONGARCH_CSR_PGDL
192
193vmalloc_done_store:
194 /* Get PGD offset in bytes */
195 srli.d t0, t0, PGDIR_SHIFT
196 andi t0, t0, (PTRS_PER_PGD - 1)
197 slli.d t0, t0, 3
198 add.d t1, t1, t0
199
200#if CONFIG_PGTABLE_LEVELS > 3
201 csrrd t0, LOONGARCH_CSR_BADV
202 ld.d t1, t1, 0
203 srli.d t0, t0, PUD_SHIFT
204 andi t0, t0, (PTRS_PER_PUD - 1)
205 slli.d t0, t0, 3
206 add.d t1, t1, t0
207#endif
208#if CONFIG_PGTABLE_LEVELS > 2
209 csrrd t0, LOONGARCH_CSR_BADV
210 ld.d t1, t1, 0
211 srli.d t0, t0, PMD_SHIFT
212 andi t0, t0, (PTRS_PER_PMD - 1)
213 slli.d t0, t0, 3
214 add.d t1, t1, t0
215#endif
216 ld.d ra, t1, 0
217
218 /*
219 * For huge tlb entries, pmde doesn't contain an address but
220 * instead contains the tlb pte. Check the PAGE_HUGE bit and
221 * see if we need to jump to huge tlb processing.
222 */
223 andi t0, ra, _PAGE_HUGE
224 bne t0, $r0, tlb_huge_update_store
225
226 csrrd t0, LOONGARCH_CSR_BADV
227 srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
228 andi t0, t0, (PTRS_PER_PTE - 1)
229 slli.d t0, t0, _PTE_T_LOG2
230 add.d t1, ra, t0
231
232 ld.d t0, t1, 0
233 tlbsrch
234
235 srli.d ra, t0, _PAGE_PRESENT_SHIFT
236 andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
237 xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
238 bne ra, $r0, nopage_tlb_store
239
240 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
241 st.d t0, t1, 0
242
243 ori t1, t1, 8
244 xori t1, t1, 8
245 ld.d t0, t1, 0
246 ld.d t1, t1, 8
247 csrwr t0, LOONGARCH_CSR_TLBELO0
248 csrwr t1, LOONGARCH_CSR_TLBELO1
249 tlbwr
250leave_store:
251 csrrd t0, EXCEPTION_KS0
252 csrrd t1, EXCEPTION_KS1
253 csrrd ra, EXCEPTION_KS2
254 ertn
255#ifdef CONFIG_64BIT
256vmalloc_store:
257 la.abs t1, swapper_pg_dir
258 b vmalloc_done_store
259#endif
260
261 /*
262 * This is the entry point when build_tlbchange_handler_head
263 * spots a huge page.
264 */
265tlb_huge_update_store:
266 ld.d t0, t1, 0
267 srli.d ra, t0, _PAGE_PRESENT_SHIFT
268 andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
269 xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
270 bne ra, $r0, nopage_tlb_store
271
272 tlbsrch
273 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
274
275 st.d t0, t1, 0
276 addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
277 addi.d ra, t1, 0
278 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
279 tlbwr
280
281 csrxchg $r0, t1, LOONGARCH_CSR_TLBIDX
282 /*
283 * A huge PTE describes an area the size of the
284 * configured huge page size. This is twice the
285 * of the large TLB entry size we intend to use.
286 * A TLB entry half the size of the configured
287 * huge page size is configured into entrylo0
288 * and entrylo1 to cover the contiguous huge PTE
289 * address space.
290 */
291 /* Huge page: Move Global bit */
292 xori t0, t0, _PAGE_HUGE
293 lu12i.w t1, _PAGE_HGLOBAL >> 12
294 and t1, t0, t1
295 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
296 or t0, t0, t1
297
298 addi.d ra, t0, 0
299 csrwr t0, LOONGARCH_CSR_TLBELO0
300 addi.d t0, ra, 0
301
302 /* Convert to entrylo1 */
303 addi.d t1, $r0, 1
304 slli.d t1, t1, (HPAGE_SHIFT - 1)
305 add.d t0, t0, t1
306 csrwr t0, LOONGARCH_CSR_TLBELO1
307
308 /* Set huge page tlb entry size */
309 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
310 addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
311 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
312
313 tlbfill
314
315 /* Reset default page size */
316 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
317 addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
318 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
319
320nopage_tlb_store:
321 csrrd ra, EXCEPTION_KS2
322 la.abs t0, tlb_do_page_fault_1
323 jirl $r0, t0, 0
324SYM_FUNC_END(handle_tlb_store)
325
326SYM_FUNC_START(handle_tlb_modify)
327 csrwr t0, EXCEPTION_KS0
328 csrwr t1, EXCEPTION_KS1
329 csrwr ra, EXCEPTION_KS2
330
331 /*
332 * The vmalloc handling is not in the hotpath.
333 */
334 csrrd t0, LOONGARCH_CSR_BADV
335 blt t0, $r0, vmalloc_modify
336 csrrd t1, LOONGARCH_CSR_PGDL
337
338vmalloc_done_modify:
339 /* Get PGD offset in bytes */
340 srli.d t0, t0, PGDIR_SHIFT
341 andi t0, t0, (PTRS_PER_PGD - 1)
342 slli.d t0, t0, 3
343 add.d t1, t1, t0
344#if CONFIG_PGTABLE_LEVELS > 3
345 csrrd t0, LOONGARCH_CSR_BADV
346 ld.d t1, t1, 0
347 srli.d t0, t0, PUD_SHIFT
348 andi t0, t0, (PTRS_PER_PUD - 1)
349 slli.d t0, t0, 3
350 add.d t1, t1, t0
351#endif
352#if CONFIG_PGTABLE_LEVELS > 2
353 csrrd t0, LOONGARCH_CSR_BADV
354 ld.d t1, t1, 0
355 srli.d t0, t0, PMD_SHIFT
356 andi t0, t0, (PTRS_PER_PMD - 1)
357 slli.d t0, t0, 3
358 add.d t1, t1, t0
359#endif
360 ld.d ra, t1, 0
361
362 /*
363 * For huge tlb entries, pmde doesn't contain an address but
364 * instead contains the tlb pte. Check the PAGE_HUGE bit and
365 * see if we need to jump to huge tlb processing.
366 */
367 andi t0, ra, _PAGE_HUGE
368 bne t0, $r0, tlb_huge_update_modify
369
370 csrrd t0, LOONGARCH_CSR_BADV
371 srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
372 andi t0, t0, (PTRS_PER_PTE - 1)
373 slli.d t0, t0, _PTE_T_LOG2
374 add.d t1, ra, t0
375
376 ld.d t0, t1, 0
377 tlbsrch
378
379 srli.d ra, t0, _PAGE_WRITE_SHIFT
380 andi ra, ra, 1
381 beq ra, $r0, nopage_tlb_modify
382
383 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
384 st.d t0, t1, 0
385 ori t1, t1, 8
386 xori t1, t1, 8
387 ld.d t0, t1, 0
388 ld.d t1, t1, 8
389 csrwr t0, LOONGARCH_CSR_TLBELO0
390 csrwr t1, LOONGARCH_CSR_TLBELO1
391 tlbwr
392leave_modify:
393 csrrd t0, EXCEPTION_KS0
394 csrrd t1, EXCEPTION_KS1
395 csrrd ra, EXCEPTION_KS2
396 ertn
397#ifdef CONFIG_64BIT
398vmalloc_modify:
399 la.abs t1, swapper_pg_dir
400 b vmalloc_done_modify
401#endif
402
403 /*
404 * This is the entry point when
405 * build_tlbchange_handler_head spots a huge page.
406 */
407tlb_huge_update_modify:
408 ld.d t0, t1, 0
409
410 srli.d ra, t0, _PAGE_WRITE_SHIFT
411 andi ra, ra, 1
412 beq ra, $r0, nopage_tlb_modify
413
414 tlbsrch
415 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
416
417 st.d t0, t1, 0
418 /*
419 * A huge PTE describes an area the size of the
420 * configured huge page size. This is twice the
421 * of the large TLB entry size we intend to use.
422 * A TLB entry half the size of the configured
423 * huge page size is configured into entrylo0
424 * and entrylo1 to cover the contiguous huge PTE
425 * address space.
426 */
427 /* Huge page: Move Global bit */
428 xori t0, t0, _PAGE_HUGE
429 lu12i.w t1, _PAGE_HGLOBAL >> 12
430 and t1, t0, t1
431 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
432 or t0, t0, t1
433
434 addi.d ra, t0, 0
435 csrwr t0, LOONGARCH_CSR_TLBELO0
436 addi.d t0, ra, 0
437
438 /* Convert to entrylo1 */
439 addi.d t1, $r0, 1
440 slli.d t1, t1, (HPAGE_SHIFT - 1)
441 add.d t0, t0, t1
442 csrwr t0, LOONGARCH_CSR_TLBELO1
443
444 /* Set huge page tlb entry size */
445 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
446 addu16i.d t1, $r0, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
447 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
448
449 tlbwr
450
451 /* Reset default page size */
452 addu16i.d t0, $r0, (CSR_TLBIDX_PS >> 16)
453 addu16i.d t1, $r0, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
454 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
455
456nopage_tlb_modify:
457 csrrd ra, EXCEPTION_KS2
458 la.abs t0, tlb_do_page_fault_1
459 jirl $r0, t0, 0
460SYM_FUNC_END(handle_tlb_modify)
461
462SYM_FUNC_START(handle_tlb_refill)
463 csrwr t0, LOONGARCH_CSR_TLBRSAVE
464 csrrd t0, LOONGARCH_CSR_PGD
465 lddir t0, t0, 3
466#if CONFIG_PGTABLE_LEVELS > 3
467 lddir t0, t0, 2
468#endif
469#if CONFIG_PGTABLE_LEVELS > 2
470 lddir t0, t0, 1
471#endif
472 ldpte t0, 0
473 ldpte t0, 1
474 tlbfill
475 csrrd t0, LOONGARCH_CSR_TLBRSAVE
476 ertn
477SYM_FUNC_END(handle_tlb_refill)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t