/* SPDX-License-Identifier: GPL-2.0-only *//* * Copyright (C) 2012 ARM Ltd. */#ifndef __ASM_PGTABLE_H#define __ASM_PGTABLE_H #include <asm/bug.h>#include <asm/proc-fns.h> #include <asm/memory.h>#include <asm/mte.h>#include <asm/pgtable-hwdef.h>#include <asm/pgtable-prot.h>#include <asm/tlbflush.h> /* * VMALLOC range. * * VMALLOC_START: beginning of the kernel vmalloc space * VMALLOC_END: extends to the available space below vmemmap */#define VMALLOC_START (MODULES_END)#if VA_BITS == VA_BITS_MIN#define VMALLOC_END (VMEMMAP_START - SZ_8M)#else#define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT)#define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)#endif #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) #ifndef __ASSEMBLER__ #include <asm/cmpxchg.h>#include <asm/fixmap.h>#include <asm/por.h>#include <linux/mmdebug.h>#include <linux/mm_types.h>#include <linux/sched.h>#include <linux/page_table_check.h> static inline void emit_pte_barriers(void){ /* * These barriers are emitted under certain conditions after a pte entry * was modified (see e.g. __set_pte_complete()). The dsb makes the store * visible to the table walker. The isb ensures that any previous * speculative "invalid translation" marker that is in the CPU's * pipeline gets cleared, so that any access to that address after * setting the pte to valid won't cause a spurious fault. If the thread * gets preempted after storing to the pgtable but before emitting these * barriers, __switch_to() emits a dsb which ensure the walker gets to * see the store. There is no guarantee of an isb being issued though. * This is safe because it will still get issued (albeit on a * potentially different CPU) when the thread starts running again, * before any access to the address. */ dsb(ishst); isb();} static inline void queue_pte_barriers(void){ if (is_lazy_mmu_mode_active()) { /* Avoid the atomic op if already set. */ if (!test_thread_flag(TIF_LAZY_MMU_PENDING)) set_thread_flag(TIF_LAZY_MMU_PENDING); } else { emit_pte_barriers(); }} static inline void arch_enter_lazy_mmu_mode(void) {} static inline void arch_flush_lazy_mmu_mode(void){ if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING)) emit_pte_barriers();} static inline void arch_leave_lazy_mmu_mode(void){ arch_flush_lazy_mmu_mode();} #ifdef CONFIG_TRANSPARENT_HUGEPAGE#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE /* Set stride and tlb_level in flush_*_tlb_range */#define flush_pmd_tlb_range(vma, addr, end) \ __flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)#define flush_pud_tlb_range(vma, addr, end) \ __flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ /* * We use local TLB invalidation instruction when reusing page in * write protection fault handler to avoid TLBI broadcast in the hot * path. This will cause spurious page faults if stale read-only TLB * entries exist. */#define flush_tlb_fix_spurious_fault(vma, address, ptep) \ __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) #ifdef CONFIG_ARM64_PA_BITS_52static inline phys_addr_t __pte_to_phys(pte_t pte){ pte_val(pte) &= ~PTE_MAYBE_SHARED; return (pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);}static inline pteval_t __phys_to_pte_val(phys_addr_t phys){ return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;}#elsestatic inline phys_addr_t __pte_to_phys(pte_t pte){ return pte_val(pte) & PTE_ADDR_LOW;} static inline pteval_t __phys_to_pte_val(phys_addr_t phys){ return phys;}#endif #define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT)#define pfn_pte(pfn,prot) \ __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_none(pte) (!pte_val(pte))#define pte_page(pte) (pfn_to_page(pte_pfn(pte))) #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))#define pte_present_invalid(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * The following only work if pte_present(). Undefined behaviour otherwise. */static __always_inline bool pte_present(pte_t pte){ return pte_valid(pte) || pte_present_invalid(pte);}#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))#define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY))#define pte_user(pte) (!!(pte_val(pte) & PTE_USER))#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ PTE_ATTRINDX(MT_NORMAL_TAGGED)) #define pte_cont_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ (__boundary - 1 < (end) - 1) ? __boundary : (end); \}) #define pmd_cont_addr_end(addr, end) \({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \ (__boundary - 1 < (end) - 1) ? __boundary : (end); \}) #define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte))#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. */#define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))/* * Returns true if the pte is valid and has the contiguous bit set. */#define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte))/* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been * remapped as PROT_NONE but are yet to be flushed from the TLB. * Note that we can't make any assumptions based on the state of the access * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the * TLB. */#define pte_accessible(mm, pte) \ (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) static inline bool por_el0_allows_pkey(u8 pkey, bool write, bool execute){ u64 por; if (!system_supports_poe()) return true; por = read_sysreg_s(SYS_POR_EL0); if (write) return por_elx_allows_write(por, pkey); if (execute) return por_elx_allows_exec(por, pkey); return por_elx_allows_read(por, pkey);} /* * p??_access_permitted() is true for valid user mappings (PTE_USER * bit set, subject to the write permission check). For execute-only * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits * not set) must return false. PROT_NONE mappings do not have the * PTE_VALID bit set. */#define pte_access_permitted_no_overlay(pte, write) \ (((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))#define pte_access_permitted(pte, write) \ (pte_access_permitted_no_overlay(pte, write) && \ por_el0_allows_pkey(FIELD_GET(PTE_PO_IDX_MASK, pte_val(pte)), write, false))#define pmd_access_permitted(pmd, write) \ (pte_access_permitted(pmd_pte(pmd), (write)))#define pud_access_permitted(pud, write) \ (pte_access_permitted(pud_pte(pud), (write))) static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot){ pte_val(pte) &= ~pgprot_val(prot); return pte;} static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot){ pte_val(pte) |= pgprot_val(prot); return pte;} static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot){ pmd_val(pmd) &= ~pgprot_val(prot); return pmd;} static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot){ pmd_val(pmd) |= pgprot_val(prot); return pmd;} static inline pte_t pte_mkwrite_novma(pte_t pte){ pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); if (pte_sw_dirty(pte)) pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte;} static inline pte_t pte_mkclean(pte_t pte){ pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte;} static inline pte_t pte_mkdirty(pte_t pte){ pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); if (pte_write(pte)) pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte;} static inline pte_t pte_wrprotect(pte_t pte){ /* * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY * clear), set the PTE_DIRTY bit. */ if (pte_hw_dirty(pte)) pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte;} static inline pte_t pte_mkold(pte_t pte){ return clear_pte_bit(pte, __pgprot(PTE_AF));} static inline pte_t pte_mkyoung(pte_t pte){ return set_pte_bit(pte, __pgprot(PTE_AF));} static inline pte_t pte_mkspecial(pte_t pte){ return set_pte_bit(pte, __pgprot(PTE_SPECIAL));} static inline pte_t pte_mkcont(pte_t pte){ return set_pte_bit(pte, __pgprot(PTE_CONT));} static inline pte_t pte_mknoncont(pte_t pte){ return clear_pte_bit(pte, __pgprot(PTE_CONT));} static inline pte_t pte_mkvalid_k(pte_t pte){ pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL)); return pte;} static inline pte_t pte_mkinvalid(pte_t pte){ pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); pte = clear_pte_bit(pte, __pgprot(PTE_VALID)); return pte;} static inline pmd_t pmd_mkcont(pmd_t pmd){ return __pmd(pmd_val(pmd) | PMD_SECT_CONT);} static inline pmd_t pmd_mknoncont(pmd_t pmd){ return __pmd(pmd_val(pmd) & ~PMD_SECT_CONT);} #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WPstatic inline int pte_uffd(pte_t pte){ return !!(pte_val(pte) & PTE_UFFD);} static inline pte_t pte_mkuffd(pte_t pte){ return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD)));} static inline pte_t pte_clear_uffd(pte_t pte){ return clear_pte_bit(pte, __pgprot(PTE_UFFD));}#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ static inline void __set_pte_nosync(pte_t *ptep, pte_t pte){ WRITE_ONCE(*ptep, pte);} static inline void __set_pte_complete(pte_t pte){ /* * Only if the new pte is valid and kernel, otherwise TLB maintenance * has the necessary barriers. */ if (pte_valid_not_user(pte)) queue_pte_barriers();} static inline void __set_pte(pte_t *ptep, pte_t pte){ __set_pte_nosync(ptep, pte); __set_pte_complete(pte);} static inline pte_t __ptep_get(pte_t *ptep){ return READ_ONCE(*ptep);} extern void __sync_icache_dcache(pte_t pteval);bool pgattr_change_is_safe(pteval_t old, pteval_t new); /* * PTE bits configuration in the presence of hardware Dirty Bit Management * (PTE_WRITE == PTE_DBM): * * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) * 0 0 | 1 0 0 * 0 1 | 1 1 0 * 1 0 | 1 0 1 * 1 1 | 0 1 x * * When hardware DBM is not present, the software PTE_DIRTY bit is updated via * the page fault mechanism. Checking the dirty status of a pte becomes: * * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) */ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, pte_t pte){ pte_t old_pte; if (!IS_ENABLED(CONFIG_DEBUG_VM)) return; old_pte = __ptep_get(ptep); if (!pte_valid(old_pte) || !pte_valid(pte)) return; if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1) return; /* * Check for potential race with hardware updates of the pte * (__ptep_set_access_flags safely changes valid ptes without going * through an invalid entry). */ VM_WARN_ONCE(!pte_young(pte), "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte)); VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte)); VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), "%s: unsafe attribute change: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte));} static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages){ if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) __sync_icache_dcache(pte); /* * If the PTE would provide user space access to the tags associated * with it then ensure that the MTE tags are synchronised. Although * pte_access_permitted_no_overlay() returns false for exec only * mappings, they don't expose tags (instruction fetches don't check * tags). */ if (system_supports_mte() && pte_access_permitted_no_overlay(pte, false) && !pte_special(pte) && pte_tagged(pte)) mte_sync_tags(pte, nr_pages);} /* * Select all bits except the pfn */#define pte_pgprot pte_pgprotstatic inline pgprot_t pte_pgprot(pte_t pte){ unsigned long pfn = pte_pfn(pte); return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));} #define pte_advance_pfn pte_advance_pfnstatic inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr){ return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));} /* * Hugetlb definitions. */#define HUGE_MAX_HSTATE 4#define HPAGE_SHIFT PMD_SHIFT#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)#define HPAGE_MASK (~(HPAGE_SIZE - 1))#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) static inline pte_t pgd_pte(pgd_t pgd){ return __pte(pgd_val(pgd));} static inline pte_t p4d_pte(p4d_t p4d){ return __pte(p4d_val(p4d));} static inline pte_t pud_pte(pud_t pud){ return __pte(pud_val(pud));} static inline pud_t pte_pud(pte_t pte){ return __pud(pte_val(pte));Lines 1–500 of 1983. Load more
/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2012 ARM Ltd. */ #ifndef __ASM_PGTABLE_H #define __ASM_PGTABLE_H #include <asm/bug.h> #include <asm/proc-fns.h> #include <asm/memory.h> #include <asm/mte.h> #include <asm/pgtable-hwdef.h> #include <asm/pgtable-prot.h> #include <asm/tlbflush.h> /* * VMALLOC range. * * VMALLOC_START: beginning of the kernel vmalloc space * VMALLOC_END: extends to the available space below vmemmap */ #define VMALLOC_START (MODULES_END) #if VA_BITS == VA_BITS_MIN #define VMALLOC_END (VMEMMAP_START - SZ_8M) #else #define VMEMMAP_UNUSED_NPAGES ((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT) #define VMALLOC_END (VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M) #endif #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) #ifndef __ASSEMBLER__ #include <asm/cmpxchg.h> #include <asm/fixmap.h> #include <asm/por.h> #include <linux/mmdebug.h> #include <linux/mm_types.h> #include <linux/sched.h> #include <linux/page_table_check.h> static inline void emit_pte_barriers(void) { /* * These barriers are emitted under certain conditions after a pte entry * was modified (see e.g. __set_pte_complete()). The dsb makes the store * visible to the table walker. The isb ensures that any previous * speculative "invalid translation" marker that is in the CPU's * pipeline gets cleared, so that any access to that address after * setting the pte to valid won't cause a spurious fault. If the thread * gets preempted after storing to the pgtable but before emitting these * barriers, __switch_to() emits a dsb which ensure the walker gets to * see the store. There is no guarantee of an isb being issued though. * This is safe because it will still get issued (albeit on a * potentially different CPU) when the thread starts running again, * before any access to the address. */ dsb(ishst); isb(); } static inline void queue_pte_barriers(void) { if (is_lazy_mmu_mode_active()) { /* Avoid the atomic op if already set. */ if (!test_thread_flag(TIF_LAZY_MMU_PENDING)) set_thread_flag(TIF_LAZY_MMU_PENDING); } else { emit_pte_barriers(); } } static inline void arch_enter_lazy_mmu_mode(void) {} static inline void arch_flush_lazy_mmu_mode(void) { if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING)) emit_pte_barriers(); } static inline void arch_leave_lazy_mmu_mode(void) { arch_flush_lazy_mmu_mode(); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE /* Set stride and tlb_level in flush_*_tlb_range */ #define flush_pmd_tlb_range(vma, addr, end) \ __flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE) #define flush_pud_tlb_range(vma, addr, end) \ __flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ /* * We use local TLB invalidation instruction when reusing page in * write protection fault handler to avoid TLBI broadcast in the hot * path. This will cause spurious page faults if stale read-only TLB * entries exist. */ #define flush_tlb_fix_spurious_fault(vma, address, ptep) \ __flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY) #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \ __flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \ TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE) #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e)) #ifdef CONFIG_ARM64_PA_BITS_52 static inline phys_addr_t __pte_to_phys(pte_t pte) { pte_val(pte) &= ~PTE_MAYBE_SHARED; return (pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT); } static inline pteval_t __phys_to_pte_val(phys_addr_t phys) { return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK; } #else static inline phys_addr_t __pte_to_phys(pte_t pte) { return pte_val(pte) & PTE_ADDR_LOW; } static inline pteval_t __phys_to_pte_val(phys_addr_t phys) { return phys; } #endif #define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT) #define pfn_pte(pfn,prot) \ __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_none(pte) (!pte_val(pte)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) #define pte_present_invalid(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * The following only work if pte_present(). Undefined behaviour otherwise. */ static __always_inline bool pte_present(pte_t pte) { return pte_valid(pte) || pte_present_invalid(pte); } #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) #define pte_rdonly(pte) (!!(pte_val(pte) & PTE_RDONLY)) #define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) #define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN)) #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) #define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \ PTE_ATTRINDX(MT_NORMAL_TAGGED)) #define pte_cont_addr_end(addr, end) \ ({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ }) #define pmd_cont_addr_end(addr, end) \ ({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \ (__boundary - 1 < (end) - 1) ? __boundary : (end); \ }) #define pte_hw_dirty(pte) (pte_write(pte) && !pte_rdonly(pte)) #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. */ #define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN)) /* * Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont(pte) (pte_valid(pte) && pte_cont(pte)) /* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been * remapped as PROT_NONE but are yet to be flushed from the TLB. * Note that we can't make any assumptions based on the state of the access * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the * TLB. */ #define pte_accessible(mm, pte) \ (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte)) static inline bool por_el0_allows_pkey(u8 pkey, bool write, bool execute) { u64 por; if (!system_supports_poe()) return true; por = read_sysreg_s(SYS_POR_EL0); if (write) return por_elx_allows_write(por, pkey); if (execute) return por_elx_allows_exec(por, pkey); return por_elx_allows_read(por, pkey); } /* * p??_access_permitted() is true for valid user mappings (PTE_USER * bit set, subject to the write permission check). For execute-only * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits * not set) must return false. PROT_NONE mappings do not have the * PTE_VALID bit set. */ #define pte_access_permitted_no_overlay(pte, write) \ (((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte))) #define pte_access_permitted(pte, write) \ (pte_access_permitted_no_overlay(pte, write) && \ por_el0_allows_pkey(FIELD_GET(PTE_PO_IDX_MASK, pte_val(pte)), write, false)) #define pmd_access_permitted(pmd, write) \ (pte_access_permitted(pmd_pte(pmd), (write))) #define pud_access_permitted(pud, write) \ (pte_access_permitted(pud_pte(pud), (write))) static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) { pte_val(pte) &= ~pgprot_val(prot); return pte; } static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) { pte_val(pte) |= pgprot_val(prot); return pte; } static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot) { pmd_val(pmd) &= ~pgprot_val(prot); return pmd; } static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) { pmd_val(pmd) |= pgprot_val(prot); return pmd; } static inline pte_t pte_mkwrite_novma(pte_t pte) { pte = set_pte_bit(pte, __pgprot(PTE_WRITE)); if (pte_sw_dirty(pte)) pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte; } static inline pte_t pte_mkclean(pte_t pte) { pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY)); pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte; } static inline pte_t pte_mkdirty(pte_t pte) { pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); if (pte_write(pte)) pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte; } static inline pte_t pte_wrprotect(pte_t pte) { /* * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY * clear), set the PTE_DIRTY bit. */ if (pte_hw_dirty(pte)) pte = set_pte_bit(pte, __pgprot(PTE_DIRTY)); pte = clear_pte_bit(pte, __pgprot(PTE_WRITE)); pte = set_pte_bit(pte, __pgprot(PTE_RDONLY)); return pte; } static inline pte_t pte_mkold(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_AF)); } static inline pte_t pte_mkyoung(pte_t pte) { return set_pte_bit(pte, __pgprot(PTE_AF)); } static inline pte_t pte_mkspecial(pte_t pte) { return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); } static inline pte_t pte_mkcont(pte_t pte) { return set_pte_bit(pte, __pgprot(PTE_CONT)); } static inline pte_t pte_mknoncont(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_CONT)); } static inline pte_t pte_mkvalid_k(pte_t pte) { pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL)); return pte; } static inline pte_t pte_mkinvalid(pte_t pte) { pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); pte = clear_pte_bit(pte, __pgprot(PTE_VALID)); return pte; } static inline pmd_t pmd_mkcont(pmd_t pmd) { return __pmd(pmd_val(pmd) | PMD_SECT_CONT); } static inline pmd_t pmd_mknoncont(pmd_t pmd) { return __pmd(pmd_val(pmd) & ~PMD_SECT_CONT); } #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP static inline int pte_uffd(pte_t pte) { return !!(pte_val(pte) & PTE_UFFD); } static inline pte_t pte_mkuffd(pte_t pte) { return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD))); } static inline pte_t pte_clear_uffd(pte_t pte) { return clear_pte_bit(pte, __pgprot(PTE_UFFD)); } #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */ static inline void __set_pte_nosync(pte_t *ptep, pte_t pte) { WRITE_ONCE(*ptep, pte); } static inline void __set_pte_complete(pte_t pte) { /* * Only if the new pte is valid and kernel, otherwise TLB maintenance * has the necessary barriers. */ if (pte_valid_not_user(pte)) queue_pte_barriers(); } static inline void __set_pte(pte_t *ptep, pte_t pte) { __set_pte_nosync(ptep, pte); __set_pte_complete(pte); } static inline pte_t __ptep_get(pte_t *ptep) { return READ_ONCE(*ptep); } extern void __sync_icache_dcache(pte_t pteval); bool pgattr_change_is_safe(pteval_t old, pteval_t new); /* * PTE bits configuration in the presence of hardware Dirty Bit Management * (PTE_WRITE == PTE_DBM): * * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) * 0 0 | 1 0 0 * 0 1 | 1 1 0 * 1 0 | 1 0 1 * 1 1 | 0 1 x * * When hardware DBM is not present, the software PTE_DIRTY bit is updated via * the page fault mechanism. Checking the dirty status of a pte becomes: * * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) */ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, pte_t pte) { pte_t old_pte; if (!IS_ENABLED(CONFIG_DEBUG_VM)) return; old_pte = __ptep_get(ptep); if (!pte_valid(old_pte) || !pte_valid(pte)) return; if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1) return; /* * Check for potential race with hardware updates of the pte * (__ptep_set_access_flags safely changes valid ptes without going * through an invalid entry). */ VM_WARN_ONCE(!pte_young(pte), "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte)); VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte), "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte)); VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)), "%s: unsafe attribute change: 0x%016llx -> 0x%016llx", __func__, pte_val(old_pte), pte_val(pte)); } static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages) { if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) __sync_icache_dcache(pte); /* * If the PTE would provide user space access to the tags associated * with it then ensure that the MTE tags are synchronised. Although * pte_access_permitted_no_overlay() returns false for exec only * mappings, they don't expose tags (instruction fetches don't check * tags). */ if (system_supports_mte() && pte_access_permitted_no_overlay(pte, false) && !pte_special(pte) && pte_tagged(pte)) mte_sync_tags(pte, nr_pages); } /* * Select all bits except the pfn */ #define pte_pgprot pte_pgprot static inline pgprot_t pte_pgprot(pte_t pte) { unsigned long pfn = pte_pfn(pte); return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); } #define pte_advance_pfn pte_advance_pfn static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) { return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } /* * Hugetlb definitions. */ #define HUGE_MAX_HSTATE 4 #define HPAGE_SHIFT PMD_SHIFT #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) static inline pte_t pgd_pte(pgd_t pgd) { return __pte(pgd_val(pgd)); } static inline pte_t p4d_pte(p4d_t p4d) { return __pte(p4d_val(p4d)); } static inline pte_t pud_pte(pud_t pud) { return __pte(pud_val(pud)); } static inline pud_t pte_pud(pte_t pte) { return __pud(pte_val(pte));