// SPDX-License-Identifier: GPL-2.0-only/* *  Copyright (C) 1993  Linus Torvalds *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 *  Numa awareness, Christoph Lameter, SGI, June 2005 *  Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 */ #include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/highmem.h>#include <linux/sched/signal.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/interrupt.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/set_memory.h>#include <linux/debugobjects.h>#include <linux/kallsyms.h>#include <linux/list.h>#include <linux/notifier.h>#include <linux/rbtree.h>#include <linux/xarray.h>#include <linux/io.h>#include <linux/rcupdate.h>#include <linux/pfn.h>#include <linux/kmemleak.h>#include <linux/atomic.h>#include <linux/compiler.h>#include <linux/memcontrol.h>#include <linux/llist.h>#include <linux/bitops.h>#include <linux/rbtree_augmented.h>#include <linux/overflow.h>#include <linux/pgtable.h>#include <linux/uaccess.h>#include <linux/hugetlb.h>#include <linux/sched/mm.h>#include <asm/tlbflush.h>#include <asm/shmparam.h> #include "internal.h"#include "pgalloc-track.h" #ifdef CONFIG_HAVE_ARCH_HUGE_VMAPstatic unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; static int __init set_nohugeiomap(char *str){	ioremap_max_page_shift = PAGE_SHIFT;	return 0;}early_param("nohugeiomap", set_nohugeiomap);#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */static const unsigned int ioremap_max_page_shift = PAGE_SHIFT;#endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */ #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOCstatic bool __ro_after_init vmap_allow_huge = true; static int __init set_nohugevmalloc(char *str){	vmap_allow_huge = false;	return 0;}early_param("nohugevmalloc", set_nohugevmalloc);#else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */static const bool vmap_allow_huge = false;#endif	/* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ bool is_vmalloc_addr(const void *x){	unsigned long addr = (unsigned long)kasan_reset_tag(x); 	return addr >= VMALLOC_START && addr < VMALLOC_END;}EXPORT_SYMBOL(is_vmalloc_addr); struct vfree_deferred {	struct llist_head list;	struct work_struct wq;};static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); static void __vunmap(const void *, int); static void free_work(struct work_struct *w){	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);	struct llist_node *t, *llnode; 	llist_for_each_safe(llnode, t, llist_del_all(&p->list))		__vunmap((void *)llnode, 1);} /*** Page table manipulation functions ***/static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pte_t *pte;	u64 pfn;	unsigned long size = PAGE_SIZE; 	pfn = phys_addr >> PAGE_SHIFT;	pte = pte_alloc_kernel_track(pmd, addr, mask);	if (!pte)		return -ENOMEM;	do {		BUG_ON(!pte_none(*pte)); #ifdef CONFIG_HUGETLB_PAGE		size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift);		if (size != PAGE_SIZE) {			pte_t entry = pfn_pte(pfn, prot); 			entry = arch_make_huge_pte(entry, ilog2(size), 0);			set_huge_pte_at(&init_mm, addr, pte, entry);			pfn += PFN_DOWN(size);			continue;		}#endif		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));		pfn++;	} while (pte += PFN_DOWN(size), addr += size, addr != end);	*mask |= PGTBL_PTE_MODIFIED;	return 0;} static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < PMD_SHIFT)		return 0; 	if (!arch_vmap_pmd_supported(prot))		return 0; 	if ((end - addr) != PMD_SIZE)		return 0; 	if (!IS_ALIGNED(addr, PMD_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, PMD_SIZE))		return 0; 	if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))		return 0; 	return pmd_set_huge(pmd, phys_addr, prot);} static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next; 	pmd = pmd_alloc_track(&init_mm, pud, addr, mask);	if (!pmd)		return -ENOMEM;	do {		next = pmd_addr_end(addr, end); 		if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_PMD_MODIFIED;			continue;		} 		if (vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask))			return -ENOMEM;	} while (pmd++, phys_addr += (next - addr), addr = next, addr != end);	return 0;} static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < PUD_SHIFT)		return 0; 	if (!arch_vmap_pud_supported(prot))		return 0; 	if ((end - addr) != PUD_SIZE)		return 0; 	if (!IS_ALIGNED(addr, PUD_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, PUD_SIZE))		return 0; 	if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))		return 0; 	return pud_set_huge(pud, phys_addr, prot);} static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next; 	pud = pud_alloc_track(&init_mm, p4d, addr, mask);	if (!pud)		return -ENOMEM;	do {		next = pud_addr_end(addr, end); 		if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_PUD_MODIFIED;			continue;		} 		if (vmap_pmd_range(pud, addr, next, phys_addr, prot,					max_page_shift, mask))			return -ENOMEM;	} while (pud++, phys_addr += (next - addr), addr = next, addr != end);	return 0;} static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < P4D_SHIFT)		return 0; 	if (!arch_vmap_p4d_supported(prot))		return 0; 	if ((end - addr) != P4D_SIZE)		return 0; 	if (!IS_ALIGNED(addr, P4D_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, P4D_SIZE))		return 0; 	if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))		return 0; 	return p4d_set_huge(p4d, phys_addr, prot);} static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next; 	p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);	if (!p4d)		return -ENOMEM;	do {		next = p4d_addr_end(addr, end); 		if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_P4D_MODIFIED;			continue;		} 		if (vmap_pud_range(p4d, addr, next, phys_addr, prot,					max_page_shift, mask))			return -ENOMEM;	} while (p4d++, phys_addr += (next - addr), addr = next, addr != end);	return 0;} static int vmap_range_noflush(unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	pgd_t *pgd;	unsigned long start;	unsigned long next;	int err;	pgtbl_mod_mask mask = 0; 	might_sleep();	BUG_ON(addr >= end); 	start = addr;	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		err = vmap_p4d_range(pgd, addr, next, phys_addr, prot,					max_page_shift, &mask);		if (err)			break;	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)		arch_sync_kernel_mappings(start, end); 	return err;} int ioremap_page_range(unsigned long addr, unsigned long end,		phys_addr_t phys_addr, pgprot_t prot){	int err; 	err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),				 ioremap_max_page_shift);	flush_cache_vmap(addr, end);	return err;} static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pte_t *pte; 	pte = pte_offset_kernel(pmd, addr);	do {		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);		WARN_ON(!pte_none(ptent) && !pte_present(ptent));	} while (pte++, addr += PAGE_SIZE, addr != end);	*mask |= PGTBL_PTE_MODIFIED;} static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next;	int cleared; 	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end); 		cleared = pmd_clear_huge(pmd);		if (cleared || pmd_bad(*pmd))			*mask |= PGTBL_PMD_MODIFIED; 		if (cleared)			continue;		if (pmd_none_or_clear_bad(pmd))			continue;		vunmap_pte_range(pmd, addr, next, mask); 		cond_resched();	} while (pmd++, addr = next, addr != end);} static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next;	int cleared; 	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end); 		cleared = pud_clear_huge(pud);		if (cleared || pud_bad(*pud))			*mask |= PGTBL_PUD_MODIFIED; 		if (cleared)			continue;		if (pud_none_or_clear_bad(pud))			continue;		vunmap_pmd_range(pud, addr, next, mask);	} while (pud++, addr = next, addr != end);} static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next;	int cleared; 	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end); 		cleared = p4d_clear_huge(p4d);		if (cleared || p4d_bad(*p4d))			*mask |= PGTBL_P4D_MODIFIED; 		if (cleared)			continue;		if (p4d_none_or_clear_bad(p4d))			continue;		vunmap_pud_range(p4d, addr, next, mask);	} while (p4d++, addr = next, addr != end);} /* * vunmap_range_noflush is similar to vunmap_range, but does not * flush caches or TLBs. * * The caller is responsible for calling flush_cache_vmap() before calling * this function, and flush_tlb_kernel_range after it has returned * successfully (and before the addresses are expected to cause a page fault * or be re-mapped for something else, if TLB flushes are being delayed or * coalesced). * * This is an internal function only. Do not use outside mm/. */void vunmap_range_noflush(unsigned long start, unsigned long end){	unsigned long next;	pgd_t *pgd;	unsigned long addr = start;	pgtbl_mod_mask mask = 0; 	BUG_ON(addr >= end);	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_bad(*pgd))			mask |= PGTBL_PGD_MODIFIED;		if (pgd_none_or_clear_bad(pgd))			continue;		vunmap_p4d_range(pgd, addr, next, &mask);	} while (pgd++, addr = next, addr != end); 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)		arch_sync_kernel_mappings(start, end);} /** * vunmap_range - unmap kernel virtual addresses * @addr: start of the VM area to unmap * @end: end of the VM area to unmap (non-inclusive) * * Clears any present PTEs in the virtual address range, flushes TLBs and * caches. Any subsequent access to the address before it has been re-mapped * is a kernel bug. */void vunmap_range(unsigned long addr, unsigned long end){	flush_cache_vunmap(addr, end);	vunmap_range_noflush(addr, end);	flush_tlb_kernel_range(addr, end);} static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	pte_t *pte; 	/*	 * nr is a running index into the array which helps higher level	 * callers keep track of where we're up to.	 */ 	pte = pte_alloc_kernel_track(pmd, addr, mask);	if (!pte)		return -ENOMEM;	do {		struct page *page = pages[*nr]; 		if (WARN_ON(!pte_none(*pte)))			return -EBUSY;		if (WARN_ON(!page))			return -ENOMEM;		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));		(*nr)++;	} while (pte++, addr += PAGE_SIZE, addr != end);	*mask |= PGTBL_PTE_MODIFIED;	return 0;} static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next; 	pmd = pmd_alloc_track(&init_mm, pud, addr, mask);	if (!pmd)		return -ENOMEM;	do {		next = pmd_addr_end(addr, end);		if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask))

Lines 1–500 of 4147. Load more