// SPDX-License-Identifier: GPL-2.0-only/* *  Copyright (C) 1993  Linus Torvalds *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 *  Numa awareness, Christoph Lameter, SGI, June 2005 *  Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 */ #include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/highmem.h>#include <linux/sched/signal.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/interrupt.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/set_memory.h>#include <linux/debugobjects.h>#include <linux/kallsyms.h>#include <linux/list.h>#include <linux/notifier.h>#include <linux/rbtree.h>#include <linux/xarray.h>#include <linux/io.h>#include <linux/rcupdate.h>#include <linux/pfn.h>#include <linux/kmemleak.h>#include <linux/atomic.h>#include <linux/compiler.h>#include <linux/memcontrol.h>#include <linux/llist.h>#include <linux/uio.h>#include <linux/bitops.h>#include <linux/rbtree_augmented.h>#include <linux/overflow.h>#include <linux/pgtable.h>#include <linux/hugetlb.h>#include <linux/sched/mm.h>#include <asm/tlbflush.h>#include <asm/shmparam.h>#include <linux/page_owner.h>#include <linux/cleanup.h> #define CREATE_TRACE_POINTS#include <trace/events/vmalloc.h> #include "internal.h"#include "pgalloc-track.h" #ifdef CONFIG_HAVE_ARCH_HUGE_VMAPstatic unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; static int __init set_nohugeiomap(char *str){	ioremap_max_page_shift = PAGE_SHIFT;	return 0;}early_param("nohugeiomap", set_nohugeiomap);#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */static const unsigned int ioremap_max_page_shift = PAGE_SHIFT;#endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */ #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOCstatic bool __ro_after_init vmap_allow_huge = true; static int __init set_nohugevmalloc(char *str){	vmap_allow_huge = false;	return 0;}early_param("nohugevmalloc", set_nohugevmalloc);#else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */static const bool vmap_allow_huge = false;#endif	/* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ bool is_vmalloc_addr(const void *x){	unsigned long addr = (unsigned long)kasan_reset_tag(x); 	return addr >= VMALLOC_START && addr < VMALLOC_END;}EXPORT_SYMBOL(is_vmalloc_addr); struct vfree_deferred {	struct llist_head list;	struct work_struct wq;};static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); /*** Page table manipulation functions ***/static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pte_t *pte;	u64 pfn;	struct page *page;	unsigned long size = PAGE_SIZE; 	if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr)))		return -EINVAL; 	pfn = phys_addr >> PAGE_SHIFT;	pte = pte_alloc_kernel_track(pmd, addr, mask);	if (!pte)		return -ENOMEM; 	lazy_mmu_mode_enable(); 	do {		if (unlikely(!pte_none(ptep_get(pte)))) {			if (pfn_valid(pfn)) {				page = pfn_to_page(pfn);				dump_page(page, "remapping already mapped page");			}			BUG();		} #ifdef CONFIG_HUGETLB_PAGE		size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift);		if (size != PAGE_SIZE) {			pte_t entry = pfn_pte(pfn, prot); 			entry = arch_make_huge_pte(entry, ilog2(size), 0);			set_huge_pte_at(&init_mm, addr, pte, entry, size);			pfn += PFN_DOWN(size);			continue;		}#endif		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));		pfn++;	} while (pte += PFN_DOWN(size), addr += size, addr != end); 	lazy_mmu_mode_disable();	*mask |= PGTBL_PTE_MODIFIED;	return 0;} static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < PMD_SHIFT)		return 0; 	if (!arch_vmap_pmd_supported(prot))		return 0; 	if ((end - addr) != PMD_SIZE)		return 0; 	if (!IS_ALIGNED(addr, PMD_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, PMD_SIZE))		return 0; 	if (!pmd_present(*pmd))		return pmd_set_huge(pmd, phys_addr, prot); 	/*	 * Acquire the mmap read lock to exclude ptdump, which walks	 * kernel page tables it does not own under the mmap write lock.	 *	 * Concurrent read lock holders are safe: each exclusively owns	 * the range it operates on and cannot reach this page table.	 */	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {		if (!pmd_free_pte_page(pmd, addr))			return 0;		return pmd_set_huge(pmd, phys_addr, prot);	}} static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next;	int err = 0; 	pmd = pmd_alloc_track(&init_mm, pud, addr, mask);	if (!pmd)		return -ENOMEM;	do {		next = pmd_addr_end(addr, end); 		if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_PMD_MODIFIED;			continue;		} 		err = vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask);		if (err)			break;	} while (pmd++, phys_addr += (next - addr), addr = next, addr != end);	return err;} static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < PUD_SHIFT)		return 0; 	if (!arch_vmap_pud_supported(prot))		return 0; 	if ((end - addr) != PUD_SIZE)		return 0; 	if (!IS_ALIGNED(addr, PUD_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, PUD_SIZE))		return 0; 	if (!pud_present(*pud))		return pud_set_huge(pud, phys_addr, prot); 	/* See comment in vmap_try_huge_pmd(). */	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {		if (!pud_free_pmd_page(pud, addr))			return 0;		return pud_set_huge(pud, phys_addr, prot);	}} static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next;	int err = 0; 	pud = pud_alloc_track(&init_mm, p4d, addr, mask);	if (!pud)		return -ENOMEM;	do {		next = pud_addr_end(addr, end); 		if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_PUD_MODIFIED;			continue;		} 		err = vmap_pmd_range(pud, addr, next, phys_addr, prot, max_page_shift, mask);		if (err)			break;	} while (pud++, phys_addr += (next - addr), addr = next, addr != end);	return err;} static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	if (max_page_shift < P4D_SHIFT)		return 0; 	if (!arch_vmap_p4d_supported(prot))		return 0; 	if ((end - addr) != P4D_SIZE)		return 0; 	if (!IS_ALIGNED(addr, P4D_SIZE))		return 0; 	if (!IS_ALIGNED(phys_addr, P4D_SIZE))		return 0; 	if (!p4d_present(*p4d))		return p4d_set_huge(p4d, phys_addr, prot); 	/* See comment in vmap_try_huge_pmd(). */	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {		if (!p4d_free_pud_page(p4d, addr))			return 0;		return p4d_set_huge(p4d, phys_addr, prot);	}} static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift, pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next;	int err = 0; 	p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);	if (!p4d)		return -ENOMEM;	do {		next = p4d_addr_end(addr, end); 		if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot,					max_page_shift)) {			*mask |= PGTBL_P4D_MODIFIED;			continue;		} 		err = vmap_pud_range(p4d, addr, next, phys_addr, prot, max_page_shift, mask);		if (err)			break;	} while (p4d++, phys_addr += (next - addr), addr = next, addr != end);	return err;} static int vmap_range_noflush(unsigned long addr, unsigned long end,			phys_addr_t phys_addr, pgprot_t prot,			unsigned int max_page_shift){	pgd_t *pgd;	unsigned long start;	unsigned long next;	int err;	pgtbl_mod_mask mask = 0; 	/*	 * Might allocate pagetables (for most archs a more precise annotation	 * would be might_alloc(GFP_PGTABLE_KERNEL)). Also might shootdown TLB	 * (requires IRQs enabled on x86).	 */	might_sleep();	BUG_ON(addr >= end); 	start = addr;	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		err = vmap_p4d_range(pgd, addr, next, phys_addr, prot,					max_page_shift, &mask);		if (err)			break;	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)		arch_sync_kernel_mappings(start, end); 	return err;} int vmap_page_range(unsigned long addr, unsigned long end,		    phys_addr_t phys_addr, pgprot_t prot){	int err; 	err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),				 ioremap_max_page_shift);	flush_cache_vmap(addr, end);	if (!err)		err = kmsan_ioremap_page_range(addr, end, phys_addr, prot,					       ioremap_max_page_shift);	return err;} int ioremap_page_range(unsigned long addr, unsigned long end,		phys_addr_t phys_addr, pgprot_t prot){	struct vm_struct *area; 	area = find_vm_area((void *)addr);	if (!area || !(area->flags & VM_IOREMAP)) {		WARN_ONCE(1, "vm_area at addr %lx is not marked as VM_IOREMAP\n", addr);		return -EINVAL;	}	if (addr != (unsigned long)area->addr ||	    (void *)end != area->addr + get_vm_area_size(area)) {		WARN_ONCE(1, "ioremap request [%lx,%lx) doesn't match vm_area [%lx, %lx)\n",			  addr, end, (long)area->addr,			  (long)area->addr + get_vm_area_size(area));		return -ERANGE;	}	return vmap_page_range(addr, end, phys_addr, prot);} static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pte_t *pte;	pte_t ptent;	unsigned long size = PAGE_SIZE; 	pte = pte_offset_kernel(pmd, addr);	lazy_mmu_mode_enable(); 	do {#ifdef CONFIG_HUGETLB_PAGE		size = arch_vmap_pte_range_unmap_size(addr, pte);		if (size != PAGE_SIZE) {			if (WARN_ON(!IS_ALIGNED(addr, size))) {				addr = ALIGN_DOWN(addr, size);				pte = PTR_ALIGN_DOWN(pte, sizeof(*pte) * (size >> PAGE_SHIFT));			}			ptent = huge_ptep_get_and_clear(&init_mm, addr, pte, size);			if (WARN_ON(end - addr < size))				size = end - addr;		} else#endif			ptent = ptep_get_and_clear(&init_mm, addr, pte);		WARN_ON(!pte_none(ptent) && !pte_present(ptent));	} while (pte += (size >> PAGE_SHIFT), addr += size, addr != end); 	lazy_mmu_mode_disable();	*mask |= PGTBL_PTE_MODIFIED;} static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next;	int cleared; 	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end); 		cleared = pmd_clear_huge(pmd);		if (cleared || pmd_bad(*pmd))			*mask |= PGTBL_PMD_MODIFIED; 		if (cleared) {			WARN_ON(next - addr < PMD_SIZE);			continue;		}		if (pmd_none_or_clear_bad(pmd))			continue;		vunmap_pte_range(pmd, addr, next, mask); 		cond_resched();	} while (pmd++, addr = next, addr != end);} static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next;	int cleared; 	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end); 		cleared = pud_clear_huge(pud);		if (cleared || pud_bad(*pud))			*mask |= PGTBL_PUD_MODIFIED; 		if (cleared) {			WARN_ON(next - addr < PUD_SIZE);			continue;		}		if (pud_none_or_clear_bad(pud))			continue;		vunmap_pmd_range(pud, addr, next, mask);	} while (pud++, addr = next, addr != end);} static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next; 	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end); 		p4d_clear_huge(p4d);		if (p4d_bad(*p4d))			*mask |= PGTBL_P4D_MODIFIED; 		if (p4d_none_or_clear_bad(p4d))			continue;		vunmap_pud_range(p4d, addr, next, mask);	} while (p4d++, addr = next, addr != end);} /* * vunmap_range_noflush is similar to vunmap_range, but does not * flush caches or TLBs. * * The caller is responsible for calling flush_cache_vmap() before calling * this function, and flush_tlb_kernel_range after it has returned * successfully (and before the addresses are expected to cause a page fault * or be re-mapped for something else, if TLB flushes are being delayed or * coalesced). *

Lines 1–500 of 5592. Load more