// SPDX-License-Identifier: GPL-2.0-only/* *  Copyright (C) 1993  Linus Torvalds *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 *  Numa awareness, Christoph Lameter, SGI, June 2005 *  Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 */ #include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/highmem.h>#include <linux/sched/signal.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/interrupt.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/set_memory.h>#include <linux/debugobjects.h>#include <linux/kallsyms.h>#include <linux/list.h>#include <linux/notifier.h>#include <linux/rbtree.h>#include <linux/xarray.h>#include <linux/rcupdate.h>#include <linux/pfn.h>#include <linux/kmemleak.h>#include <linux/atomic.h>#include <linux/compiler.h>#include <linux/llist.h>#include <linux/bitops.h>#include <linux/rbtree_augmented.h>#include <linux/overflow.h> #include <linux/uaccess.h>#include <asm/tlbflush.h>#include <asm/shmparam.h> #include "internal.h"#include "pgalloc-track.h" bool is_vmalloc_addr(const void *x){	unsigned long addr = (unsigned long)x; 	return addr >= VMALLOC_START && addr < VMALLOC_END;}EXPORT_SYMBOL(is_vmalloc_addr); struct vfree_deferred {	struct llist_head list;	struct work_struct wq;};static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); static void __vunmap(const void *, int); static void free_work(struct work_struct *w){	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);	struct llist_node *t, *llnode; 	llist_for_each_safe(llnode, t, llist_del_all(&p->list))		__vunmap((void *)llnode, 1);} /*** Page table manipulation functions ***/ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pte_t *pte; 	pte = pte_offset_kernel(pmd, addr);	do {		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);		WARN_ON(!pte_none(ptent) && !pte_present(ptent));	} while (pte++, addr += PAGE_SIZE, addr != end);	*mask |= PGTBL_PTE_MODIFIED;} static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next;	int cleared; 	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end); 		cleared = pmd_clear_huge(pmd);		if (cleared || pmd_bad(*pmd))			*mask |= PGTBL_PMD_MODIFIED; 		if (cleared)			continue;		if (pmd_none_or_clear_bad(pmd))			continue;		vunmap_pte_range(pmd, addr, next, mask); 		cond_resched();	} while (pmd++, addr = next, addr != end);} static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next;	int cleared; 	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end); 		cleared = pud_clear_huge(pud);		if (cleared || pud_bad(*pud))			*mask |= PGTBL_PUD_MODIFIED; 		if (cleared)			continue;		if (pud_none_or_clear_bad(pud))			continue;		vunmap_pmd_range(pud, addr, next, mask);	} while (pud++, addr = next, addr != end);} static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,			     pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next;	int cleared; 	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end); 		cleared = p4d_clear_huge(p4d);		if (cleared || p4d_bad(*p4d))			*mask |= PGTBL_P4D_MODIFIED; 		if (cleared)			continue;		if (p4d_none_or_clear_bad(p4d))			continue;		vunmap_pud_range(p4d, addr, next, mask);	} while (p4d++, addr = next, addr != end);} /** * unmap_kernel_range_noflush - unmap kernel VM area * @start: start of the VM area to unmap * @size: size of the VM area to unmap * * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size specify * should have been allocated using get_vm_area() and its friends. * * NOTE: * This function does NOT do any cache flushing.  The caller is responsible * for calling flush_cache_vunmap() on to-be-mapped areas before calling this * function and flush_tlb_kernel_range() after. */void unmap_kernel_range_noflush(unsigned long start, unsigned long size){	unsigned long end = start + size;	unsigned long next;	pgd_t *pgd;	unsigned long addr = start;	pgtbl_mod_mask mask = 0; 	BUG_ON(addr >= end);	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_bad(*pgd))			mask |= PGTBL_PGD_MODIFIED;		if (pgd_none_or_clear_bad(pgd))			continue;		vunmap_p4d_range(pgd, addr, next, &mask);	} while (pgd++, addr = next, addr != end); 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)		arch_sync_kernel_mappings(start, end);} static int vmap_pte_range(pmd_t *pmd, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	pte_t *pte; 	/*	 * nr is a running index into the array which helps higher level	 * callers keep track of where we're up to.	 */ 	pte = pte_alloc_kernel_track(pmd, addr, mask);	if (!pte)		return -ENOMEM;	do {		struct page *page = pages[*nr]; 		if (WARN_ON(!pte_none(*pte)))			return -EBUSY;		if (WARN_ON(!page))			return -ENOMEM;		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));		(*nr)++;	} while (pte++, addr += PAGE_SIZE, addr != end);	*mask |= PGTBL_PTE_MODIFIED;	return 0;} static int vmap_pmd_range(pud_t *pud, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	pmd_t *pmd;	unsigned long next; 	pmd = pmd_alloc_track(&init_mm, pud, addr, mask);	if (!pmd)		return -ENOMEM;	do {		next = pmd_addr_end(addr, end);		if (vmap_pte_range(pmd, addr, next, prot, pages, nr, mask))			return -ENOMEM;	} while (pmd++, addr = next, addr != end);	return 0;} static int vmap_pud_range(p4d_t *p4d, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	pud_t *pud;	unsigned long next; 	pud = pud_alloc_track(&init_mm, p4d, addr, mask);	if (!pud)		return -ENOMEM;	do {		next = pud_addr_end(addr, end);		if (vmap_pmd_range(pud, addr, next, prot, pages, nr, mask))			return -ENOMEM;	} while (pud++, addr = next, addr != end);	return 0;} static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr,		pgtbl_mod_mask *mask){	p4d_t *p4d;	unsigned long next; 	p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);	if (!p4d)		return -ENOMEM;	do {		next = p4d_addr_end(addr, end);		if (vmap_pud_range(p4d, addr, next, prot, pages, nr, mask))			return -ENOMEM;	} while (p4d++, addr = next, addr != end);	return 0;} /** * map_kernel_range_noflush - map kernel VM area with the specified pages * @addr: start of the VM area to map * @size: size of the VM area to map * @prot: page protection flags to use * @pages: pages to map * * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size specify should * have been allocated using get_vm_area() and its friends. * * NOTE: * This function does NOT do any cache flushing.  The caller is responsible for * calling flush_cache_vmap() on to-be-mapped areas before calling this * function. * * RETURNS: * 0 on success, -errno on failure. */int map_kernel_range_noflush(unsigned long addr, unsigned long size,			     pgprot_t prot, struct page **pages){	unsigned long start = addr;	unsigned long end = addr + size;	unsigned long next;	pgd_t *pgd;	int err = 0;	int nr = 0;	pgtbl_mod_mask mask = 0; 	BUG_ON(addr >= end);	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_bad(*pgd))			mask |= PGTBL_PGD_MODIFIED;		err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr, &mask);		if (err)			return err;	} while (pgd++, addr = next, addr != end); 	if (mask & ARCH_PAGE_TABLE_SYNC_MASK)		arch_sync_kernel_mappings(start, end); 	return 0;} int map_kernel_range(unsigned long start, unsigned long size, pgprot_t prot,		struct page **pages){	int ret; 	ret = map_kernel_range_noflush(start, size, prot, pages);	flush_cache_vmap(start, start + size);	return ret;} int is_vmalloc_or_module_addr(const void *x){	/*	 * ARM, x86-64 and sparc64 put modules in a special place,	 * and fall back on vmalloc() if that fails. Others	 * just put it in the vmalloc space.	 */#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)	unsigned long addr = (unsigned long)x;	if (addr >= MODULES_VADDR && addr < MODULES_END)		return 1;#endif	return is_vmalloc_addr(x);} /* * Walk a vmap address to the struct page it maps. */struct page *vmalloc_to_page(const void *vmalloc_addr){	unsigned long addr = (unsigned long) vmalloc_addr;	struct page *page = NULL;	pgd_t *pgd = pgd_offset_k(addr);	p4d_t *p4d;	pud_t *pud;	pmd_t *pmd;	pte_t *ptep, pte; 	/*	 * XXX we might need to change this if we add VIRTUAL_BUG_ON for	 * architectures that do not vmalloc module space	 */	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 	if (pgd_none(*pgd))		return NULL;	p4d = p4d_offset(pgd, addr);	if (p4d_none(*p4d))		return NULL;	pud = pud_offset(p4d, addr); 	/*	 * Don't dereference bad PUD or PMD (below) entries. This will also	 * identify huge mappings, which we may encounter on architectures	 * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be	 * identified as vmalloc addresses by is_vmalloc_addr(), but are	 * not [unambiguously] associated with a struct page, so there is	 * no correct value to return for them.	 */	WARN_ON_ONCE(pud_bad(*pud));	if (pud_none(*pud) || pud_bad(*pud))		return NULL;	pmd = pmd_offset(pud, addr);	WARN_ON_ONCE(pmd_bad(*pmd));	if (pmd_none(*pmd) || pmd_bad(*pmd))		return NULL; 	ptep = pte_offset_map(pmd, addr);	pte = *ptep;	if (pte_present(pte))		page = pte_page(pte);	pte_unmap(ptep);	return page;}EXPORT_SYMBOL(vmalloc_to_page); /* * Map a vmalloc()-space virtual address to the physical page frame number. */unsigned long vmalloc_to_pfn(const void *vmalloc_addr){	return page_to_pfn(vmalloc_to_page(vmalloc_addr));}EXPORT_SYMBOL(vmalloc_to_pfn);  /*** Global kva allocator ***/ #define DEBUG_AUGMENT_PROPAGATE_CHECK 0#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0  static DEFINE_SPINLOCK(vmap_area_lock);static DEFINE_SPINLOCK(free_vmap_area_lock);/* Export for kexec only */LIST_HEAD(vmap_area_list);static struct rb_root vmap_area_root = RB_ROOT;static bool vmap_initialized __read_mostly; static struct rb_root purge_vmap_area_root = RB_ROOT;static LIST_HEAD(purge_vmap_area_list);static DEFINE_SPINLOCK(purge_vmap_area_lock); /* * This kmem_cache is used for vmap_area objects. Instead of * allocating from slab we reuse an object from this cache to * make things faster. Especially in "no edge" splitting of * free block. */static struct kmem_cache *vmap_area_cachep; /* * This linked list is used in pair with free_vmap_area_root. * It gives O(1) access to prev/next to perform fast coalescing. */static LIST_HEAD(free_vmap_area_list); /* * This augment red-black tree represents the free vmap space. * All vmap_area objects in this tree are sorted by va->va_start * address. It is used for allocation and merging when a vmap * object is released. * * Each vmap_area node contains a maximum available free block * of its sub-tree, right or left. Therefore it is possible to * find a lowest match of free area. */static struct rb_root free_vmap_area_root = RB_ROOT; /* * Preload a CPU with one object for "no edge" split case. The * aim is to get rid of allocations from the atomic context, thus * to use more permissive allocation masks. */static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); static __always_inline unsigned longva_size(struct vmap_area *va){	return (va->va_end - va->va_start);} static __always_inline unsigned longget_subtree_max_size(struct rb_node *node){	struct vmap_area *va; 	va = rb_entry_safe(node, struct vmap_area, rb_node);	return va ? va->subtree_max_size : 0;} /* * Gets called when remove the node and rotate. */static __always_inline unsigned longcompute_subtree_max_size(struct vmap_area *va){	return max3(va_size(va),		get_subtree_max_size(va->rb_node.rb_left),		get_subtree_max_size(va->rb_node.rb_right));} RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,	struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) static void purge_vmap_area_lazy(void);static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);static unsigned long lazy_max_pages(void); static atomic_long_t nr_vmalloc_pages; unsigned long vmalloc_nr_pages(void){	return atomic_long_read(&nr_vmalloc_pages);} static struct vmap_area *__find_vmap_area(unsigned long addr){	struct rb_node *n = vmap_area_root.rb_node; 	while (n) {

Lines 1–500 of 3609. Load more