// SPDX-License-Identifier: GPL-2.0-only/* *  linux/mm/vmalloc.c * *  Copyright (C) 1993  Linus Torvalds *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 *  Numa awareness, Christoph Lameter, SGI, June 2005 */ #include <linux/vmalloc.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/highmem.h>#include <linux/sched/signal.h>#include <linux/slab.h>#include <linux/spinlock.h>#include <linux/interrupt.h>#include <linux/proc_fs.h>#include <linux/seq_file.h>#include <linux/set_memory.h>#include <linux/debugobjects.h>#include <linux/kallsyms.h>#include <linux/list.h>#include <linux/notifier.h>#include <linux/rbtree.h>#include <linux/radix-tree.h>#include <linux/rcupdate.h>#include <linux/pfn.h>#include <linux/kmemleak.h>#include <linux/atomic.h>#include <linux/compiler.h>#include <linux/llist.h>#include <linux/bitops.h>#include <linux/rbtree_augmented.h> #include <linux/uaccess.h>#include <asm/tlbflush.h>#include <asm/shmparam.h> #include "internal.h" struct vfree_deferred {	struct llist_head list;	struct work_struct wq;};static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); static void __vunmap(const void *, int); static void free_work(struct work_struct *w){	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);	struct llist_node *t, *llnode; 	llist_for_each_safe(llnode, t, llist_del_all(&p->list))		__vunmap((void *)llnode, 1);} /*** Page table manipulation functions ***/ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end){	pte_t *pte; 	pte = pte_offset_kernel(pmd, addr);	do {		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);		WARN_ON(!pte_none(ptent) && !pte_present(ptent));	} while (pte++, addr += PAGE_SIZE, addr != end);} static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end){	pmd_t *pmd;	unsigned long next; 	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_clear_huge(pmd))			continue;		if (pmd_none_or_clear_bad(pmd))			continue;		vunmap_pte_range(pmd, addr, next);	} while (pmd++, addr = next, addr != end);} static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end){	pud_t *pud;	unsigned long next; 	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end);		if (pud_clear_huge(pud))			continue;		if (pud_none_or_clear_bad(pud))			continue;		vunmap_pmd_range(pud, addr, next);	} while (pud++, addr = next, addr != end);} static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end){	p4d_t *p4d;	unsigned long next; 	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end);		if (p4d_clear_huge(p4d))			continue;		if (p4d_none_or_clear_bad(p4d))			continue;		vunmap_pud_range(p4d, addr, next);	} while (p4d++, addr = next, addr != end);} static void vunmap_page_range(unsigned long addr, unsigned long end){	pgd_t *pgd;	unsigned long next; 	BUG_ON(addr >= end);	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		vunmap_p4d_range(pgd, addr, next);	} while (pgd++, addr = next, addr != end);} static int vmap_pte_range(pmd_t *pmd, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr){	pte_t *pte; 	/*	 * nr is a running index into the array which helps higher level	 * callers keep track of where we're up to.	 */ 	pte = pte_alloc_kernel(pmd, addr);	if (!pte)		return -ENOMEM;	do {		struct page *page = pages[*nr]; 		if (WARN_ON(!pte_none(*pte)))			return -EBUSY;		if (WARN_ON(!page))			return -ENOMEM;		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));		(*nr)++;	} while (pte++, addr += PAGE_SIZE, addr != end);	return 0;} static int vmap_pmd_range(pud_t *pud, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr){	pmd_t *pmd;	unsigned long next; 	pmd = pmd_alloc(&init_mm, pud, addr);	if (!pmd)		return -ENOMEM;	do {		next = pmd_addr_end(addr, end);		if (vmap_pte_range(pmd, addr, next, prot, pages, nr))			return -ENOMEM;	} while (pmd++, addr = next, addr != end);	return 0;} static int vmap_pud_range(p4d_t *p4d, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr){	pud_t *pud;	unsigned long next; 	pud = pud_alloc(&init_mm, p4d, addr);	if (!pud)		return -ENOMEM;	do {		next = pud_addr_end(addr, end);		if (vmap_pmd_range(pud, addr, next, prot, pages, nr))			return -ENOMEM;	} while (pud++, addr = next, addr != end);	return 0;} static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,		unsigned long end, pgprot_t prot, struct page **pages, int *nr){	p4d_t *p4d;	unsigned long next; 	p4d = p4d_alloc(&init_mm, pgd, addr);	if (!p4d)		return -ENOMEM;	do {		next = p4d_addr_end(addr, end);		if (vmap_pud_range(p4d, addr, next, prot, pages, nr))			return -ENOMEM;	} while (p4d++, addr = next, addr != end);	return 0;} /* * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and * will have pfns corresponding to the "pages" array. * * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] */static int vmap_page_range_noflush(unsigned long start, unsigned long end,				   pgprot_t prot, struct page **pages){	pgd_t *pgd;	unsigned long next;	unsigned long addr = start;	int err = 0;	int nr = 0; 	BUG_ON(addr >= end);	pgd = pgd_offset_k(addr);	do {		next = pgd_addr_end(addr, end);		err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr);		if (err)			return err;	} while (pgd++, addr = next, addr != end); 	return nr;} static int vmap_page_range(unsigned long start, unsigned long end,			   pgprot_t prot, struct page **pages){	int ret; 	ret = vmap_page_range_noflush(start, end, prot, pages);	flush_cache_vmap(start, end);	return ret;} int is_vmalloc_or_module_addr(const void *x){	/*	 * ARM, x86-64 and sparc64 put modules in a special place,	 * and fall back on vmalloc() if that fails. Others	 * just put it in the vmalloc space.	 */#if defined(CONFIG_MODULES) && defined(MODULES_VADDR)	unsigned long addr = (unsigned long)x;	if (addr >= MODULES_VADDR && addr < MODULES_END)		return 1;#endif	return is_vmalloc_addr(x);} /* * Walk a vmap address to the struct page it maps. */struct page *vmalloc_to_page(const void *vmalloc_addr){	unsigned long addr = (unsigned long) vmalloc_addr;	struct page *page = NULL;	pgd_t *pgd = pgd_offset_k(addr);	p4d_t *p4d;	pud_t *pud;	pmd_t *pmd;	pte_t *ptep, pte; 	/*	 * XXX we might need to change this if we add VIRTUAL_BUG_ON for	 * architectures that do not vmalloc module space	 */	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 	if (pgd_none(*pgd))		return NULL;	p4d = p4d_offset(pgd, addr);	if (p4d_none(*p4d))		return NULL;	pud = pud_offset(p4d, addr); 	/*	 * Don't dereference bad PUD or PMD (below) entries. This will also	 * identify huge mappings, which we may encounter on architectures	 * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be	 * identified as vmalloc addresses by is_vmalloc_addr(), but are	 * not [unambiguously] associated with a struct page, so there is	 * no correct value to return for them.	 */	WARN_ON_ONCE(pud_bad(*pud));	if (pud_none(*pud) || pud_bad(*pud))		return NULL;	pmd = pmd_offset(pud, addr);	WARN_ON_ONCE(pmd_bad(*pmd));	if (pmd_none(*pmd) || pmd_bad(*pmd))		return NULL; 	ptep = pte_offset_map(pmd, addr);	pte = *ptep;	if (pte_present(pte))		page = pte_page(pte);	pte_unmap(ptep);	return page;}EXPORT_SYMBOL(vmalloc_to_page); /* * Map a vmalloc()-space virtual address to the physical page frame number. */unsigned long vmalloc_to_pfn(const void *vmalloc_addr){	return page_to_pfn(vmalloc_to_page(vmalloc_addr));}EXPORT_SYMBOL(vmalloc_to_pfn);  /*** Global kva allocator ***/ #define DEBUG_AUGMENT_PROPAGATE_CHECK 0#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 #define VM_LAZY_FREE	0x02#define VM_VM_AREA	0x04 static DEFINE_SPINLOCK(vmap_area_lock);/* Export for kexec only */LIST_HEAD(vmap_area_list);static LLIST_HEAD(vmap_purge_list);static struct rb_root vmap_area_root = RB_ROOT;static bool vmap_initialized __read_mostly; /* * This kmem_cache is used for vmap_area objects. Instead of * allocating from slab we reuse an object from this cache to * make things faster. Especially in "no edge" splitting of * free block. */static struct kmem_cache *vmap_area_cachep; /* * This linked list is used in pair with free_vmap_area_root. * It gives O(1) access to prev/next to perform fast coalescing. */static LIST_HEAD(free_vmap_area_list); /* * This augment red-black tree represents the free vmap space. * All vmap_area objects in this tree are sorted by va->va_start * address. It is used for allocation and merging when a vmap * object is released. * * Each vmap_area node contains a maximum available free block * of its sub-tree, right or left. Therefore it is possible to * find a lowest match of free area. */static struct rb_root free_vmap_area_root = RB_ROOT; static __always_inline unsigned longva_size(struct vmap_area *va){	return (va->va_end - va->va_start);} static __always_inline unsigned longget_subtree_max_size(struct rb_node *node){	struct vmap_area *va; 	va = rb_entry_safe(node, struct vmap_area, rb_node);	return va ? va->subtree_max_size : 0;} /* * Gets called when remove the node and rotate. */static __always_inline unsigned longcompute_subtree_max_size(struct vmap_area *va){	return max3(va_size(va),		get_subtree_max_size(va->rb_node.rb_left),		get_subtree_max_size(va->rb_node.rb_right));} RB_DECLARE_CALLBACKS(static, free_vmap_area_rb_augment_cb,	struct vmap_area, rb_node, unsigned long, subtree_max_size,	compute_subtree_max_size) static void purge_vmap_area_lazy(void);static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);static unsigned long lazy_max_pages(void); static struct vmap_area *__find_vmap_area(unsigned long addr){	struct rb_node *n = vmap_area_root.rb_node; 	while (n) {		struct vmap_area *va; 		va = rb_entry(n, struct vmap_area, rb_node);		if (addr < va->va_start)			n = n->rb_left;		else if (addr >= va->va_end)			n = n->rb_right;		else			return va;	} 	return NULL;} /* * This function returns back addresses of parent node * and its left or right link for further processing. */static __always_inline struct rb_node **find_va_links(struct vmap_area *va,	struct rb_root *root, struct rb_node *from,	struct rb_node **parent){	struct vmap_area *tmp_va;	struct rb_node **link; 	if (root) {		link = &root->rb_node;		if (unlikely(!*link)) {			*parent = NULL;			return link;		}	} else {		link = &from;	} 	/*	 * Go to the bottom of the tree. When we hit the last point	 * we end up with parent rb_node and correct direction, i name	 * it link, where the new va->rb_node will be attached to.	 */	do {		tmp_va = rb_entry(*link, struct vmap_area, rb_node); 		/*		 * During the traversal we also do some sanity check.		 * Trigger the BUG() if there are sides(left/right)		 * or full overlaps.		 */		if (va->va_start < tmp_va->va_end &&				va->va_end <= tmp_va->va_start)			link = &(*link)->rb_left;		else if (va->va_end > tmp_va->va_start &&				va->va_start >= tmp_va->va_end)			link = &(*link)->rb_right;		else			BUG();	} while (*link); 	*parent = &tmp_va->rb_node;	return link;} static __always_inline struct list_head *get_va_next_sibling(struct rb_node *parent, struct rb_node **link){	struct list_head *list; 	if (unlikely(!parent))		/*		 * The red-black tree where we try to find VA neighbors		 * before merging or inserting is empty, i.e. it means		 * there is no free vmap space. Normally it does not		 * happen but we handle this case anyway.		 */		return NULL; 	list = &rb_entry(parent, struct vmap_area, rb_node)->list;	return (&parent->rb_right == link ? list->next : list);} static __always_inline voidlink_va(struct vmap_area *va, struct rb_root *root,	struct rb_node *parent, struct rb_node **link, struct list_head *head){	/*	 * VA is still not in the list, but we can	 * identify its future previous list_head node.	 */	if (likely(parent)) {		head = &rb_entry(parent, struct vmap_area, rb_node)->list;		if (&parent->rb_right != link)			head = head->prev;	}

Lines 1–500 of 3470. Load more