// SPDX-License-Identifier: GPL-2.0-only/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. *              Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/mm_inline.h>#include <linux/sched/mm.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/task.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/memremap.h>#include <linux/kmsan.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/writeback.h>#include <linux/memcontrol.h>#include <linux/mmu_notifier.h>#include <linux/leafops.h>#include <linux/elf.h>#include <linux/gfp.h>#include <linux/migrate.h>#include <linux/string.h>#include <linux/shmem_fs.h>#include <linux/memory-tiers.h>#include <linux/debugfs.h>#include <linux/userfaultfd_k.h>#include <linux/dax.h>#include <linux/oom.h>#include <linux/numa.h>#include <linux/perf_event.h>#include <linux/ptrace.h>#include <linux/vmalloc.h>#include <linux/sched/sysctl.h>#include <linux/pgalloc.h>#include <linux/uaccess.h> #include <trace/events/kmem.h> #include <asm/io.h>#include <asm/mmu_context.h>#include <asm/tlb.h>#include <asm/tlbflush.h> #include "pgalloc-track.h"#include "internal.h"#include "swap.h" #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif static vm_fault_t do_fault(struct vm_fault *vmf);static vm_fault_t do_anonymous_page(struct vm_fault *vmf);static bool vmf_pte_changed(struct vm_fault *vmf); /* * Return true if the original pte was a uffd-wp pte marker (so the pte was * wr-protected). */static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf){	if (!userfaultfd_wp(vmf->vma))		return false;	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))		return false; 	return pte_is_uffd_wp_marker(vmf->orig_pte);} /* * Randomize the address space (stacks, mmaps, brk, etc.). * * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, *   as ancient (libc5 based) binaries can segfault. ) */int randomize_va_space __read_mostly =#ifdef CONFIG_COMPAT_BRK					1;#else					2;#endif static const struct ctl_table mmu_sysctl_table[] = {	{		.procname	= "randomize_va_space",		.data		= &randomize_va_space,		.maxlen		= sizeof(int),		.mode		= 0644,		.proc_handler	= proc_dointvec,	},}; static int __init init_mm_sysctl(void){	register_sysctl_init("kernel", mmu_sysctl_table);	return 0;} subsys_initcall(init_mm_sysctl); #ifndef arch_wants_old_prefaulted_ptestatic inline bool arch_wants_old_prefaulted_pte(void){	/*	 * Transitioning a PTE from 'old' to 'young' can be expensive on	 * some architectures, even if it's performed in hardware. By	 * default, "false" means prefaulted entries will be 'young'.	 */	return false;}#endif static int __init disable_randmaps(char *s){	randomize_va_space = 0;	return 1;}__setup("norandmaps", disable_randmaps); unsigned long highest_memmap_pfn __read_mostly; void mm_trace_rss_stat(struct mm_struct *mm, int member){	trace_rss_stat(mm, member);} /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,			   unsigned long addr){	pgtable_t token = pmd_pgtable(*pmd);	pmd_clear(pmd);	pte_free_tlb(tlb, token, addr);	mm_dec_nr_ptes(tlb->mm);} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd, addr);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd, start);	mm_dec_nr_pmds(tlb->mm);} static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= P4D_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= P4D_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pud = pud_offset(p4d, start);	p4d_clear(p4d);	pud_free_tlb(tlb, pud, start);	mm_dec_nr_puds(tlb->mm);} static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	p4d_t *p4d;	unsigned long next;	unsigned long start; 	start = addr;	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end);		if (p4d_none_or_clear_bad(p4d))			continue;		free_pud_range(tlb, p4d, addr, next, floor, ceiling);	} while (p4d++, addr = next, addr != end); 	start &= PGDIR_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PGDIR_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	p4d = p4d_offset(pgd, start);	pgd_clear(pgd);	p4d_free_tlb(tlb, p4d, start);} /** * free_pgd_range - Unmap and free page tables in the range * @tlb: the mmu_gather containing pending TLB flush info * @addr: virtual address start * @end: virtual address end * @floor: lowest address boundary * @ceiling: highest address boundary * * This function tears down all user-level page tables in the * specified virtual address range [@addr..@end). It is part of * the memory unmap flow. */void free_pgd_range(struct mmu_gather *tlb,			unsigned long addr, unsigned long end,			unsigned long floor, unsigned long ceiling){	pgd_t *pgd;	unsigned long next; 	/*	 * The next few lines have given us lots of grief...	 *	 * Why are we testing PMD* at this top level?  Because often	 * there will be no work to do at all, and we'd prefer not to	 * go all the way down to the bottom just to discover that.	 *	 * Why all these "- 1"s?  Because 0 represents both the bottom	 * of the address space and the top of it (using -1 for the	 * top wouldn't help much: the masks would do the wrong thing).	 * The rule is that addr 0 and floor 0 refer to the bottom of	 * the address space, but end 0 and ceiling 0 refer to the top	 * Comparisons need to use "end - 1" and "ceiling - 1" (though	 * that end 0 case should be mythical).	 *	 * Wherever addr is brought up or ceiling brought down, we must	 * be careful to reject "the opposite 0" before it confuses the	 * subsequent tests.  But what about where end is brought down	 * by PMD_SIZE below? no, end can't go down to 0 there.	 *	 * Whereas we round start (addr) and ceiling down, by different	 * masks at different levels, in order to test whether a table	 * now has no other vmas using it, so can be freed, we don't	 * bother to round floor or end up - the tests don't need that.	 */ 	addr &= PMD_MASK;	if (addr < floor) {		addr += PMD_SIZE;		if (!addr)			return;	}	if (ceiling) {		ceiling &= PMD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		end -= PMD_SIZE;	if (addr > end - 1)		return;	/*	 * We add page table cache pages with PAGE_SIZE,	 * (see pte_free_tlb()), flush the tlb if we need	 */	tlb_change_page_size(tlb, PAGE_SIZE);	pgd = pgd_offset(tlb->mm, addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);	} while (pgd++, addr = next, addr != end);} /** * free_pgtables() - Free a range of page tables * @tlb: The mmu gather * @unmap: The unmap_desc * * Note: pg_start and pg_end are provided to indicate the absolute range of the * page tables that should be removed.  This can differ from the vma mappings on * some archs that may have mappings that need to be removed outside the vmas. * Note that the prev->vm_end and next->vm_start are often used. * * The vma_end differs from the pg_end when a dup_mmap() failed and the tree has * unrelated data to the mm_struct being torn down. */void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *unmap){	struct unlink_vma_file_batch vb;	struct ma_state *mas = unmap->mas;	struct vm_area_struct *vma = unmap->first; 	/*	 * Note: USER_PGTABLES_CEILING may be passed as the value of pg_end and	 * may be 0.  Underflow is expected in this case.  Otherwise the	 * pagetable end is exclusive.  vma_end is exclusive.  The last vma	 * address should never be larger than the pagetable end.	 */	WARN_ON_ONCE(unmap->vma_end - 1 > unmap->pg_end - 1); 	tlb_free_vmas(tlb); 	do {		unsigned long addr = vma->vm_start;		struct vm_area_struct *next; 		next = mas_find(mas, unmap->tree_end - 1); 		/*		 * Hide vma from rmap and truncate_pagecache before freeing		 * pgtables		 */		if (unmap->mm_wr_locked)			vma_start_write(vma);		unlink_anon_vmas(vma); 		unlink_file_vma_batch_init(&vb);		unlink_file_vma_batch_add(&vb, vma); 		/*		 * Optimization: gather nearby vmas into one call down		 */		while (next && next->vm_start <= vma->vm_end + PMD_SIZE) {			vma = next;			next = mas_find(mas, unmap->tree_end - 1);			if (unmap->mm_wr_locked)				vma_start_write(vma);			unlink_anon_vmas(vma);			unlink_file_vma_batch_add(&vb, vma);		}		unlink_file_vma_batch_final(&vb); 		free_pgd_range(tlb, addr, vma->vm_end, unmap->pg_start,			       next ? next->vm_start : unmap->pg_end);		vma = next;	} while (vma);} void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte){	spinlock_t *ptl = pmd_lock(mm, pmd); 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		mm_inc_nr_ptes(mm);		/*		 * Ensure all pte setup (eg. pte page lock and page clearing) are		 * visible before the pte is made visible to other CPUs by being		 * put into page tables.		 *		 * The other side of the story is the pointer chasing in the page		 * table walking code (when walking the page table without locking;		 * ie. most of the time). Fortunately, these data accesses consist		 * of a chain of data-dependent loads, meaning most CPUs (alpha		 * being the notable exception) will already guarantee loads are		 * seen in-order. See the alpha page table accessors for the		 * smp_rmb() barriers in page table walking code.		 */		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */		pmd_populate(mm, pmd, *pte);		*pte = NULL;	}	spin_unlock(ptl);} int __pte_alloc(struct mm_struct *mm, pmd_t *pmd){	pgtable_t new = pte_alloc_one(mm);	if (!new)		return -ENOMEM; 	pmd_install(mm, pmd, &new);	if (new)		pte_free(mm, new);	return 0;} int __pte_alloc_kernel(pmd_t *pmd){	pte_t *new = pte_alloc_one_kernel(&init_mm);	if (!new)		return -ENOMEM; 	spin_lock(&init_mm.page_table_lock);	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		smp_wmb(); /* See comment in pmd_install() */		pmd_populate_kernel(&init_mm, pmd, new);		new = NULL;	}	spin_unlock(&init_mm.page_table_lock);	if (new)		pte_free_kernel(&init_mm, new);	return 0;} static inline void init_rss_vec(int *rss){	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);} static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss){	int i; 	for (i = 0; i < NR_MM_COUNTERS; i++)		if (rss[i])			add_mm_counter(mm, i, rss[i]);} static bool is_bad_page_map_ratelimited(void){	static unsigned long resume;	static unsigned long nr_shown;	static unsigned long nr_unshown; 

Lines 1–500 of 7745. Load more