// SPDX-License-Identifier: GPL-2.0-only/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. *              Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/mm_inline.h>#include <linux/sched/mm.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/task.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/memremap.h>#include <linux/kmsan.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/pfn_t.h>#include <linux/writeback.h>#include <linux/memcontrol.h>#include <linux/mmu_notifier.h>#include <linux/swapops.h>#include <linux/elf.h>#include <linux/gfp.h>#include <linux/migrate.h>#include <linux/string.h>#include <linux/memory-tiers.h>#include <linux/debugfs.h>#include <linux/userfaultfd_k.h>#include <linux/dax.h>#include <linux/oom.h>#include <linux/numa.h>#include <linux/perf_event.h>#include <linux/ptrace.h>#include <linux/vmalloc.h>#include <linux/sched/sysctl.h> #include <trace/events/kmem.h> #include <asm/io.h>#include <asm/mmu_context.h>#include <asm/pgalloc.h>#include <linux/uaccess.h>#include <asm/tlb.h>#include <asm/tlbflush.h> #include "pgalloc-track.h"#include "internal.h"#include "swap.h" #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif #ifndef CONFIG_NUMAunsigned long max_mapnr;EXPORT_SYMBOL(max_mapnr); struct page *mem_map;EXPORT_SYMBOL(mem_map);#endif static vm_fault_t do_fault(struct vm_fault *vmf);static vm_fault_t do_anonymous_page(struct vm_fault *vmf);static bool vmf_pte_changed(struct vm_fault *vmf); /* * Return true if the original pte was a uffd-wp pte marker (so the pte was * wr-protected). */static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf){	if (!userfaultfd_wp(vmf->vma))		return false;	if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))		return false; 	return pte_marker_uffd_wp(vmf->orig_pte);} /* * A number of key systems in x86 including ioremap() rely on the assumption * that high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL. */void *high_memory;EXPORT_SYMBOL(high_memory); /* * Randomize the address space (stacks, mmaps, brk, etc.). * * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, *   as ancient (libc5 based) binaries can segfault. ) */int randomize_va_space __read_mostly =#ifdef CONFIG_COMPAT_BRK					1;#else					2;#endif #ifndef arch_wants_old_prefaulted_ptestatic inline bool arch_wants_old_prefaulted_pte(void){	/*	 * Transitioning a PTE from 'old' to 'young' can be expensive on	 * some architectures, even if it's performed in hardware. By	 * default, "false" means prefaulted entries will be 'young'.	 */	return false;}#endif static int __init disable_randmaps(char *s){	randomize_va_space = 0;	return 1;}__setup("norandmaps", disable_randmaps); unsigned long zero_pfn __read_mostly;EXPORT_SYMBOL(zero_pfn); unsigned long highest_memmap_pfn __read_mostly; /* * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() */static int __init init_zero_pfn(void){	zero_pfn = page_to_pfn(ZERO_PAGE(0));	return 0;}early_initcall(init_zero_pfn); void mm_trace_rss_stat(struct mm_struct *mm, int member){	trace_rss_stat(mm, member);} /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,			   unsigned long addr){	pgtable_t token = pmd_pgtable(*pmd);	pmd_clear(pmd);	pte_free_tlb(tlb, token, addr);	mm_dec_nr_ptes(tlb->mm);} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd, addr);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd, start);	mm_dec_nr_pmds(tlb->mm);} static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= P4D_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= P4D_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pud = pud_offset(p4d, start);	p4d_clear(p4d);	pud_free_tlb(tlb, pud, start);	mm_dec_nr_puds(tlb->mm);} static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	p4d_t *p4d;	unsigned long next;	unsigned long start; 	start = addr;	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end);		if (p4d_none_or_clear_bad(p4d))			continue;		free_pud_range(tlb, p4d, addr, next, floor, ceiling);	} while (p4d++, addr = next, addr != end); 	start &= PGDIR_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PGDIR_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	p4d = p4d_offset(pgd, start);	pgd_clear(pgd);	p4d_free_tlb(tlb, p4d, start);} /* * This function frees user-level page tables of a process. */void free_pgd_range(struct mmu_gather *tlb,			unsigned long addr, unsigned long end,			unsigned long floor, unsigned long ceiling){	pgd_t *pgd;	unsigned long next; 	/*	 * The next few lines have given us lots of grief...	 *	 * Why are we testing PMD* at this top level?  Because often	 * there will be no work to do at all, and we'd prefer not to	 * go all the way down to the bottom just to discover that.	 *	 * Why all these "- 1"s?  Because 0 represents both the bottom	 * of the address space and the top of it (using -1 for the	 * top wouldn't help much: the masks would do the wrong thing).	 * The rule is that addr 0 and floor 0 refer to the bottom of	 * the address space, but end 0 and ceiling 0 refer to the top	 * Comparisons need to use "end - 1" and "ceiling - 1" (though	 * that end 0 case should be mythical).	 *	 * Wherever addr is brought up or ceiling brought down, we must	 * be careful to reject "the opposite 0" before it confuses the	 * subsequent tests.  But what about where end is brought down	 * by PMD_SIZE below? no, end can't go down to 0 there.	 *	 * Whereas we round start (addr) and ceiling down, by different	 * masks at different levels, in order to test whether a table	 * now has no other vmas using it, so can be freed, we don't	 * bother to round floor or end up - the tests don't need that.	 */ 	addr &= PMD_MASK;	if (addr < floor) {		addr += PMD_SIZE;		if (!addr)			return;	}	if (ceiling) {		ceiling &= PMD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		end -= PMD_SIZE;	if (addr > end - 1)		return;	/*	 * We add page table cache pages with PAGE_SIZE,	 * (see pte_free_tlb()), flush the tlb if we need	 */	tlb_change_page_size(tlb, PAGE_SIZE);	pgd = pgd_offset(tlb->mm, addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);	} while (pgd++, addr = next, addr != end);} void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,		   struct vm_area_struct *vma, unsigned long floor,		   unsigned long ceiling, bool mm_wr_locked){	struct unlink_vma_file_batch vb; 	do {		unsigned long addr = vma->vm_start;		struct vm_area_struct *next; 		/*		 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may		 * be 0.  This will underflow and is okay.		 */		next = mas_find(mas, ceiling - 1);		if (unlikely(xa_is_zero(next)))			next = NULL; 		/*		 * Hide vma from rmap and truncate_pagecache before freeing		 * pgtables		 */		if (mm_wr_locked)			vma_start_write(vma);		unlink_anon_vmas(vma); 		if (is_vm_hugetlb_page(vma)) {			unlink_file_vma(vma);			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,				floor, next ? next->vm_start : ceiling);		} else {			unlink_file_vma_batch_init(&vb);			unlink_file_vma_batch_add(&vb, vma); 			/*			 * Optimization: gather nearby vmas into one call down			 */			while (next && next->vm_start <= vma->vm_end + PMD_SIZE			       && !is_vm_hugetlb_page(next)) {				vma = next;				next = mas_find(mas, ceiling - 1);				if (unlikely(xa_is_zero(next)))					next = NULL;				if (mm_wr_locked)					vma_start_write(vma);				unlink_anon_vmas(vma);				unlink_file_vma_batch_add(&vb, vma);			}			unlink_file_vma_batch_final(&vb);			free_pgd_range(tlb, addr, vma->vm_end,				floor, next ? next->vm_start : ceiling);		}		vma = next;	} while (vma);} void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte){	spinlock_t *ptl = pmd_lock(mm, pmd); 	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		mm_inc_nr_ptes(mm);		/*		 * Ensure all pte setup (eg. pte page lock and page clearing) are		 * visible before the pte is made visible to other CPUs by being		 * put into page tables.		 *		 * The other side of the story is the pointer chasing in the page		 * table walking code (when walking the page table without locking;		 * ie. most of the time). Fortunately, these data accesses consist		 * of a chain of data-dependent loads, meaning most CPUs (alpha		 * being the notable exception) will already guarantee loads are		 * seen in-order. See the alpha page table accessors for the		 * smp_rmb() barriers in page table walking code.		 */		smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */		pmd_populate(mm, pmd, *pte);		*pte = NULL;	}	spin_unlock(ptl);} int __pte_alloc(struct mm_struct *mm, pmd_t *pmd){	pgtable_t new = pte_alloc_one(mm);	if (!new)		return -ENOMEM; 	pmd_install(mm, pmd, &new);	if (new)		pte_free(mm, new);	return 0;} int __pte_alloc_kernel(pmd_t *pmd){	pte_t *new = pte_alloc_one_kernel(&init_mm);	if (!new)		return -ENOMEM; 	spin_lock(&init_mm.page_table_lock);	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		smp_wmb(); /* See comment in pmd_install() */		pmd_populate_kernel(&init_mm, pmd, new);		new = NULL;	}	spin_unlock(&init_mm.page_table_lock);	if (new)		pte_free_kernel(&init_mm, new);	return 0;} static inline void init_rss_vec(int *rss){	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);} static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss){	int i; 	for (i = 0; i < NR_MM_COUNTERS; i++)		if (rss[i])			add_mm_counter(mm, i, rss[i]);} /* * This function is called to print an error when a bad pte * is found. For example, we might have a PFN-mapped pte in * a region that doesn't allow it. * * The calling function must still handle the error. */static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,			  pte_t pte, struct page *page){	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);	p4d_t *p4d = p4d_offset(pgd, addr);	pud_t *pud = pud_offset(p4d, addr);

Lines 1–500 of 6978. Load more