// SPDX-License-Identifier: GPL-2.0-only/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. *              Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/sched/mm.h>#include <linux/sched/coredump.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/task.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/memremap.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/pfn_t.h>#include <linux/writeback.h>#include <linux/memcontrol.h>#include <linux/mmu_notifier.h>#include <linux/swapops.h>#include <linux/elf.h>#include <linux/gfp.h>#include <linux/migrate.h>#include <linux/string.h>#include <linux/debugfs.h>#include <linux/userfaultfd_k.h>#include <linux/dax.h>#include <linux/oom.h>#include <linux/numa.h>#include <linux/perf_event.h>#include <linux/ptrace.h>#include <linux/vmalloc.h> #include <trace/events/kmem.h> #include <asm/io.h>#include <asm/mmu_context.h>#include <asm/pgalloc.h>#include <linux/uaccess.h>#include <asm/tlb.h>#include <asm/tlbflush.h> #include "pgalloc-track.h"#include "internal.h" #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif #ifndef CONFIG_NEED_MULTIPLE_NODES/* use the per-pgdat data instead for discontigmem - mbligh */unsigned long max_mapnr;EXPORT_SYMBOL(max_mapnr); struct page *mem_map;EXPORT_SYMBOL(mem_map);#endif /* * A number of key systems in x86 including ioremap() rely on the assumption * that high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL * and ZONE_HIGHMEM. */void *high_memory;EXPORT_SYMBOL(high_memory); /* * Randomize the address space (stacks, mmaps, brk, etc.). * * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, *   as ancient (libc5 based) binaries can segfault. ) */int randomize_va_space __read_mostly =#ifdef CONFIG_COMPAT_BRK					1;#else					2;#endif #ifndef arch_faults_on_old_ptestatic inline bool arch_faults_on_old_pte(void){	/*	 * Those arches which don't have hw access flag feature need to	 * implement their own helper. By default, "true" means pagefault	 * will be hit on old pte.	 */	return true;}#endif #ifndef arch_wants_old_prefaulted_ptestatic inline bool arch_wants_old_prefaulted_pte(void){	/*	 * Transitioning a PTE from 'old' to 'young' can be expensive on	 * some architectures, even if it's performed in hardware. By	 * default, "false" means prefaulted entries will be 'young'.	 */	return false;}#endif static int __init disable_randmaps(char *s){	randomize_va_space = 0;	return 1;}__setup("norandmaps", disable_randmaps); unsigned long zero_pfn __read_mostly;EXPORT_SYMBOL(zero_pfn); unsigned long highest_memmap_pfn __read_mostly; /* * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() */static int __init init_zero_pfn(void){	zero_pfn = page_to_pfn(ZERO_PAGE(0));	return 0;}early_initcall(init_zero_pfn); void mm_trace_rss_stat(struct mm_struct *mm, int member, long count){	trace_rss_stat(mm, member, count);} #if defined(SPLIT_RSS_COUNTING) void sync_mm_rss(struct mm_struct *mm){	int i; 	for (i = 0; i < NR_MM_COUNTERS; i++) {		if (current->rss_stat.count[i]) {			add_mm_counter(mm, i, current->rss_stat.count[i]);			current->rss_stat.count[i] = 0;		}	}	current->rss_stat.events = 0;} static void add_mm_counter_fast(struct mm_struct *mm, int member, int val){	struct task_struct *task = current; 	if (likely(task->mm == mm))		task->rss_stat.count[member] += val;	else		add_mm_counter(mm, member, val);}#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) /* sync counter once per 64 page faults */#define TASK_RSS_EVENTS_THRESH	(64)static void check_sync_rss_stat(struct task_struct *task){	if (unlikely(task != current))		return;	if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))		sync_mm_rss(task->mm);}#else /* SPLIT_RSS_COUNTING */ #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) static void check_sync_rss_stat(struct task_struct *task){} #endif /* SPLIT_RSS_COUNTING */ /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,			   unsigned long addr){	pgtable_t token = pmd_pgtable(*pmd);	pmd_clear(pmd);	pte_free_tlb(tlb, token, addr);	mm_dec_nr_ptes(tlb->mm);} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd, addr);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd, start);	mm_dec_nr_pmds(tlb->mm);} static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= P4D_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= P4D_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pud = pud_offset(p4d, start);	p4d_clear(p4d);	pud_free_tlb(tlb, pud, start);	mm_dec_nr_puds(tlb->mm);} static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	p4d_t *p4d;	unsigned long next;	unsigned long start; 	start = addr;	p4d = p4d_offset(pgd, addr);	do {		next = p4d_addr_end(addr, end);		if (p4d_none_or_clear_bad(p4d))			continue;		free_pud_range(tlb, p4d, addr, next, floor, ceiling);	} while (p4d++, addr = next, addr != end); 	start &= PGDIR_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PGDIR_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	p4d = p4d_offset(pgd, start);	pgd_clear(pgd);	p4d_free_tlb(tlb, p4d, start);} /* * This function frees user-level page tables of a process. */void free_pgd_range(struct mmu_gather *tlb,			unsigned long addr, unsigned long end,			unsigned long floor, unsigned long ceiling){	pgd_t *pgd;	unsigned long next; 	/*	 * The next few lines have given us lots of grief...	 *	 * Why are we testing PMD* at this top level?  Because often	 * there will be no work to do at all, and we'd prefer not to	 * go all the way down to the bottom just to discover that.	 *	 * Why all these "- 1"s?  Because 0 represents both the bottom	 * of the address space and the top of it (using -1 for the	 * top wouldn't help much: the masks would do the wrong thing).	 * The rule is that addr 0 and floor 0 refer to the bottom of	 * the address space, but end 0 and ceiling 0 refer to the top	 * Comparisons need to use "end - 1" and "ceiling - 1" (though	 * that end 0 case should be mythical).	 *	 * Wherever addr is brought up or ceiling brought down, we must	 * be careful to reject "the opposite 0" before it confuses the	 * subsequent tests.  But what about where end is brought down	 * by PMD_SIZE below? no, end can't go down to 0 there.	 *	 * Whereas we round start (addr) and ceiling down, by different	 * masks at different levels, in order to test whether a table	 * now has no other vmas using it, so can be freed, we don't	 * bother to round floor or end up - the tests don't need that.	 */ 	addr &= PMD_MASK;	if (addr < floor) {		addr += PMD_SIZE;		if (!addr)			return;	}	if (ceiling) {		ceiling &= PMD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		end -= PMD_SIZE;	if (addr > end - 1)		return;	/*	 * We add page table cache pages with PAGE_SIZE,	 * (see pte_free_tlb()), flush the tlb if we need	 */	tlb_change_page_size(tlb, PAGE_SIZE);	pgd = pgd_offset(tlb->mm, addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		free_p4d_range(tlb, pgd, addr, next, floor, ceiling);	} while (pgd++, addr = next, addr != end);} void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,		unsigned long floor, unsigned long ceiling){	while (vma) {		struct vm_area_struct *next = vma->vm_next;		unsigned long addr = vma->vm_start; 		/*		 * Hide vma from rmap and truncate_pagecache before freeing		 * pgtables		 */		unlink_anon_vmas(vma);		unlink_file_vma(vma); 		if (is_vm_hugetlb_page(vma)) {			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,				floor, next ? next->vm_start : ceiling);		} else {			/*			 * Optimization: gather nearby vmas into one call down			 */			while (next && next->vm_start <= vma->vm_end + PMD_SIZE			       && !is_vm_hugetlb_page(next)) {				vma = next;				next = vma->vm_next;				unlink_anon_vmas(vma);				unlink_file_vma(vma);			}			free_pgd_range(tlb, addr, vma->vm_end,				floor, next ? next->vm_start : ceiling);		}		vma = next;	}} int __pte_alloc(struct mm_struct *mm, pmd_t *pmd){	spinlock_t *ptl;	pgtable_t new = pte_alloc_one(mm);	if (!new)		return -ENOMEM; 	/*	 * Ensure all pte setup (eg. pte page lock and page clearing) are	 * visible before the pte is made visible to other CPUs by being	 * put into page tables.	 *	 * The other side of the story is the pointer chasing in the page	 * table walking code (when walking the page table without locking;	 * ie. most of the time). Fortunately, these data accesses consist	 * of a chain of data-dependent loads, meaning most CPUs (alpha	 * being the notable exception) will already guarantee loads are	 * seen in-order. See the alpha page table accessors for the	 * smp_rmb() barriers in page table walking code.	 */	smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ 	ptl = pmd_lock(mm, pmd);	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		mm_inc_nr_ptes(mm);		pmd_populate(mm, pmd, new);		new = NULL;	}	spin_unlock(ptl);	if (new)		pte_free(mm, new);	return 0;} int __pte_alloc_kernel(pmd_t *pmd){	pte_t *new = pte_alloc_one_kernel(&init_mm);	if (!new)		return -ENOMEM; 	smp_wmb(); /* See comment in __pte_alloc */ 	spin_lock(&init_mm.page_table_lock);	if (likely(pmd_none(*pmd))) {	/* Has another populated it ? */		pmd_populate_kernel(&init_mm, pmd, new);		new = NULL;	}	spin_unlock(&init_mm.page_table_lock);	if (new)		pte_free_kernel(&init_mm, new);	return 0;} static inline void init_rss_vec(int *rss){	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);} static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss){	int i; 	if (current->mm == mm)		sync_mm_rss(mm);

Lines 1–500 of 5323. Load more