/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. *              Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/sched/mm.h>#include <linux/sched/coredump.h>#include <linux/sched/numa_balancing.h>#include <linux/sched/task.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/ksm.h>#include <linux/rmap.h>#include <linux/export.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/pfn_t.h>#include <linux/writeback.h>#include <linux/memcontrol.h>#include <linux/mmu_notifier.h>#include <linux/kallsyms.h>#include <linux/swapops.h>#include <linux/elf.h>#include <linux/gfp.h>#include <linux/migrate.h>#include <linux/string.h>#include <linux/dma-debug.h>#include <linux/debugfs.h>#include <linux/userfaultfd_k.h>#include <linux/dax.h>#include <linux/oom.h> #include <asm/io.h>#include <asm/mmu_context.h>#include <asm/pgalloc.h>#include <linux/uaccess.h>#include <asm/tlb.h>#include <asm/tlbflush.h>#include <asm/pgtable.h> #include "internal.h" #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.#endif #ifndef CONFIG_NEED_MULTIPLE_NODES/* use the per-pgdat data instead for discontigmem - mbligh */unsigned long max_mapnr;EXPORT_SYMBOL(max_mapnr); struct page *mem_map;EXPORT_SYMBOL(mem_map);#endif /* * A number of key systems in x86 including ioremap() rely on the assumption * that high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL * and ZONE_HIGHMEM. */void *high_memory;EXPORT_SYMBOL(high_memory); /* * Randomize the address space (stacks, mmaps, brk, etc.). * * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization, *   as ancient (libc5 based) binaries can segfault. ) */int randomize_va_space __read_mostly =#ifdef CONFIG_COMPAT_BRK					1;#else					2;#endif static int __init disable_randmaps(char *s){	randomize_va_space = 0;	return 1;}__setup("norandmaps", disable_randmaps); unsigned long zero_pfn __read_mostly;EXPORT_SYMBOL(zero_pfn); unsigned long highest_memmap_pfn __read_mostly; /* * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init() */static int __init init_zero_pfn(void){	zero_pfn = page_to_pfn(ZERO_PAGE(0));	return 0;}core_initcall(init_zero_pfn);  #if defined(SPLIT_RSS_COUNTING) void sync_mm_rss(struct mm_struct *mm){	int i; 	for (i = 0; i < NR_MM_COUNTERS; i++) {		if (current->rss_stat.count[i]) {			add_mm_counter(mm, i, current->rss_stat.count[i]);			current->rss_stat.count[i] = 0;		}	}	current->rss_stat.events = 0;} static void add_mm_counter_fast(struct mm_struct *mm, int member, int val){	struct task_struct *task = current; 	if (likely(task->mm == mm))		task->rss_stat.count[member] += val;	else		add_mm_counter(mm, member, val);}#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1) /* sync counter once per 64 page faults */#define TASK_RSS_EVENTS_THRESH	(64)static void check_sync_rss_stat(struct task_struct *task){	if (unlikely(task != current))		return;	if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))		sync_mm_rss(task->mm);}#else /* SPLIT_RSS_COUNTING */ #define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member) static void check_sync_rss_stat(struct task_struct *task){} #endif /* SPLIT_RSS_COUNTING */ #ifdef HAVE_GENERIC_MMU_GATHER static bool tlb_next_batch(struct mmu_gather *tlb){	struct mmu_gather_batch *batch; 	batch = tlb->active;	if (batch->next) {		tlb->active = batch->next;		return true;	} 	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)		return false; 	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);	if (!batch)		return false; 	tlb->batch_count++;	batch->next = NULL;	batch->nr   = 0;	batch->max  = MAX_GATHER_BATCH; 	tlb->active->next = batch;	tlb->active = batch; 	return true;} void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,				unsigned long start, unsigned long end){	tlb->mm = mm; 	/* Is it from 0 to ~0? */	tlb->fullmm     = !(start | (end+1));	tlb->need_flush_all = 0;	tlb->local.next = NULL;	tlb->local.nr   = 0;	tlb->local.max  = ARRAY_SIZE(tlb->__pages);	tlb->active     = &tlb->local;	tlb->batch_count = 0; #ifdef CONFIG_HAVE_RCU_TABLE_FREE	tlb->batch = NULL;#endif	tlb->page_size = 0; 	__tlb_reset_range(tlb);} static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb){	if (!tlb->end)		return; 	tlb_flush(tlb);	mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);#ifdef CONFIG_HAVE_RCU_TABLE_FREE	tlb_table_flush(tlb);#endif	__tlb_reset_range(tlb);} static void tlb_flush_mmu_free(struct mmu_gather *tlb){	struct mmu_gather_batch *batch; 	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {		free_pages_and_swap_cache(batch->pages, batch->nr);		batch->nr = 0;	}	tlb->active = &tlb->local;} void tlb_flush_mmu(struct mmu_gather *tlb){	tlb_flush_mmu_tlbonly(tlb);	tlb_flush_mmu_free(tlb);} /* tlb_finish_mmu *	Called at the end of the shootdown operation to free up any resources *	that were required. */void arch_tlb_finish_mmu(struct mmu_gather *tlb,		unsigned long start, unsigned long end, bool force){	struct mmu_gather_batch *batch, *next; 	if (force)		__tlb_adjust_range(tlb, start, end - start); 	tlb_flush_mmu(tlb); 	/* keep the page table cache within bounds */	check_pgt_cache(); 	for (batch = tlb->local.next; batch; batch = next) {		next = batch->next;		free_pages((unsigned long)batch, 0);	}	tlb->local.next = NULL;} /* __tlb_remove_page *	Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while *	handling the additional races in SMP caused by other CPUs caching valid *	mappings in their TLBs. Returns the number of free page slots left. *	When out of page slots we must call tlb_flush_mmu(). *returns true if the caller should flush. */bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size){	struct mmu_gather_batch *batch; 	VM_BUG_ON(!tlb->end);	VM_WARN_ON(tlb->page_size != page_size); 	batch = tlb->active;	/*	 * Add the page and check if we are full. If so	 * force a flush.	 */	batch->pages[batch->nr++] = page;	if (batch->nr == batch->max) {		if (!tlb_next_batch(tlb))			return true;		batch = tlb->active;	}	VM_BUG_ON_PAGE(batch->nr > batch->max, page); 	return false;} #endif /* HAVE_GENERIC_MMU_GATHER */ #ifdef CONFIG_HAVE_RCU_TABLE_FREE /* * See the comment near struct mmu_table_batch. */ static void tlb_remove_table_smp_sync(void *arg){	/* Simply deliver the interrupt */} static void tlb_remove_table_one(void *table){	/*	 * This isn't an RCU grace period and hence the page-tables cannot be	 * assumed to be actually RCU-freed.	 *	 * It is however sufficient for software page-table walkers that rely on	 * IRQ disabling. See the comment near struct mmu_table_batch.	 */	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);	__tlb_remove_table(table);} static void tlb_remove_table_rcu(struct rcu_head *head){	struct mmu_table_batch *batch;	int i; 	batch = container_of(head, struct mmu_table_batch, rcu); 	for (i = 0; i < batch->nr; i++)		__tlb_remove_table(batch->tables[i]); 	free_page((unsigned long)batch);} void tlb_table_flush(struct mmu_gather *tlb){	struct mmu_table_batch **batch = &tlb->batch; 	if (*batch) {		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);		*batch = NULL;	}} void tlb_remove_table(struct mmu_gather *tlb, void *table){	struct mmu_table_batch **batch = &tlb->batch; 	/*	 * When there's less then two users of this mm there cannot be a	 * concurrent page-table walk.	 */	if (atomic_read(&tlb->mm->mm_users) < 2) {		__tlb_remove_table(table);		return;	} 	if (*batch == NULL) {		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);		if (*batch == NULL) {			tlb_remove_table_one(table);			return;		}		(*batch)->nr = 0;	}	(*batch)->tables[(*batch)->nr++] = table;	if ((*batch)->nr == MAX_TABLE_BATCH)		tlb_table_flush(tlb);} #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ /* tlb_gather_mmu *	Called to initialize an (on-stack) mmu_gather structure for page-table *	tear-down from @mm. The @fullmm argument is used when @mm is without *	users and we're going to destroy the full address space (exit/execve). */void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,			unsigned long start, unsigned long end){	arch_tlb_gather_mmu(tlb, mm, start, end);	inc_tlb_flush_pending(tlb->mm);} void tlb_finish_mmu(struct mmu_gather *tlb,		unsigned long start, unsigned long end){	/*	 * If there are parallel threads are doing PTE changes on same range	 * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB	 * flush by batching, a thread has stable TLB entry can fail to flush	 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB	 * forcefully if we detect parallel PTE batching threads.	 */	bool force = mm_tlb_flush_nested(tlb->mm); 	arch_tlb_finish_mmu(tlb, start, end, force);	dec_tlb_flush_pending(tlb->mm);} /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,			   unsigned long addr){	pgtable_t token = pmd_pgtable(*pmd);	pmd_clear(pmd);	pte_free_tlb(tlb, token, addr);	atomic_long_dec(&tlb->mm->nr_ptes);} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd, addr);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd, start);	mm_dec_nr_pmds(tlb->mm);} static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(p4d, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= P4D_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= P4D_MASK;		if (!ceiling)			return;

Lines 1–500 of 4471. Load more