/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. * 		Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/rmap.h>#include <linux/module.h>#include <linux/delayacct.h>#include <linux/init.h>#include <linux/writeback.h> #include <asm/pgalloc.h>#include <asm/uaccess.h>#include <asm/tlb.h>#include <asm/tlbflush.h>#include <asm/pgtable.h> #include <linux/swapops.h>#include <linux/elf.h> #ifndef CONFIG_NEED_MULTIPLE_NODES/* use the per-pgdat data instead for discontigmem - mbligh */unsigned long max_mapnr;struct page *mem_map; EXPORT_SYMBOL(max_mapnr);EXPORT_SYMBOL(mem_map);#endif unsigned long num_physpages;/* * A number of key systems in x86 including ioremap() rely on the assumption * that high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL * and ZONE_HIGHMEM. */void * high_memory;unsigned long vmalloc_earlyreserve; EXPORT_SYMBOL(num_physpages);EXPORT_SYMBOL(high_memory);EXPORT_SYMBOL(vmalloc_earlyreserve); int randomize_va_space __read_mostly = 1; static int __init disable_randmaps(char *s){	randomize_va_space = 0;	return 1;}__setup("norandmaps", disable_randmaps);  /* * If a p?d_bad entry is found while walking page tables, report * the error, before resetting entry to p?d_none.  Usually (but * very seldom) called out from the p?d_none_or_clear_bad macros. */ void pgd_clear_bad(pgd_t *pgd){	pgd_ERROR(*pgd);	pgd_clear(pgd);} void pud_clear_bad(pud_t *pud){	pud_ERROR(*pud);	pud_clear(pud);} void pmd_clear_bad(pmd_t *pmd){	pmd_ERROR(*pmd);	pmd_clear(pmd);} /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd){	struct page *page = pmd_page(*pmd);	pmd_clear(pmd);	pte_lock_deinit(page);	pte_free_tlb(tlb, page);	dec_zone_page_state(page, NR_PAGETABLE);	tlb->mm->nr_ptes--;} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd);} static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(pgd, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= PGDIR_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PGDIR_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pud = pud_offset(pgd, start);	pgd_clear(pgd);	pud_free_tlb(tlb, pud);} /* * This function frees user-level page tables of a process. * * Must be called with pagetable lock held. */void free_pgd_range(struct mmu_gather **tlb,			unsigned long addr, unsigned long end,			unsigned long floor, unsigned long ceiling){	pgd_t *pgd;	unsigned long next;	unsigned long start; 	/*	 * The next few lines have given us lots of grief...	 *	 * Why are we testing PMD* at this top level?  Because often	 * there will be no work to do at all, and we'd prefer not to	 * go all the way down to the bottom just to discover that.	 *	 * Why all these "- 1"s?  Because 0 represents both the bottom	 * of the address space and the top of it (using -1 for the	 * top wouldn't help much: the masks would do the wrong thing).	 * The rule is that addr 0 and floor 0 refer to the bottom of	 * the address space, but end 0 and ceiling 0 refer to the top	 * Comparisons need to use "end - 1" and "ceiling - 1" (though	 * that end 0 case should be mythical).	 *	 * Wherever addr is brought up or ceiling brought down, we must	 * be careful to reject "the opposite 0" before it confuses the	 * subsequent tests.  But what about where end is brought down	 * by PMD_SIZE below? no, end can't go down to 0 there.	 *	 * Whereas we round start (addr) and ceiling down, by different	 * masks at different levels, in order to test whether a table	 * now has no other vmas using it, so can be freed, we don't	 * bother to round floor or end up - the tests don't need that.	 */ 	addr &= PMD_MASK;	if (addr < floor) {		addr += PMD_SIZE;		if (!addr)			return;	}	if (ceiling) {		ceiling &= PMD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		end -= PMD_SIZE;	if (addr > end - 1)		return; 	start = addr;	pgd = pgd_offset((*tlb)->mm, addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		free_pud_range(*tlb, pgd, addr, next, floor, ceiling);	} while (pgd++, addr = next, addr != end); 	if (!(*tlb)->fullmm)		flush_tlb_pgtables((*tlb)->mm, start, end);} void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,		unsigned long floor, unsigned long ceiling){	while (vma) {		struct vm_area_struct *next = vma->vm_next;		unsigned long addr = vma->vm_start; 		/*		 * Hide vma from rmap and vmtruncate before freeing pgtables		 */		anon_vma_unlink(vma);		unlink_file_vma(vma); 		if (is_vm_hugetlb_page(vma)) {			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,				floor, next? next->vm_start: ceiling);		} else {			/*			 * Optimization: gather nearby vmas into one call down			 */			while (next && next->vm_start <= vma->vm_end + PMD_SIZE			       && !is_vm_hugetlb_page(next)) {				vma = next;				next = vma->vm_next;				anon_vma_unlink(vma);				unlink_file_vma(vma);			}			free_pgd_range(tlb, addr, vma->vm_end,				floor, next? next->vm_start: ceiling);		}		vma = next;	}} int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address){	struct page *new = pte_alloc_one(mm, address);	if (!new)		return -ENOMEM; 	pte_lock_init(new);	spin_lock(&mm->page_table_lock);	if (pmd_present(*pmd)) {	/* Another has populated it */		pte_lock_deinit(new);		pte_free(new);	} else {		mm->nr_ptes++;		inc_zone_page_state(new, NR_PAGETABLE);		pmd_populate(mm, pmd, new);	}	spin_unlock(&mm->page_table_lock);	return 0;} int __pte_alloc_kernel(pmd_t *pmd, unsigned long address){	pte_t *new = pte_alloc_one_kernel(&init_mm, address);	if (!new)		return -ENOMEM; 	spin_lock(&init_mm.page_table_lock);	if (pmd_present(*pmd))		/* Another has populated it */		pte_free_kernel(new);	else		pmd_populate_kernel(&init_mm, pmd, new);	spin_unlock(&init_mm.page_table_lock);	return 0;} static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss){	if (file_rss)		add_mm_counter(mm, file_rss, file_rss);	if (anon_rss)		add_mm_counter(mm, anon_rss, anon_rss);} /* * This function is called to print an error when a bad pte * is found. For example, we might have a PFN-mapped pte in * a region that doesn't allow it. * * The calling function must still handle the error. */void print_bad_pte(struct vm_area_struct *vma, pte_t pte, unsigned long vaddr){	printk(KERN_ERR "Bad pte = %08llx, process = %s, "			"vm_flags = %lx, vaddr = %lx\n",		(long long)pte_val(pte),		(vma->vm_mm == current->mm ? current->comm : "???"),		vma->vm_flags, vaddr);	dump_stack();} static inline int is_cow_mapping(unsigned int flags){	return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;} /* * This function gets the "struct page" associated with a pte. * * NOTE! Some mappings do not have "struct pages". A raw PFN mapping * will have each page table entry just pointing to a raw page frame * number, and as far as the VM layer is concerned, those do not have * pages associated with them - even if the PFN might point to memory * that otherwise is perfectly fine and has a "struct page". * * The way we recognize those mappings is through the rules set up * by "remap_pfn_range()": the vma will have the VM_PFNMAP bit set, * and the vm_pgoff will point to the first PFN mapped: thus every * page that is a raw mapping will always honor the rule * *	pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT) * * and if that isn't true, the page has been COW'ed (in which case it * _does_ have a "struct page" associated with it even if it is in a * VM_PFNMAP range). */struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte){	unsigned long pfn = pte_pfn(pte); 	if (unlikely(vma->vm_flags & VM_PFNMAP)) {		unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT;		if (pfn == vma->vm_pgoff + off)			return NULL;		if (!is_cow_mapping(vma->vm_flags))			return NULL;	} 	/*	 * Add some anal sanity checks for now. Eventually,	 * we should just do "return pfn_to_page(pfn)", but	 * in the meantime we check that we get a valid pfn,	 * and that the resulting page looks ok.	 */	if (unlikely(!pfn_valid(pfn))) {		print_bad_pte(vma, pte, addr);		return NULL;	} 	/*	 * NOTE! We still have PageReserved() pages in the page 	 * tables. 	 *	 * The PAGE_ZERO() pages and various VDSO mappings can	 * cause them to exist.	 */	return pfn_to_page(pfn);} /* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range * covered by this vma. */ static inline voidcopy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,		unsigned long addr, int *rss){	unsigned long vm_flags = vma->vm_flags;	pte_t pte = *src_pte;	struct page *page; 	/* pte contains position in swap or file, so copy. */	if (unlikely(!pte_present(pte))) {		if (!pte_file(pte)) {			swp_entry_t entry = pte_to_swp_entry(pte); 			swap_duplicate(entry);			/* make sure dst_mm is on swapoff's mmlist. */			if (unlikely(list_empty(&dst_mm->mmlist))) {				spin_lock(&mmlist_lock);				if (list_empty(&dst_mm->mmlist))					list_add(&dst_mm->mmlist,						 &src_mm->mmlist);				spin_unlock(&mmlist_lock);			}			if (is_write_migration_entry(entry) &&					is_cow_mapping(vm_flags)) {				/*				 * COW mappings require pages in both parent				 * and child to be set to read.				 */				make_migration_entry_read(&entry);				pte = swp_entry_to_pte(entry);				set_pte_at(src_mm, addr, src_pte, pte);			}		}		goto out_set_pte;	} 	/*	 * If it's a COW mapping, write protect it both	 * in the parent and the child	 */	if (is_cow_mapping(vm_flags)) {		ptep_set_wrprotect(src_mm, addr, src_pte);		pte = pte_wrprotect(pte);	} 	/*	 * If it's a shared mapping, mark it clean in	 * the child	 */	if (vm_flags & VM_SHARED)		pte = pte_mkclean(pte);	pte = pte_mkold(pte); 	page = vm_normal_page(vma, addr, pte);	if (page) {		get_page(page);		page_dup_rmap(page);		rss[!!PageAnon(page)]++;	} out_set_pte:	set_pte_at(dst_mm, addr, dst_pte, pte);} static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,		unsigned long addr, unsigned long end){	pte_t *src_pte, *dst_pte;	spinlock_t *src_ptl, *dst_ptl;	int progress = 0;	int rss[2]; 

Lines 1–500 of 2738. Load more