/* *  linux/mm/memory.c * *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds */ /* * demand-loading started 01.12.91 - seems it is high on the list of * things wanted, and it should be easy to implement. - Linus */ /* * Ok, demand-loading was easy, shared pages a little bit tricker. Shared * pages started 02.12.91, seems to work. - Linus. * * Tested sharing by executing about 30 /bin/sh: under the old kernel it * would have taken more than the 6M I have free, but it worked well as * far as I could see. * * Also corrected some "invalidate()"s - I wasn't doing enough of them. */ /* * Real VM (paging to/from disk) started 18.12.91. Much more work and * thought has to go into this. Oh, well.. * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why. *		Found it. Everything seems to work now. * 20.12.91  -  Ok, making the swap-device changeable like the root. */ /* * 05.04.94  -  Multi-page memory management added for v1.1. * 		Idea by Alex Bligh (alex@cconcepts.co.uk) * * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG *		(Gerhard.Wichert@pdb.siemens.de) * * Aug/Sep 2004 Changed to four level page tables (Andi Kleen) */ #include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/hugetlb.h>#include <linux/mman.h>#include <linux/swap.h>#include <linux/highmem.h>#include <linux/pagemap.h>#include <linux/rmap.h>#include <linux/module.h>#include <linux/init.h> #include <asm/pgalloc.h>#include <asm/uaccess.h>#include <asm/tlb.h>#include <asm/tlbflush.h>#include <asm/pgtable.h> #include <linux/swapops.h>#include <linux/elf.h> #ifndef CONFIG_NEED_MULTIPLE_NODES/* use the per-pgdat data instead for discontigmem - mbligh */unsigned long max_mapnr;struct page *mem_map; EXPORT_SYMBOL(max_mapnr);EXPORT_SYMBOL(mem_map);#endif unsigned long num_physpages;/* * A number of key systems in x86 including ioremap() rely on the assumption * that high_memory defines the upper bound on direct map memory, then end * of ZONE_NORMAL.  Under CONFIG_DISCONTIG this means that max_low_pfn and * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL * and ZONE_HIGHMEM. */void * high_memory;unsigned long vmalloc_earlyreserve; EXPORT_SYMBOL(num_physpages);EXPORT_SYMBOL(high_memory);EXPORT_SYMBOL(vmalloc_earlyreserve); /* * If a p?d_bad entry is found while walking page tables, report * the error, before resetting entry to p?d_none.  Usually (but * very seldom) called out from the p?d_none_or_clear_bad macros. */ void pgd_clear_bad(pgd_t *pgd){	pgd_ERROR(*pgd);	pgd_clear(pgd);} void pud_clear_bad(pud_t *pud){	pud_ERROR(*pud);	pud_clear(pud);} void pmd_clear_bad(pmd_t *pmd){	pmd_ERROR(*pmd);	pmd_clear(pmd);} /* * Note: this doesn't free the actual pages themselves. That * has been handled earlier when unmapping all the memory regions. */static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd){	struct page *page = pmd_page(*pmd);	pmd_clear(pmd);	pte_free_tlb(tlb, page);	dec_page_state(nr_page_table_pages);	tlb->mm->nr_ptes--;} static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pmd_t *pmd;	unsigned long next;	unsigned long start; 	start = addr;	pmd = pmd_offset(pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(pmd))			continue;		free_pte_range(tlb, pmd);	} while (pmd++, addr = next, addr != end); 	start &= PUD_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PUD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pmd = pmd_offset(pud, start);	pud_clear(pud);	pmd_free_tlb(tlb, pmd);} static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,				unsigned long addr, unsigned long end,				unsigned long floor, unsigned long ceiling){	pud_t *pud;	unsigned long next;	unsigned long start; 	start = addr;	pud = pud_offset(pgd, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(pud))			continue;		free_pmd_range(tlb, pud, addr, next, floor, ceiling);	} while (pud++, addr = next, addr != end); 	start &= PGDIR_MASK;	if (start < floor)		return;	if (ceiling) {		ceiling &= PGDIR_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		return; 	pud = pud_offset(pgd, start);	pgd_clear(pgd);	pud_free_tlb(tlb, pud);} /* * This function frees user-level page tables of a process. * * Must be called with pagetable lock held. */void free_pgd_range(struct mmu_gather **tlb,			unsigned long addr, unsigned long end,			unsigned long floor, unsigned long ceiling){	pgd_t *pgd;	unsigned long next;	unsigned long start; 	/*	 * The next few lines have given us lots of grief...	 *	 * Why are we testing PMD* at this top level?  Because often	 * there will be no work to do at all, and we'd prefer not to	 * go all the way down to the bottom just to discover that.	 *	 * Why all these "- 1"s?  Because 0 represents both the bottom	 * of the address space and the top of it (using -1 for the	 * top wouldn't help much: the masks would do the wrong thing).	 * The rule is that addr 0 and floor 0 refer to the bottom of	 * the address space, but end 0 and ceiling 0 refer to the top	 * Comparisons need to use "end - 1" and "ceiling - 1" (though	 * that end 0 case should be mythical).	 *	 * Wherever addr is brought up or ceiling brought down, we must	 * be careful to reject "the opposite 0" before it confuses the	 * subsequent tests.  But what about where end is brought down	 * by PMD_SIZE below? no, end can't go down to 0 there.	 *	 * Whereas we round start (addr) and ceiling down, by different	 * masks at different levels, in order to test whether a table	 * now has no other vmas using it, so can be freed, we don't	 * bother to round floor or end up - the tests don't need that.	 */ 	addr &= PMD_MASK;	if (addr < floor) {		addr += PMD_SIZE;		if (!addr)			return;	}	if (ceiling) {		ceiling &= PMD_MASK;		if (!ceiling)			return;	}	if (end - 1 > ceiling - 1)		end -= PMD_SIZE;	if (addr > end - 1)		return; 	start = addr;	pgd = pgd_offset((*tlb)->mm, addr);	do {		next = pgd_addr_end(addr, end);		if (pgd_none_or_clear_bad(pgd))			continue;		free_pud_range(*tlb, pgd, addr, next, floor, ceiling);	} while (pgd++, addr = next, addr != end); 	if (!tlb_is_full_mm(*tlb))		flush_tlb_pgtables((*tlb)->mm, start, end);} void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,		unsigned long floor, unsigned long ceiling){	while (vma) {		struct vm_area_struct *next = vma->vm_next;		unsigned long addr = vma->vm_start; 		if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) {			hugetlb_free_pgd_range(tlb, addr, vma->vm_end,				floor, next? next->vm_start: ceiling);		} else {			/*			 * Optimization: gather nearby vmas into one call down			 */			while (next && next->vm_start <= vma->vm_end + PMD_SIZE			  && !is_hugepage_only_range(vma->vm_mm, next->vm_start,							HPAGE_SIZE)) {				vma = next;				next = vma->vm_next;			}			free_pgd_range(tlb, addr, vma->vm_end,				floor, next? next->vm_start: ceiling);		}		vma = next;	}} pte_t fastcall *pte_alloc_map(struct mm_struct *mm, pmd_t *pmd,				unsigned long address){	if (!pmd_present(*pmd)) {		struct page *new; 		spin_unlock(&mm->page_table_lock);		new = pte_alloc_one(mm, address);		spin_lock(&mm->page_table_lock);		if (!new)			return NULL;		/*		 * Because we dropped the lock, we should re-check the		 * entry, as somebody else could have populated it..		 */		if (pmd_present(*pmd)) {			pte_free(new);			goto out;		}		mm->nr_ptes++;		inc_page_state(nr_page_table_pages);		pmd_populate(mm, pmd, new);	}out:	return pte_offset_map(pmd, address);} pte_t fastcall * pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsigned long address){	if (!pmd_present(*pmd)) {		pte_t *new; 		spin_unlock(&mm->page_table_lock);		new = pte_alloc_one_kernel(mm, address);		spin_lock(&mm->page_table_lock);		if (!new)			return NULL; 		/*		 * Because we dropped the lock, we should re-check the		 * entry, as somebody else could have populated it..		 */		if (pmd_present(*pmd)) {			pte_free_kernel(new);			goto out;		}		pmd_populate_kernel(mm, pmd, new);	}out:	return pte_offset_kernel(pmd, address);} /* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range * covered by this vma. * * dst->page_table_lock is held on entry and exit, * but may be dropped within p[mg]d_alloc() and pte_alloc_map(). */ static inline voidcopy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pte_t *dst_pte, pte_t *src_pte, unsigned long vm_flags,		unsigned long addr){	pte_t pte = *src_pte;	struct page *page;	unsigned long pfn; 	/* pte contains position in swap or file, so copy. */	if (unlikely(!pte_present(pte))) {		if (!pte_file(pte)) {			swap_duplicate(pte_to_swp_entry(pte));			/* make sure dst_mm is on swapoff's mmlist. */			if (unlikely(list_empty(&dst_mm->mmlist))) {				spin_lock(&mmlist_lock);				list_add(&dst_mm->mmlist, &src_mm->mmlist);				spin_unlock(&mmlist_lock);			}		}		set_pte_at(dst_mm, addr, dst_pte, pte);		return;	} 	pfn = pte_pfn(pte);	/* the pte points outside of valid memory, the	 * mapping is assumed to be good, meaningful	 * and not mapped via rmap - duplicate the	 * mapping as is.	 */	page = NULL;	if (pfn_valid(pfn))		page = pfn_to_page(pfn); 	if (!page || PageReserved(page)) {		set_pte_at(dst_mm, addr, dst_pte, pte);		return;	} 	/*	 * If it's a COW mapping, write protect it both	 * in the parent and the child	 */	if ((vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE) {		ptep_set_wrprotect(src_mm, addr, src_pte);		pte = *src_pte;	} 	/*	 * If it's a shared mapping, mark it clean in	 * the child	 */	if (vm_flags & VM_SHARED)		pte = pte_mkclean(pte);	pte = pte_mkold(pte);	get_page(page);	inc_mm_counter(dst_mm, rss);	if (PageAnon(page))		inc_mm_counter(dst_mm, anon_rss);	set_pte_at(dst_mm, addr, dst_pte, pte);	page_dup_rmap(page);} static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,		unsigned long addr, unsigned long end){	pte_t *src_pte, *dst_pte;	unsigned long vm_flags = vma->vm_flags;	int progress; again:	dst_pte = pte_alloc_map(dst_mm, dst_pmd, addr);	if (!dst_pte)		return -ENOMEM;	src_pte = pte_offset_map_nested(src_pmd, addr); 	progress = 0;	spin_lock(&src_mm->page_table_lock);	do {		/*		 * We are holding two locks at this point - either of them		 * could generate latencies in another task on another CPU.		 */		if (progress >= 32 && (need_resched() ||		    need_lockbreak(&src_mm->page_table_lock) ||		    need_lockbreak(&dst_mm->page_table_lock)))			break;		if (pte_none(*src_pte)) {			progress++;			continue;		}		copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vm_flags, addr);		progress += 8;	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);	spin_unlock(&src_mm->page_table_lock); 	pte_unmap_nested(src_pte - 1);	pte_unmap(dst_pte - 1);	cond_resched_lock(&dst_mm->page_table_lock);	if (addr != end)		goto again;	return 0;} static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,		unsigned long addr, unsigned long end){	pmd_t *src_pmd, *dst_pmd;	unsigned long next; 	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);	if (!dst_pmd)		return -ENOMEM;	src_pmd = pmd_offset(src_pud, addr);	do {		next = pmd_addr_end(addr, end);		if (pmd_none_or_clear_bad(src_pmd))			continue;		if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,						vma, addr, next))			return -ENOMEM;	} while (dst_pmd++, src_pmd++, addr = next, addr != end);	return 0;} static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,		pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,		unsigned long addr, unsigned long end){	pud_t *src_pud, *dst_pud;	unsigned long next; 	dst_pud = pud_alloc(dst_mm, dst_pgd, addr);	if (!dst_pud)		return -ENOMEM;	src_pud = pud_offset(src_pgd, addr);	do {		next = pud_addr_end(addr, end);		if (pud_none_or_clear_bad(src_pud))			continue;		if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,						vma, addr, next))			return -ENOMEM;	} while (dst_pud++, src_pud++, addr = next, addr != end);	return 0;} int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,		struct vm_area_struct *vma){	pgd_t *src_pgd, *dst_pgd;	unsigned long next;	unsigned long addr = vma->vm_start;	unsigned long end = vma->vm_end; 

Lines 1–500 of 2248. Load more