/* *	linux/mm/mlock.c * *  (C) Copyright 1995 Linus Torvalds *  (C) Copyright 2002 Christoph Hellwig */ #include <linux/capability.h>#include <linux/mman.h>#include <linux/mm.h>#include <linux/swap.h>#include <linux/swapops.h>#include <linux/pagemap.h>#include <linux/mempolicy.h>#include <linux/syscalls.h>#include <linux/sched.h>#include <linux/export.h>#include <linux/rmap.h>#include <linux/mmzone.h>#include <linux/hugetlb.h> #include "internal.h" int can_do_mlock(void){	if (capable(CAP_IPC_LOCK))		return 1;	if (rlimit(RLIMIT_MEMLOCK) != 0)		return 1;	return 0;}EXPORT_SYMBOL(can_do_mlock); /* * Mlocked pages are marked with PageMlocked() flag for efficient testing * in vmscan and, possibly, the fault path; and to support semi-accurate * statistics. * * An mlocked page [PageMlocked(page)] is unevictable.  As such, it will * be placed on the LRU "unevictable" list, rather than the [in]active lists. * The unevictable list is an LRU sibling list to the [in]active lists. * PageUnevictable is set to indicate the unevictable state. * * When lazy mlocking via vmscan, it is important to ensure that the * vma's VM_LOCKED status is not concurrently being modified, otherwise we * may have mlocked a page that is being munlocked. So lazy mlock must take * the mmap_sem for read, and verify that the vma really is locked * (see mm/rmap.c). */ /* *  LRU accounting for clear_page_mlock() */void __clear_page_mlock(struct page *page){	VM_BUG_ON(!PageLocked(page)); 	if (!page->mapping) {	/* truncated ? */		return;	} 	dec_zone_page_state(page, NR_MLOCK);	count_vm_event(UNEVICTABLE_PGCLEARED);	if (!isolate_lru_page(page)) {		putback_lru_page(page);	} else {		/*		 * We lost the race. the page already moved to evictable list.		 */		if (PageUnevictable(page))			count_vm_event(UNEVICTABLE_PGSTRANDED);	}} /* * Mark page as mlocked if not already. * If page on LRU, isolate and putback to move to unevictable list. */void mlock_vma_page(struct page *page){	BUG_ON(!PageLocked(page)); 	if (!TestSetPageMlocked(page)) {		inc_zone_page_state(page, NR_MLOCK);		count_vm_event(UNEVICTABLE_PGMLOCKED);		if (!isolate_lru_page(page))			putback_lru_page(page);	}} /** * munlock_vma_page - munlock a vma page * @page - page to be unlocked * * called from munlock()/munmap() path with page supposedly on the LRU. * When we munlock a page, because the vma where we found the page is being * munlock()ed or munmap()ed, we want to check whether other vmas hold the * page locked so that we can leave it on the unevictable lru list and not * bother vmscan with it.  However, to walk the page's rmap list in * try_to_munlock() we must isolate the page from the LRU.  If some other * task has removed the page from the LRU, we won't be able to do that. * So we clear the PageMlocked as we might not get another chance.  If we * can't isolate the page, we leave it for putback_lru_page() and vmscan * [page_referenced()/try_to_unmap()] to deal with. */void munlock_vma_page(struct page *page){	BUG_ON(!PageLocked(page)); 	if (TestClearPageMlocked(page)) {		dec_zone_page_state(page, NR_MLOCK);		if (!isolate_lru_page(page)) {			int ret = SWAP_AGAIN; 			/*			 * Optimization: if the page was mapped just once,			 * that's our mapping and we don't need to check all the			 * other vmas.			 */			if (page_mapcount(page) > 1)				ret = try_to_munlock(page);			/*			 * did try_to_unlock() succeed or punt?			 */			if (ret != SWAP_MLOCK)				count_vm_event(UNEVICTABLE_PGMUNLOCKED); 			putback_lru_page(page);		} else {			/*			 * Some other task has removed the page from the LRU.			 * putback_lru_page() will take care of removing the			 * page from the unevictable list, if necessary.			 * vmscan [page_referenced()] will move the page back			 * to the unevictable list if some other vma has it			 * mlocked.			 */			if (PageUnevictable(page))				count_vm_event(UNEVICTABLE_PGSTRANDED);			else				count_vm_event(UNEVICTABLE_PGMUNLOCKED);		}	}} /** * __mlock_vma_pages_range() -  mlock a range of pages in the vma. * @vma:   target vma * @start: start address * @end:   end address * * This takes care of making the pages present too. * * return 0 on success, negative error code on error. * * vma->vm_mm->mmap_sem must be held for at least read. */static long __mlock_vma_pages_range(struct vm_area_struct *vma,				    unsigned long start, unsigned long end,				    int *nonblocking){	struct mm_struct *mm = vma->vm_mm;	unsigned long addr = start;	int nr_pages = (end - start) / PAGE_SIZE;	int gup_flags; 	VM_BUG_ON(start & ~PAGE_MASK);	VM_BUG_ON(end   & ~PAGE_MASK);	VM_BUG_ON(start < vma->vm_start);	VM_BUG_ON(end   > vma->vm_end);	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); 	gup_flags = FOLL_TOUCH | FOLL_MLOCK;	/*	 * We want to touch writable mappings with a write fault in order	 * to break COW, except for shared mappings because these don't COW	 * and we would not want to dirty them for nothing.	 */	if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)		gup_flags |= FOLL_WRITE; 	/*	 * We want mlock to succeed for regions that have any permissions	 * other than PROT_NONE.	 */	if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))		gup_flags |= FOLL_FORCE; 	return __get_user_pages(current, mm, addr, nr_pages, gup_flags,				NULL, NULL, nonblocking);} /* * convert get_user_pages() return value to posix mlock() error */static int __mlock_posix_error_return(long retval){	if (retval == -EFAULT)		retval = -ENOMEM;	else if (retval == -ENOMEM)		retval = -EAGAIN;	return retval;} /** * mlock_vma_pages_range() - mlock pages in specified vma range. * @vma - the vma containing the specfied address range * @start - starting address in @vma to mlock * @end   - end address [+1] in @vma to mlock * * For mmap()/mremap()/expansion of mlocked vma. * * return 0 on success for "normal" vmas. * * return number of pages [> 0] to be removed from locked_vm on success * of "special" vmas. */long mlock_vma_pages_range(struct vm_area_struct *vma,			unsigned long start, unsigned long end){	int nr_pages = (end - start) / PAGE_SIZE;	BUG_ON(!(vma->vm_flags & VM_LOCKED)); 	/*	 * filter unlockable vmas	 */	if (vma->vm_flags & (VM_IO | VM_PFNMAP))		goto no_mlock; 	if (!((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) ||			is_vm_hugetlb_page(vma) ||			vma == get_gate_vma(current->mm))) { 		__mlock_vma_pages_range(vma, start, end, NULL); 		/* Hide errors from mmap() and other callers */		return 0;	} 	/*	 * User mapped kernel pages or huge pages:	 * make these pages present to populate the ptes, but	 * fall thru' to reset VM_LOCKED--no need to unlock, and	 * return nr_pages so these don't get counted against task's	 * locked limit.  huge pages are already counted against	 * locked vm limit.	 */	make_pages_present(start, end); no_mlock:	vma->vm_flags &= ~VM_LOCKED;	/* and don't come back! */	return nr_pages;		/* error or pages NOT mlocked */} /* * munlock_vma_pages_range() - munlock all pages in the vma range.' * @vma - vma containing range to be munlock()ed. * @start - start address in @vma of the range * @end - end of range in @vma. * *  For mremap(), munmap() and exit(). * * Called with @vma VM_LOCKED. * * Returns with VM_LOCKED cleared.  Callers must be prepared to * deal with this. * * We don't save and restore VM_LOCKED here because pages are * still on lru.  In unmap path, pages might be scanned by reclaim * and re-mlocked by try_to_{munlock|unmap} before we unmap and * free them.  This will result in freeing mlocked pages. */void munlock_vma_pages_range(struct vm_area_struct *vma,			     unsigned long start, unsigned long end){	unsigned long addr; 	lru_add_drain();	vma->vm_flags &= ~VM_LOCKED; 	for (addr = start; addr < end; addr += PAGE_SIZE) {		struct page *page;		/*		 * Although FOLL_DUMP is intended for get_dump_page(),		 * it just so happens that its special treatment of the		 * ZERO_PAGE (returning an error instead of doing get_page)		 * suits munlock very well (and if somehow an abnormal page		 * has sneaked into the range, we won't oops here: great).		 */		page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP);		if (page && !IS_ERR(page)) {			lock_page(page);			/*			 * Like in __mlock_vma_pages_range(),			 * because we lock page here and migration is			 * blocked by the elevated reference, we need			 * only check for file-cache page truncation.			 */			if (page->mapping)				munlock_vma_page(page);			unlock_page(page);			put_page(page);		}		cond_resched();	}} /* * mlock_fixup  - handle mlock[all]/munlock[all] requests. * * Filters out "special" vmas -- VM_LOCKED never gets set for these, and * munlock is a no-op.  However, for some special vmas, we go ahead and * populate the ptes via make_pages_present(). * * For vmas that pass the filters, merge/split as appropriate. */static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,	unsigned long start, unsigned long end, vm_flags_t newflags){	struct mm_struct *mm = vma->vm_mm;	pgoff_t pgoff;	int nr_pages;	int ret = 0;	int lock = !!(newflags & VM_LOCKED); 	if (newflags == vma->vm_flags || (vma->vm_flags & VM_SPECIAL) ||	    is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm))		goto out;	/* don't set VM_LOCKED,  don't count */ 	pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);	*prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,			  vma->vm_file, pgoff, vma_policy(vma));	if (*prev) {		vma = *prev;		goto success;	} 	if (start != vma->vm_start) {		ret = split_vma(mm, vma, start, 1);		if (ret)			goto out;	} 	if (end != vma->vm_end) {		ret = split_vma(mm, vma, end, 0);		if (ret)			goto out;	} success:	/*	 * Keep track of amount of locked VM.	 */	nr_pages = (end - start) >> PAGE_SHIFT;	if (!lock)		nr_pages = -nr_pages;	mm->locked_vm += nr_pages; 	/*	 * vm_flags is protected by the mmap_sem held in write mode.	 * It's okay if try_to_unmap_one unmaps a page just after we	 * set VM_LOCKED, __mlock_vma_pages_range will bring it back.	 */ 	if (lock)		vma->vm_flags = newflags;	else		munlock_vma_pages_range(vma, start, end); out:	*prev = vma;	return ret;} static int do_mlock(unsigned long start, size_t len, int on){	unsigned long nstart, end, tmp;	struct vm_area_struct * vma, * prev;	int error; 	VM_BUG_ON(start & ~PAGE_MASK);	VM_BUG_ON(len != PAGE_ALIGN(len));	end = start + len;	if (end < start)		return -EINVAL;	if (end == start)		return 0;	vma = find_vma(current->mm, start);	if (!vma || vma->vm_start > start)		return -ENOMEM; 	prev = vma->vm_prev;	if (start > vma->vm_start)		prev = vma; 	for (nstart = start ; ; ) {		vm_flags_t newflags; 		/* Here we know that  vma->vm_start <= nstart < vma->vm_end. */ 		newflags = vma->vm_flags | VM_LOCKED;		if (!on)			newflags &= ~VM_LOCKED; 		tmp = vma->vm_end;		if (tmp > end)			tmp = end;		error = mlock_fixup(vma, &prev, nstart, tmp, newflags);		if (error)			break;		nstart = tmp;		if (nstart < prev->vm_end)			nstart = prev->vm_end;		if (nstart >= end)			break; 		vma = prev->vm_next;		if (!vma || vma->vm_start != nstart) {			error = -ENOMEM;			break;		}	}	return error;} static int do_mlock_pages(unsigned long start, size_t len, int ignore_errors){	struct mm_struct *mm = current->mm;	unsigned long end, nstart, nend;	struct vm_area_struct *vma = NULL;	int locked = 0;	int ret = 0; 	VM_BUG_ON(start & ~PAGE_MASK);	VM_BUG_ON(len != PAGE_ALIGN(len));	end = start + len; 	for (nstart = start; nstart < end; nstart = nend) {		/*		 * We want to fault in pages for [nstart; end) address range.		 * Find first corresponding VMA.		 */		if (!locked) {			locked = 1;			down_read(&mm->mmap_sem);			vma = find_vma(mm, nstart);		} else if (nstart >= vma->vm_end)			vma = vma->vm_next;		if (!vma || vma->vm_start >= end)			break;		/*		 * Set [nstart; nend) to intersection of desired address		 * range with the first VMA. Also, skip undesirable VMA types.		 */		nend = min(end, vma->vm_end);		if (vma->vm_flags & (VM_IO | VM_PFNMAP))			continue;		if (nstart < vma->vm_start)			nstart = vma->vm_start;		/*		 * Now fault in a range of pages. __mlock_vma_pages_range()		 * double checks the vma flags, so that it won't mlock pages		 * if the vma was already munlocked.		 */		ret = __mlock_vma_pages_range(vma, nstart, nend, &locked);		if (ret < 0) {			if (ignore_errors) {				ret = 0;				continue;	/* continue at next VMA */			}			ret = __mlock_posix_error_return(ret);			break;		}		nend = nstart + ret * PAGE_SIZE;		ret = 0;	}	if (locked)		up_read(&mm->mmap_sem);	return ret;	/* 0 or negative error code */} SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len){	unsigned long locked;	unsigned long lock_limit;	int error = -ENOMEM; 	if (!can_do_mlock())		return -EPERM; 	lru_add_drain_all();	/* flush pagevec */ 	down_write(&current->mm->mmap_sem);	len = PAGE_ALIGN(len + (start & ~PAGE_MASK));	start &= PAGE_MASK; 	locked = len >> PAGE_SHIFT;	locked += current->mm->locked_vm; 	lock_limit = rlimit(RLIMIT_MEMLOCK);

Lines 1–500 of 626. Load more