2261 lines · 56.1 KB · C
RawDownload
/* *	linux/mm/filemap.c * * Copyright (C) 1994-1999  Linus Torvalds */ /* * This file handles the generic file mmap semantics used by * most "normal" filesystems (but you don't /have/ to use this: * the NFS filesystem used to do this differently, for example) */#include <linux/config.h>#include <linux/module.h>#include <linux/slab.h>#include <linux/compiler.h>#include <linux/fs.h>#include <linux/aio.h>#include <linux/kernel_stat.h>#include <linux/mm.h>#include <linux/swap.h>#include <linux/mman.h>#include <linux/pagemap.h>#include <linux/file.h>#include <linux/uio.h>#include <linux/hash.h>#include <linux/writeback.h>#include <linux/pagevec.h>#include <linux/blkdev.h>#include <linux/security.h>#include <linux/syscalls.h>#include "filemap.h"/* * FIXME: remove all knowledge of the buffer layer from the core VM */#include <linux/buffer_head.h> /* for generic_osync_inode */ #include <asm/uaccess.h>#include <asm/mman.h> /* * Shared mappings implemented 30.11.1994. It's not fully working yet, * though. * * Shared mappings now work. 15.8.1995  Bruno. * * finished 'unifying' the page and buffer cache and SMP-threaded the * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> * * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> */ /* * Lock ordering: * *  ->i_mmap_lock		(vmtruncate) *    ->private_lock		(__free_pte->__set_page_dirty_buffers) *      ->swap_list_lock *        ->swap_device_lock	(exclusive_swap_page, others) *          ->mapping->tree_lock * *  ->i_sem *    ->i_mmap_lock		(truncate->unmap_mapping_range) * *  ->mmap_sem *    ->i_mmap_lock *      ->page_table_lock	(various places, mainly in mmap.c) *        ->mapping->tree_lock	(arch-dependent flush_dcache_mmap_lock) * *  ->mmap_sem *    ->lock_page		(access_process_vm) * *  ->mmap_sem *    ->i_sem			(msync) * *  ->i_sem *    ->i_alloc_sem             (various) * *  ->inode_lock *    ->sb_lock			(fs/fs-writeback.c) *    ->mapping->tree_lock	(__sync_single_inode) * *  ->i_mmap_lock *    ->anon_vma.lock		(vma_adjust) * *  ->anon_vma.lock *    ->page_table_lock		(anon_vma_prepare and various) * *  ->page_table_lock *    ->swap_device_lock	(try_to_unmap_one) *    ->private_lock		(try_to_unmap_one) *    ->tree_lock		(try_to_unmap_one) *    ->zone.lru_lock		(follow_page->mark_page_accessed) *    ->private_lock		(page_remove_rmap->set_page_dirty) *    ->tree_lock		(page_remove_rmap->set_page_dirty) *    ->inode_lock		(page_remove_rmap->set_page_dirty) *    ->inode_lock		(zap_pte_range->set_page_dirty) *    ->private_lock		(zap_pte_range->__set_page_dirty_buffers) * *  ->task->proc_lock *    ->dcache_lock		(proc_pid_lookup) */ /* * Remove a page from the page cache and free it. Caller has to make * sure the page is locked and that nobody else uses it - or that usage * is safe.  The caller must hold a write_lock on the mapping's tree_lock. */void __remove_from_page_cache(struct page *page){	struct address_space *mapping = page->mapping; 	radix_tree_delete(&mapping->page_tree, page->index);	page->mapping = NULL;	mapping->nrpages--;	pagecache_acct(-1);} void remove_from_page_cache(struct page *page){	struct address_space *mapping = page->mapping; 	BUG_ON(!PageLocked(page)); 	write_lock_irq(&mapping->tree_lock);	__remove_from_page_cache(page);	write_unlock_irq(&mapping->tree_lock);} static int sync_page(void *word){	struct address_space *mapping;	struct page *page; 	page = container_of((page_flags_t *)word, struct page, flags); 	/*	 * page_mapping() is being called without PG_locked held.	 * Some knowledge of the state and use of the page is used to	 * reduce the requirements down to a memory barrier.	 * The danger here is of a stale page_mapping() return value	 * indicating a struct address_space different from the one it's	 * associated with when it is associated with one.	 * After smp_mb(), it's either the correct page_mapping() for	 * the page, or an old page_mapping() and the page's own	 * page_mapping() has gone NULL.	 * The ->sync_page() address_space operation must tolerate	 * page_mapping() going NULL. By an amazing coincidence,	 * this comes about because none of the users of the page	 * in the ->sync_page() methods make essential use of the	 * page_mapping(), merely passing the page down to the backing	 * device's unplug functions when it's non-NULL, which in turn	 * ignore it for all cases but swap, where only page->private is	 * of interest. When page_mapping() does go NULL, the entire	 * call stack gracefully ignores the page and returns.	 * -- wli	 */	smp_mb();	mapping = page_mapping(page);	if (mapping && mapping->a_ops && mapping->a_ops->sync_page)		mapping->a_ops->sync_page(page);	io_schedule();	return 0;} /** * filemap_fdatawrite_range - start writeback against all of a mapping's * dirty pages that lie within the byte offsets <start, end> * @mapping:	address space structure to write * @start:	offset in bytes where the range starts * @end:	offset in bytes where the range ends * @sync_mode:	enable synchronous operation * * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as * opposed to a regular memory * cleansing writeback.  The difference between * these two operations is that if a dirty page/buffer is encountered, it must * be waited upon, and not just skipped over. */static int __filemap_fdatawrite_range(struct address_space *mapping,	loff_t start, loff_t end, int sync_mode){	int ret;	struct writeback_control wbc = {		.sync_mode = sync_mode,		.nr_to_write = mapping->nrpages * 2,		.start = start,		.end = end,	}; 	if (!mapping_cap_writeback_dirty(mapping))		return 0; 	ret = do_writepages(mapping, &wbc);	return ret;} static inline int __filemap_fdatawrite(struct address_space *mapping,	int sync_mode){	return __filemap_fdatawrite_range(mapping, 0, 0, sync_mode);} int filemap_fdatawrite(struct address_space *mapping){	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);}EXPORT_SYMBOL(filemap_fdatawrite); static int filemap_fdatawrite_range(struct address_space *mapping,	loff_t start, loff_t end){	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);} /* * This is a mostly non-blocking flush.  Not suitable for data-integrity * purposes - I/O may not be started against all dirty pages. */int filemap_flush(struct address_space *mapping){	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);}EXPORT_SYMBOL(filemap_flush); /* * Wait for writeback to complete against pages indexed by start->end * inclusive */static int wait_on_page_writeback_range(struct address_space *mapping,				pgoff_t start, pgoff_t end){	struct pagevec pvec;	int nr_pages;	int ret = 0;	pgoff_t index; 	if (end < start)		return 0; 	pagevec_init(&pvec, 0);	index = start;	while ((index <= end) &&			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,			PAGECACHE_TAG_WRITEBACK,			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {		unsigned i; 		for (i = 0; i < nr_pages; i++) {			struct page *page = pvec.pages[i]; 			/* until radix tree lookup accepts end_index */			if (page->index > end)				continue; 			wait_on_page_writeback(page);			if (PageError(page))				ret = -EIO;		}		pagevec_release(&pvec);		cond_resched();	} 	/* Check for outstanding write errors */	if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))		ret = -ENOSPC;	if (test_and_clear_bit(AS_EIO, &mapping->flags))		ret = -EIO; 	return ret;} /* * Write and wait upon all the pages in the passed range.  This is a "data * integrity" operation.  It waits upon in-flight writeout before starting and * waiting upon new writeout.  If there was an IO error, return it. * * We need to re-take i_sem during the generic_osync_inode list walk because * it is otherwise livelockable. */int sync_page_range(struct inode *inode, struct address_space *mapping,			loff_t pos, size_t count){	pgoff_t start = pos >> PAGE_CACHE_SHIFT;	pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;	int ret; 	if (!mapping_cap_writeback_dirty(mapping) || !count)		return 0;	ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);	if (ret == 0) {		down(&inode->i_sem);		ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);		up(&inode->i_sem);	}	if (ret == 0)		ret = wait_on_page_writeback_range(mapping, start, end);	return ret;}EXPORT_SYMBOL(sync_page_range); /* * Note: Holding i_sem across sync_page_range_nolock is not a good idea * as it forces O_SYNC writers to different parts of the same file * to be serialised right until io completion. */int sync_page_range_nolock(struct inode *inode, struct address_space *mapping,			loff_t pos, size_t count){	pgoff_t start = pos >> PAGE_CACHE_SHIFT;	pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;	int ret; 	if (!mapping_cap_writeback_dirty(mapping) || !count)		return 0;	ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);	if (ret == 0)		ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);	if (ret == 0)		ret = wait_on_page_writeback_range(mapping, start, end);	return ret;}EXPORT_SYMBOL(sync_page_range_nolock); /** * filemap_fdatawait - walk the list of under-writeback pages of the given *     address space and wait for all of them. * * @mapping: address space structure to wait for */int filemap_fdatawait(struct address_space *mapping){	loff_t i_size = i_size_read(mapping->host); 	if (i_size == 0)		return 0; 	return wait_on_page_writeback_range(mapping, 0,				(i_size - 1) >> PAGE_CACHE_SHIFT);}EXPORT_SYMBOL(filemap_fdatawait); int filemap_write_and_wait(struct address_space *mapping){	int retval = 0; 	if (mapping->nrpages) {		retval = filemap_fdatawrite(mapping);		if (retval == 0)			retval = filemap_fdatawait(mapping);	}	return retval;} int filemap_write_and_wait_range(struct address_space *mapping,				 loff_t lstart, loff_t lend){	int retval = 0; 	if (mapping->nrpages) {		retval = __filemap_fdatawrite_range(mapping, lstart, lend,						    WB_SYNC_ALL);		if (retval == 0)			retval = wait_on_page_writeback_range(mapping,						    lstart >> PAGE_CACHE_SHIFT,						    lend >> PAGE_CACHE_SHIFT);	}	return retval;} /* * This function is used to add newly allocated pagecache pages: * the page is new, so we can just run SetPageLocked() against it. * The other page state flags were set by rmqueue(). * * This function does not add the page to the LRU.  The caller must do that. */int add_to_page_cache(struct page *page, struct address_space *mapping,		pgoff_t offset, int gfp_mask){	int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM); 	if (error == 0) {		write_lock_irq(&mapping->tree_lock);		error = radix_tree_insert(&mapping->page_tree, offset, page);		if (!error) {			page_cache_get(page);			SetPageLocked(page);			page->mapping = mapping;			page->index = offset;			mapping->nrpages++;			pagecache_acct(1);		}		write_unlock_irq(&mapping->tree_lock);		radix_tree_preload_end();	}	return error;} EXPORT_SYMBOL(add_to_page_cache); int add_to_page_cache_lru(struct page *page, struct address_space *mapping,				pgoff_t offset, int gfp_mask){	int ret = add_to_page_cache(page, mapping, offset, gfp_mask);	if (ret == 0)		lru_cache_add(page);	return ret;} /* * In order to wait for pages to become available there must be * waitqueues associated with pages. By using a hash table of * waitqueues where the bucket discipline is to maintain all * waiters on the same queue and wake all when any of the pages * become available, and for the woken contexts to check to be * sure the appropriate page became available, this saves space * at a cost of "thundering herd" phenomena during rare hash * collisions. */static wait_queue_head_t *page_waitqueue(struct page *page){	const struct zone *zone = page_zone(page); 	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];} static inline void wake_up_page(struct page *page, int bit){	__wake_up_bit(page_waitqueue(page), &page->flags, bit);} void fastcall wait_on_page_bit(struct page *page, int bit_nr){	DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); 	if (test_bit(bit_nr, &page->flags))		__wait_on_bit(page_waitqueue(page), &wait, sync_page,							TASK_UNINTERRUPTIBLE);}EXPORT_SYMBOL(wait_on_page_bit); /** * unlock_page() - unlock a locked page * * @page: the page * * Unlocks the page and wakes up sleepers in ___wait_on_page_locked(). * Also wakes sleepers in wait_on_page_writeback() because the wakeup * mechananism between PageLocked pages and PageWriteback pages is shared. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep. * * The first mb is necessary to safely close the critical section opened by the * TestSetPageLocked(), the second mb is necessary to enforce ordering between * the clear_bit and the read of the waitqueue (to avoid SMP races with a * parallel wait_on_page_locked()). */void fastcall unlock_page(struct page *page){	smp_mb__before_clear_bit();	if (!TestClearPageLocked(page))		BUG();	smp_mb__after_clear_bit(); 	wake_up_page(page, PG_locked);}EXPORT_SYMBOL(unlock_page); /* * End writeback against a page. */void end_page_writeback(struct page *page){	if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {		if (!test_clear_page_writeback(page))			BUG();	}	smp_mb__after_clear_bit();	wake_up_page(page, PG_writeback);}EXPORT_SYMBOL(end_page_writeback); /* * Get a lock on the page, assuming we need to sleep to get it. * * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some * random driver's requestfn sets TASK_RUNNING, we could busywait.  However * chances are that on the second loop, the block layer's plug list is empty, * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. */void fastcall __lock_page(struct page *page){	DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); 	__wait_on_bit_lock(page_waitqueue(page), &wait, sync_page,							TASK_UNINTERRUPTIBLE);}EXPORT_SYMBOL(__lock_page); /* * a rather lightweight function, finding and getting a reference to a * hashed page atomically. */

Lines 1–500 of 2261.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t