3440 lines · 87.7 KB · C
RawDownload
/* * fs/dcache.c * * Complete reimplementation * (C) 1997 Thomas Schoebel-Theuer, * with heavy changes by Linus Torvalds */ /* * Notes on the allocation strategy: * * The dcache is a master of the icache - whenever a dcache entry * exists, the inode will always exist. "iput()" is done either when * the dcache entry is deleted or garbage collected. */ #include <linux/syscalls.h>#include <linux/string.h>#include <linux/mm.h>#include <linux/fs.h>#include <linux/fsnotify.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/hash.h>#include <linux/cache.h>#include <linux/export.h>#include <linux/mount.h>#include <linux/file.h>#include <asm/uaccess.h>#include <linux/security.h>#include <linux/seqlock.h>#include <linux/swap.h>#include <linux/bootmem.h>#include <linux/fs_struct.h>#include <linux/hardirq.h>#include <linux/bit_spinlock.h>#include <linux/rculist_bl.h>#include <linux/prefetch.h>#include <linux/ratelimit.h>#include <linux/list_lru.h>#include "internal.h"#include "mount.h" /* * Usage: * dcache->d_inode->i_lock protects: *   - i_dentry, d_alias, d_inode of aliases * dcache_hash_bucket lock protects: *   - the dcache hash table * s_anon bl list spinlock protects: *   - the s_anon list (see __d_drop) * dentry->d_sb->s_dentry_lru_lock protects: *   - the dcache lru lists and counters * d_lock protects: *   - d_flags *   - d_name *   - d_lru *   - d_count *   - d_unhashed() *   - d_parent and d_subdirs *   - childrens' d_child and d_parent *   - d_alias, d_inode * * Ordering: * dentry->d_inode->i_lock *   dentry->d_lock *     dentry->d_sb->s_dentry_lru_lock *     dcache_hash_bucket lock *     s_anon lock * * If there is an ancestor relationship: * dentry->d_parent->...->d_parent->d_lock *   ... *     dentry->d_parent->d_lock *       dentry->d_lock * * If no ancestor relationship: * if (dentry1 < dentry2) *   dentry1->d_lock *     dentry2->d_lock */int sysctl_vfs_cache_pressure __read_mostly = 100;EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); EXPORT_SYMBOL(rename_lock); static struct kmem_cache *dentry_cache __read_mostly; /* * This is the single most critical data structure when it comes * to the dcache: the hashtable for lookups. Somebody should try * to make this good - I've just made it work. * * This hash-function tries to avoid losing too many bits of hash * information, yet avoid using a prime hash-size or similar. */ static unsigned int d_hash_mask __read_mostly;static unsigned int d_hash_shift __read_mostly; static struct hlist_bl_head *dentry_hashtable __read_mostly; static inline struct hlist_bl_head *d_hash(const struct dentry *parent,					unsigned int hash){	hash += (unsigned long) parent / L1_CACHE_BYTES;	hash = hash + (hash >> d_hash_shift);	return dentry_hashtable + (hash & d_hash_mask);} /* Statistics gathering. */struct dentry_stat_t dentry_stat = {	.age_limit = 45,}; static DEFINE_PER_CPU(long, nr_dentry);static DEFINE_PER_CPU(long, nr_dentry_unused); #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) /* * Here we resort to our own counters instead of using generic per-cpu counters * for consistency with what the vfs inode code does. We are expected to harvest * better code and performance by having our own specialized counters. * * Please note that the loop is done over all possible CPUs, not over all online * CPUs. The reason for this is that we don't want to play games with CPUs going * on and off. If one of them goes off, we will just keep their counters. * * glommer: See cffbc8a for details, and if you ever intend to change this, * please update all vfs counters to match. */static long get_nr_dentry(void){	int i;	long sum = 0;	for_each_possible_cpu(i)		sum += per_cpu(nr_dentry, i);	return sum < 0 ? 0 : sum;} static long get_nr_dentry_unused(void){	int i;	long sum = 0;	for_each_possible_cpu(i)		sum += per_cpu(nr_dentry_unused, i);	return sum < 0 ? 0 : sum;} int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,		   size_t *lenp, loff_t *ppos){	dentry_stat.nr_dentry = get_nr_dentry();	dentry_stat.nr_unused = get_nr_dentry_unused();	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);}#endif /* * Compare 2 name strings, return 0 if they match, otherwise non-zero. * The strings are both count bytes long, and count is non-zero. */#ifdef CONFIG_DCACHE_WORD_ACCESS #include <asm/word-at-a-time.h>/* * NOTE! 'cs' and 'scount' come from a dentry, so it has a * aligned allocation for this particular component. We don't * strictly need the load_unaligned_zeropad() safety, but it * doesn't hurt either. * * In contrast, 'ct' and 'tcount' can be from a pathname, and do * need the careful unaligned handling. */static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount){	unsigned long a,b,mask; 	for (;;) {		a = *(unsigned long *)cs;		b = load_unaligned_zeropad(ct);		if (tcount < sizeof(unsigned long))			break;		if (unlikely(a != b))			return 1;		cs += sizeof(unsigned long);		ct += sizeof(unsigned long);		tcount -= sizeof(unsigned long);		if (!tcount)			return 0;	}	mask = bytemask_from_count(tcount);	return unlikely(!!((a ^ b) & mask));} #else static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount){	do {		if (*cs != *ct)			return 1;		cs++;		ct++;		tcount--;	} while (tcount);	return 0;} #endif static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount){	const unsigned char *cs;	/*	 * Be careful about RCU walk racing with rename:	 * use ACCESS_ONCE to fetch the name pointer.	 *	 * NOTE! Even if a rename will mean that the length	 * was not loaded atomically, we don't care. The	 * RCU walk will check the sequence count eventually,	 * and catch it. And we won't overrun the buffer,	 * because we're reading the name pointer atomically,	 * and a dentry name is guaranteed to be properly	 * terminated with a NUL byte.	 *	 * End result: even if 'len' is wrong, we'll exit	 * early because the data cannot match (there can	 * be no NUL in the ct/tcount data)	 */	cs = ACCESS_ONCE(dentry->d_name.name);	smp_read_barrier_depends();	return dentry_string_cmp(cs, ct, tcount);} static void __d_free(struct rcu_head *head){	struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); 	WARN_ON(!hlist_unhashed(&dentry->d_alias));	if (dname_external(dentry))		kfree(dentry->d_name.name);	kmem_cache_free(dentry_cache, dentry); } /* * no locks, please. */static void d_free(struct dentry *dentry){	BUG_ON((int)dentry->d_lockref.count > 0);	this_cpu_dec(nr_dentry);	if (dentry->d_op && dentry->d_op->d_release)		dentry->d_op->d_release(dentry); 	/* if dentry was never visible to RCU, immediate free is OK */	if (!(dentry->d_flags & DCACHE_RCUACCESS))		__d_free(&dentry->d_u.d_rcu);	else		call_rcu(&dentry->d_u.d_rcu, __d_free);} /** * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups * @dentry: the target dentry * After this call, in-progress rcu-walk path lookup will fail. This * should be called after unhashing, and after changing d_inode (if * the dentry has not already been unhashed). */static inline void dentry_rcuwalk_barrier(struct dentry *dentry){	assert_spin_locked(&dentry->d_lock);	/* Go through a barrier */	write_seqcount_barrier(&dentry->d_seq);} /* * Release the dentry's inode, using the filesystem * d_iput() operation if defined. Dentry has no refcount * and is unhashed. */static void dentry_iput(struct dentry * dentry)	__releases(dentry->d_lock)	__releases(dentry->d_inode->i_lock){	struct inode *inode = dentry->d_inode;	if (inode) {		dentry->d_inode = NULL;		hlist_del_init(&dentry->d_alias);		spin_unlock(&dentry->d_lock);		spin_unlock(&inode->i_lock);		if (!inode->i_nlink)			fsnotify_inoderemove(inode);		if (dentry->d_op && dentry->d_op->d_iput)			dentry->d_op->d_iput(dentry, inode);		else			iput(inode);	} else {		spin_unlock(&dentry->d_lock);	}} /* * Release the dentry's inode, using the filesystem * d_iput() operation if defined. dentry remains in-use. */static void dentry_unlink_inode(struct dentry * dentry)	__releases(dentry->d_lock)	__releases(dentry->d_inode->i_lock){	struct inode *inode = dentry->d_inode;	__d_clear_type(dentry);	dentry->d_inode = NULL;	hlist_del_init(&dentry->d_alias);	dentry_rcuwalk_barrier(dentry);	spin_unlock(&dentry->d_lock);	spin_unlock(&inode->i_lock);	if (!inode->i_nlink)		fsnotify_inoderemove(inode);	if (dentry->d_op && dentry->d_op->d_iput)		dentry->d_op->d_iput(dentry, inode);	else		iput(inode);} /* * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry * is in use - which includes both the "real" per-superblock * LRU list _and_ the DCACHE_SHRINK_LIST use. * * The DCACHE_SHRINK_LIST bit is set whenever the dentry is * on the shrink list (ie not on the superblock LRU list). * * The per-cpu "nr_dentry_unused" counters are updated with * the DCACHE_LRU_LIST bit. * * These helper functions make sure we always follow the * rules. d_lock must be held by the caller. */#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))static void d_lru_add(struct dentry *dentry){	D_FLAG_VERIFY(dentry, 0);	dentry->d_flags |= DCACHE_LRU_LIST;	this_cpu_inc(nr_dentry_unused);	WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));} static void d_lru_del(struct dentry *dentry){	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);	dentry->d_flags &= ~DCACHE_LRU_LIST;	this_cpu_dec(nr_dentry_unused);	WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));} static void d_shrink_del(struct dentry *dentry){	D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);	list_del_init(&dentry->d_lru);	dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);	this_cpu_dec(nr_dentry_unused);} static void d_shrink_add(struct dentry *dentry, struct list_head *list){	D_FLAG_VERIFY(dentry, 0);	list_add(&dentry->d_lru, list);	dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;	this_cpu_inc(nr_dentry_unused);} /* * These can only be called under the global LRU lock, ie during the * callback for freeing the LRU list. "isolate" removes it from the * LRU lists entirely, while shrink_move moves it to the indicated * private list. */static void d_lru_isolate(struct dentry *dentry){	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);	dentry->d_flags &= ~DCACHE_LRU_LIST;	this_cpu_dec(nr_dentry_unused);	list_del_init(&dentry->d_lru);} static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list){	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);	dentry->d_flags |= DCACHE_SHRINK_LIST;	list_move_tail(&dentry->d_lru, list);} /* * dentry_lru_(add|del)_list) must be called with d_lock held. */static void dentry_lru_add(struct dentry *dentry){	if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))		d_lru_add(dentry);} /* * Remove a dentry with references from the LRU. * * If we are on the shrink list, then we can get to try_prune_one_dentry() and * lose our last reference through the parent walk. In this case, we need to * remove ourselves from the shrink list, not the LRU. */static void dentry_lru_del(struct dentry *dentry){	if (dentry->d_flags & DCACHE_LRU_LIST) {		if (dentry->d_flags & DCACHE_SHRINK_LIST)			return d_shrink_del(dentry);		d_lru_del(dentry);	}} /** * d_kill - kill dentry and return parent * @dentry: dentry to kill * @parent: parent dentry * * The dentry must already be unhashed and removed from the LRU. * * If this is the root of the dentry tree, return NULL. * * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by * d_kill. */static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)	__releases(dentry->d_lock)	__releases(parent->d_lock)	__releases(dentry->d_inode->i_lock){	list_del(&dentry->d_u.d_child);	/*	 * Inform d_walk() that we are no longer attached to the	 * dentry tree	 */	dentry->d_flags |= DCACHE_DENTRY_KILLED;	if (parent)		spin_unlock(&parent->d_lock);	dentry_iput(dentry);	/*	 * dentry_iput drops the locks, at which point nobody (except	 * transient RCU lookups) can reach this dentry.	 */	d_free(dentry);	return parent;} /** * d_drop - drop a dentry * @dentry: dentry to drop * * d_drop() unhashes the entry from the parent dentry hashes, so that it won't * be found through a VFS lookup any more. Note that this is different from * deleting the dentry - d_delete will try to mark the dentry negative if * possible, giving a successful _negative_ lookup, while d_drop will * just make the cache lookup fail. * * d_drop() is used mainly for stuff that wants to invalidate a dentry for some * reason (NFS timeouts or autofs deletes). * * __d_drop requires dentry->d_lock. */void __d_drop(struct dentry *dentry){	if (!d_unhashed(dentry)) {		struct hlist_bl_head *b;		/*		 * Hashed dentries are normally on the dentry hashtable,		 * with the exception of those newly allocated by		 * d_obtain_alias, which are always IS_ROOT:		 */		if (unlikely(IS_ROOT(dentry)))			b = &dentry->d_sb->s_anon;		else			b = d_hash(dentry->d_parent, dentry->d_name.hash); 		hlist_bl_lock(b);		__hlist_bl_del(&dentry->d_hash);		dentry->d_hash.pprev = NULL;		hlist_bl_unlock(b);		dentry_rcuwalk_barrier(dentry);	}}EXPORT_SYMBOL(__d_drop); void d_drop(struct dentry *dentry){	spin_lock(&dentry->d_lock);	__d_drop(dentry);	spin_unlock(&dentry->d_lock);}EXPORT_SYMBOL(d_drop);

Lines 1–500 of 3440.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t