// SPDX-License-Identifier: GPL-2.0-only/* * fs/dcache.c * * Complete reimplementation * (C) 1997 Thomas Schoebel-Theuer, * with heavy changes by Linus Torvalds */ /* * Notes on the allocation strategy: * * The dcache is a master of the icache - whenever a dcache entry * exists, the inode will always exist. "iput()" is done either when * the dcache entry is deleted or garbage collected. */ #include <linux/ratelimit.h>#include <linux/string.h>#include <linux/mm.h>#include <linux/fs.h>#include <linux/fscrypt.h>#include <linux/fsnotify.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/hash.h>#include <linux/cache.h>#include <linux/export.h>#include <linux/security.h>#include <linux/seqlock.h>#include <linux/memblock.h>#include <linux/bit_spinlock.h>#include <linux/rculist_bl.h>#include <linux/list_lru.h>#include "internal.h"#include "mount.h" #include <asm/runtime-const.h> /* * Usage: * dcache->d_inode->i_lock protects: *   - i_dentry, d_alias, d_inode of aliases * dcache_hash_bucket lock protects: *   - the dcache hash table * s_roots_lock protects: *   - the s_roots list (see __d_move()/dentry_unlist()/d_obtain_root()) * dentry->d_sb->s_dentry_lru_lock protects: *   - the dcache lru lists and counters * d_lock protects: *   - d_flags *   - d_name *   - d_lru *   - d_count *   - d_unhashed() *   - d_parent and d_chilren *   - childrens' d_sib and d_parent *   - d_alias, d_inode * * Ordering: * dentry->d_inode->i_lock *   dentry->d_lock *     dentry->d_sb->s_dentry_lru_lock *     dcache_hash_bucket lock *     s_roots lock * * If there is an ancestor relationship: * dentry->d_parent->...->d_parent->d_lock *   ... *     dentry->d_parent->d_lock *       dentry->d_lock * * If no ancestor relationship: * arbitrary, since it's serialized on rename_lock */static int sysctl_vfs_cache_pressure __read_mostly = 100;static int sysctl_vfs_cache_pressure_denom __read_mostly = 100; unsigned long vfs_pressure_ratio(unsigned long val){	return mult_frac(val, sysctl_vfs_cache_pressure, sysctl_vfs_cache_pressure_denom);}EXPORT_SYMBOL_GPL(vfs_pressure_ratio); __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); EXPORT_SYMBOL(rename_lock); static struct kmem_cache *__dentry_cache __ro_after_init;#define dentry_cache runtime_const_ptr(__dentry_cache) const struct qstr empty_name = QSTR_INIT("", 0);EXPORT_SYMBOL(empty_name);const struct qstr slash_name = QSTR_INIT("/", 1);EXPORT_SYMBOL(slash_name);const struct qstr dotdot_name = QSTR_INIT("..", 2);EXPORT_SYMBOL(dotdot_name); /* * This is the single most critical data structure when it comes * to the dcache: the hashtable for lookups. Somebody should try * to make this good - I've just made it work. * * This hash-function tries to avoid losing too many bits of hash * information, yet avoid using a prime hash-size or similar. * * Marking the variables "used" ensures that the compiler doesn't * optimize them away completely on architectures with runtime * constant infrastructure, this allows debuggers to see their * values. But updating these values has no effect on those arches. */ static unsigned int d_hash_shift __ro_after_init __used; static struct hlist_bl_head *dentry_hashtable __ro_after_init __used; static inline struct hlist_bl_head *d_hash(unsigned long hashlen){	return runtime_const_ptr(dentry_hashtable) +		runtime_const_shift_right_32(hashlen, d_hash_shift);} #define IN_LOOKUP_SHIFT 10static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT]; static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,					unsigned int hash){	hash += (unsigned long) parent / L1_CACHE_BYTES;	return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);} struct dentry_stat_t {	long nr_dentry;	long nr_unused;	long age_limit;		/* age in seconds */	long want_pages;	/* pages requested by system */	long nr_negative;	/* # of unused negative dentries */	long dummy;		/* Reserved for future use */}; static DEFINE_PER_CPU(long, nr_dentry);static DEFINE_PER_CPU(long, nr_dentry_unused);static DEFINE_PER_CPU(long, nr_dentry_negative);static int dentry_negative_policy; #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)/* Statistics gathering. */static struct dentry_stat_t dentry_stat = {	.age_limit = 45,}; /* * Here we resort to our own counters instead of using generic per-cpu counters * for consistency with what the vfs inode code does. We are expected to harvest * better code and performance by having our own specialized counters. * * Please note that the loop is done over all possible CPUs, not over all online * CPUs. The reason for this is that we don't want to play games with CPUs going * on and off. If one of them goes off, we will just keep their counters. * * glommer: See cffbc8a for details, and if you ever intend to change this, * please update all vfs counters to match. */static long get_nr_dentry(void){	int i;	long sum = 0;	for_each_possible_cpu(i)		sum += per_cpu(nr_dentry, i);	return sum < 0 ? 0 : sum;} static long get_nr_dentry_unused(void){	int i;	long sum = 0;	for_each_possible_cpu(i)		sum += per_cpu(nr_dentry_unused, i);	return sum < 0 ? 0 : sum;} static long get_nr_dentry_negative(void){	int i;	long sum = 0; 	for_each_possible_cpu(i)		sum += per_cpu(nr_dentry_negative, i);	return sum < 0 ? 0 : sum;} static int proc_nr_dentry(const struct ctl_table *table, int write, void *buffer,			  size_t *lenp, loff_t *ppos){	dentry_stat.nr_dentry = get_nr_dentry();	dentry_stat.nr_unused = get_nr_dentry_unused();	dentry_stat.nr_negative = get_nr_dentry_negative();	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);} static const struct ctl_table fs_dcache_sysctls[] = {	{		.procname	= "dentry-state",		.data		= &dentry_stat,		.maxlen		= 6*sizeof(long),		.mode		= 0444,		.proc_handler	= proc_nr_dentry,	},	{		.procname	= "dentry-negative",		.data		= &dentry_negative_policy,		.maxlen		= sizeof(dentry_negative_policy),		.mode		= 0644,		.proc_handler	= proc_dointvec_minmax,		.extra1		= SYSCTL_ZERO,		.extra2		= SYSCTL_ONE,	},}; static const struct ctl_table vm_dcache_sysctls[] = {	{		.procname	= "vfs_cache_pressure",		.data		= &sysctl_vfs_cache_pressure,		.maxlen		= sizeof(sysctl_vfs_cache_pressure),		.mode		= 0644,		.proc_handler	= proc_dointvec_minmax,		.extra1		= SYSCTL_ZERO,	},	{		.procname	= "vfs_cache_pressure_denom",		.data		= &sysctl_vfs_cache_pressure_denom,		.maxlen		= sizeof(sysctl_vfs_cache_pressure_denom),		.mode		= 0644,		.proc_handler	= proc_dointvec_minmax,		.extra1		= SYSCTL_ONE_HUNDRED,	},}; static int __init init_fs_dcache_sysctls(void){	register_sysctl_init("vm", vm_dcache_sysctls);	register_sysctl_init("fs", fs_dcache_sysctls);	return 0;}fs_initcall(init_fs_dcache_sysctls);#endif /* * Compare 2 name strings, return 0 if they match, otherwise non-zero. * The strings are both count bytes long, and count is non-zero. */#ifdef CONFIG_DCACHE_WORD_ACCESS #include <asm/word-at-a-time.h>/* * NOTE! 'cs' and 'scount' come from a dentry, so it has a * aligned allocation for this particular component. We don't * strictly need the load_unaligned_zeropad() safety, but it * doesn't hurt either. * * In contrast, 'ct' and 'tcount' can be from a pathname, and do * need the careful unaligned handling. */static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount){	unsigned long a,b,mask; 	for (;;) {		a = read_word_at_a_time(cs);		b = load_unaligned_zeropad(ct);		if (tcount < sizeof(unsigned long))			break;		if (unlikely(a != b))			return 1;		cs += sizeof(unsigned long);		ct += sizeof(unsigned long);		tcount -= sizeof(unsigned long);		if (!tcount)			return 0;	}	mask = bytemask_from_count(tcount);	return unlikely(!!((a ^ b) & mask));} #else static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount){	do {		if (*cs != *ct)			return 1;		cs++;		ct++;		tcount--;	} while (tcount);	return 0;} #endif static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount){	/*	 * Be careful about RCU walk racing with rename:	 * use 'READ_ONCE' to fetch the name pointer.	 *	 * NOTE! Even if a rename will mean that the length	 * was not loaded atomically, we don't care. The	 * RCU walk will check the sequence count eventually,	 * and catch it. And we won't overrun the buffer,	 * because we're reading the name pointer atomically,	 * and a dentry name is guaranteed to be properly	 * terminated with a NUL byte.	 *	 * End result: even if 'len' is wrong, we'll exit	 * early because the data cannot match (there can	 * be no NUL in the ct/tcount data)	 */	const unsigned char *cs = READ_ONCE(dentry->d_name.name); 	return dentry_string_cmp(cs, ct, tcount);} /* * long names are allocated separately from dentry and never modified. * Refcounted, freeing is RCU-delayed.  See take_dentry_name_snapshot() * for the reason why ->count and ->head can't be combined into a union. * dentry_string_cmp() relies upon ->name[] being word-aligned. */struct external_name {	atomic_t count;	struct rcu_head head;	unsigned char name[] __aligned(sizeof(unsigned long));}; static inline struct external_name *external_name(struct dentry *dentry){	return container_of(dentry->d_name.name, struct external_name, name[0]);} static void __d_free(struct rcu_head *head){	struct dentry *dentry = container_of(head, struct dentry, d_rcu); 	kmem_cache_free(dentry_cache, dentry); } static void __d_free_external(struct rcu_head *head){	struct dentry *dentry = container_of(head, struct dentry, d_rcu);	kfree(external_name(dentry));	kmem_cache_free(dentry_cache, dentry);} static inline int dname_external(const struct dentry *dentry){	return dentry->d_name.name != dentry->d_shortname.string;} void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry){	unsigned seq;	const unsigned char *s; 	rcu_read_lock();retry:	seq = read_seqcount_begin(&dentry->d_seq);	s = READ_ONCE(dentry->d_name.name);	name->name.hash_len = dentry->d_name.hash_len;	name->name.name = name->inline_name.string;	if (likely(s == dentry->d_shortname.string)) {		name->inline_name = dentry->d_shortname;	} else {		struct external_name *p;		p = container_of(s, struct external_name, name[0]);		// get a valid reference		if (unlikely(!atomic_inc_not_zero(&p->count)))			goto retry;		name->name.name = s;	}	if (read_seqcount_retry(&dentry->d_seq, seq)) {		release_dentry_name_snapshot(name);		goto retry;	}	rcu_read_unlock();}EXPORT_SYMBOL(take_dentry_name_snapshot); void release_dentry_name_snapshot(struct name_snapshot *name){	if (unlikely(name->name.name != name->inline_name.string)) {		struct external_name *p;		p = container_of(name->name.name, struct external_name, name[0]);		if (unlikely(atomic_dec_and_test(&p->count)))			kfree_rcu(p, head);	}}EXPORT_SYMBOL(release_dentry_name_snapshot); static inline void __d_set_inode_and_type(struct dentry *dentry,					  struct inode *inode,					  unsigned type_flags){	unsigned flags; 	dentry->d_inode = inode;	flags = READ_ONCE(dentry->d_flags);	flags &= ~DCACHE_ENTRY_TYPE;	flags |= type_flags;	smp_store_release(&dentry->d_flags, flags);} static inline void __d_clear_type_and_inode(struct dentry *dentry){	unsigned flags = READ_ONCE(dentry->d_flags); 	flags &= ~DCACHE_ENTRY_TYPE;	WRITE_ONCE(dentry->d_flags, flags);	dentry->d_inode = NULL;	/*	 * The negative counter only tracks dentries on the LRU. Don't inc if	 * d_lru is on another list.	 */	if ((flags & (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST)		this_cpu_inc(nr_dentry_negative);} #define DENTRY_WARN_ONCE(condition, dentry) \	WARN_ONCE((condition), "dentry=%p d_flags=0x%x\n", (dentry), (dentry)->d_flags)#define D_FLAG_VERIFY(dentry, x) \	DENTRY_WARN_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x), (dentry)) static void dentry_free(struct dentry *dentry){	DENTRY_WARN_ONCE(d_really_is_positive(dentry), dentry);	DENTRY_WARN_ONCE(!lockref_is_dead(&dentry->d_lockref), dentry);	D_FLAG_VERIFY(dentry, 0);	if (unlikely(dname_external(dentry))) {		struct external_name *p = external_name(dentry);		if (likely(atomic_dec_and_test(&p->count))) {			call_rcu(&dentry->d_rcu, __d_free_external);			return;		}	}	/* if dentry was never visible to RCU, immediate free is OK */	if (dentry->d_flags & DCACHE_NORCU)		__d_free(&dentry->d_rcu);	else		call_rcu(&dentry->d_rcu, __d_free);} /* * Release the dentry's inode, using the filesystem * d_iput() operation if defined. */static void dentry_unlink_inode(struct dentry * dentry)	__releases(dentry->d_lock)	__releases(dentry->d_inode->i_lock){	struct inode *inode = dentry->d_inode; 	raw_write_seqcount_begin(&dentry->d_seq);	__d_clear_type_and_inode(dentry);	__hlist_del(&dentry->d_alias);	/*	 * dentry becomes negative, so the space occupied by ->d_alias	 * belongs to ->waiters now.	 */	dentry->waiters = NULL;	raw_write_seqcount_end(&dentry->d_seq);	spin_unlock(&dentry->d_lock);	spin_unlock(&inode->i_lock);	if (!inode->i_nlink)		fsnotify_inoderemove(inode);	if (dentry->d_op && dentry->d_op->d_iput)		dentry->d_op->d_iput(dentry, inode);	else		iput(inode);} /* * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry * is in use - which includes both the "real" per-superblock * LRU list _and_ the DCACHE_SHRINK_LIST use. * * The DCACHE_SHRINK_LIST bit is set whenever the dentry is * on the shrink list (ie not on the superblock LRU list). * * The per-cpu "nr_dentry_unused" counters are updated with * the DCACHE_LRU_LIST bit. * * The per-cpu "nr_dentry_negative" counters are only updated * when deleted from or added to the per-superblock LRU list, not * from/to the shrink list. That is to avoid an unneeded dec/inc * pair when moving from LRU to shrink list in select_collect(). * * These helper functions make sure we always follow the * rules. d_lock must be held by the caller. */

Lines 1–500 of 3525. Load more