History

fs: use percpu counter for nr_dentry and nr_dentry_unused

The nr_dentry stat is a globally touched cacheline and atomic operation twice over the lifetime of a dentry. It is used for the benfit of userspace only. Turn it into a per-cpu counter and always decrement it in d_free instead of doing various batching operations to reduce lock hold times in the callers. Based on an earlier patch from Nick Piggin <npiggin@suse.de>.

Christoph Hellwig committed 16 years ago

Commit 312d3ca1 parent 9c82ab93 files · +35 −20Browse files

Unified · Split

3 files
  1. fs/dcache.c +32 −19
  2. include/linux/fs.h +2 −0
  3. kernel/sysctl.c +1 −1
fs/dcache.c+32−19
@@ -67,6 +67,19 @@ struct dentry_stat_t dentry_stat = { · 66 unmodified lines
6767 .age_limit = 45,
6868 };
6969
70+static struct percpu_counter nr_dentry __cacheline_aligned_in_smp;
71+static struct percpu_counter nr_dentry_unused __cacheline_aligned_in_smp;
72+
73+#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
74+int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
75+ size_t *lenp, loff_t *ppos)
76+{
77+ dentry_stat.nr_dentry = percpu_counter_sum_positive(&nr_dentry);
78+ dentry_stat.nr_unused = percpu_counter_sum_positive(&nr_dentry_unused);
79+ return proc_dointvec(table, write, buffer, lenp, ppos);
80+}
81+#endif
82+
7083 static void __d_free(struct rcu_head *head)
7184 {
7285 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
@@ -78,13 +91,14 @@ static void __d_free(struct rcu_head *head) · 5 unmodified lines
7891 }
7992
8093 /*
81− * no dcache_lock, please. The caller must decrement dentry_stat.nr_dentry
82− * inside dcache_lock.
94+ * no dcache_lock, please.
8395 */
8496 static void d_free(struct dentry *dentry)
8597 {
98+ percpu_counter_dec(&nr_dentry);
8699 if (dentry->d_op && dentry->d_op->d_release)
87100 dentry->d_op->d_release(dentry);
101+
88102 /* if dentry was never inserted into hash, immediate free is OK */
89103 if (hlist_unhashed(&dentry->d_hash))
90104 __d_free(&dentry->d_u.d_rcu);
@@ -125,14 +139,14 @@ static void dentry_lru_add(struct dentry *dentry) · 34 unmodified lines
125139 {
126140 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
127141 dentry->d_sb->s_nr_dentry_unused++;
128− dentry_stat.nr_unused++;
142+ percpu_counter_inc(&nr_dentry_unused);
129143 }
130144
131145 static void dentry_lru_add_tail(struct dentry *dentry)
132146 {
133147 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
134148 dentry->d_sb->s_nr_dentry_unused++;
135− dentry_stat.nr_unused++;
149+ percpu_counter_inc(&nr_dentry_unused);
136150 }
137151
138152 static void dentry_lru_del(struct dentry *dentry)
@@ -140,7 +154,7 @@ static void dentry_lru_del(struct dentry *dentry) · 1 unmodified line
140154 if (!list_empty(&dentry->d_lru)) {
141155 list_del(&dentry->d_lru);
142156 dentry->d_sb->s_nr_dentry_unused--;
143− dentry_stat.nr_unused--;
157+ percpu_counter_dec(&nr_dentry_unused);
144158 }
145159 }
146160
@@ -149,7 +163,7 @@ static void dentry_lru_del_init(struct dentry *dentry) · 2 unmodified lines
149163 if (likely(!list_empty(&dentry->d_lru))) {
150164 list_del_init(&dentry->d_lru);
151165 dentry->d_sb->s_nr_dentry_unused--;
152− dentry_stat.nr_unused--;
166+ percpu_counter_dec(&nr_dentry_unused);
153167 }
154168 }
155169
@@ -168,7 +182,6 @@ static struct dentry *d_kill(struct dentry *dentry) · 12 unmodified lines
168182 struct dentry *parent;
169183
170184 list_del(&dentry->d_u.d_child);
171− dentry_stat.nr_dentry--; /* For d_free, below */
172185 /*drops the locks, at that point nobody can reach this dentry */
173186 dentry_iput(dentry);
174187 if (IS_ROOT(dentry))
@@ -314,7 +327,6 @@ int d_invalidate(struct dentry * dentry) · 139 unmodified lines
314327 EXPORT_SYMBOL(d_invalidate);
315328
316329 /* This should be called _only_ with dcache_lock held */
317−
318330 static inline struct dentry * __dget_locked(struct dentry *dentry)
319331 {
320332 atomic_inc(&dentry->d_count);
@@ -534,7 +546,7 @@ static void prune_dcache(int count) · 213 unmodified lines
534546 {
535547 struct super_block *sb, *p = NULL;
536548 int w_count;
537− int unused = dentry_stat.nr_unused;
549+ int unused = percpu_counter_sum_positive(&nr_dentry_unused);
538550 int prune_ratio;
539551 int pruned;
540552
@@ -699,20 +711,13 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry) · 158 unmodified lines
699711 * otherwise we ascend to the parent and move to the
700712 * next sibling if there is one */
701713 if (!parent)
702− goto out;
703−
714+ return;
704715 dentry = parent;
705−
706716 } while (list_empty(&dentry->d_subdirs));
707717
708718 dentry = list_entry(dentry->d_subdirs.next,
709719 struct dentry, d_u.d_child);
710720 }
711−out:
712− /* several dentries were freed, need to correct nr_dentry */
713− spin_lock(&dcache_lock);
714− dentry_stat.nr_dentry -= detached;
715− spin_unlock(&dcache_lock);
716721 }
717722
718723 /*
@@ -896,12 +901,16 @@ EXPORT_SYMBOL(shrink_dcache_parent); · 177 unmodified lines
896901 */
897902 static int shrink_dcache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)
898903 {
904+ int nr_unused;
905+
899906 if (nr) {
900907 if (!(gfp_mask & __GFP_FS))
901908 return -1;
902909 prune_dcache(nr);
903910 }
904− return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;
911+
912+ nr_unused = percpu_counter_sum_positive(&nr_dentry_unused);
913+ return (nr_unused / 100) * sysctl_vfs_cache_pressure;
905914 }
906915
907916 static struct shrinker dcache_shrinker = {
@@ -968,9 +977,10 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) · 60 unmodified lines
968977 spin_lock(&dcache_lock);
969978 if (parent)
970979 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
971− dentry_stat.nr_dentry++;
972980 spin_unlock(&dcache_lock);
973981
982+ percpu_counter_inc(&nr_dentry);
983+
974984 return dentry;
975985 }
976986 EXPORT_SYMBOL(d_alloc);
@@ -2417,6 +2427,9 @@ static void __init dcache_init(void) · 1440 unmodified lines
24172427 {
24182428 int loop;
24192429
2430+ percpu_counter_init(&nr_dentry, 0);
2431+ percpu_counter_init(&nr_dentry_unused, 0);
2432+
24202433 /*
24212434 * A constructor could be added for stable state like the lists,
24222435 * but it is probably not worth it because of the cache nature
include/linux/fs.h+2−0
@@ -2490,6 +2490,8 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, · 2489 unmodified lines
24902490 struct ctl_table;
24912491 int proc_nr_files(struct ctl_table *table, int write,
24922492 void __user *buffer, size_t *lenp, loff_t *ppos);
2493+int proc_nr_dentry(struct ctl_table *table, int write,
2494+ void __user *buffer, size_t *lenp, loff_t *ppos);
24932495 int proc_nr_inodes(struct ctl_table *table, int write,
24942496 void __user *buffer, size_t *lenp, loff_t *ppos);
24952497 int __init get_filesystem_list(char *buf);
kernel/sysctl.c+1−1
@@ -1377,7 +1377,7 @@ static struct ctl_table fs_table[] = { · 1376 unmodified lines
13771377 .data = &dentry_stat,
13781378 .maxlen = 6*sizeof(int),
13791379 .mode = 0444,
1380− .proc_handler = proc_dointvec,
1380+ .proc_handler = proc_nr_dentry,
13811381 },
13821382 {
13831383 .procname = "overflowuid",