History

fs: use percpu counter for nr_dentry and nr_dentry_unused

The nr_dentry stat is a globally touched cacheline and atomic operation twice over the lifetime of a dentry. It is used for the benfit of userspace only. Turn it into a per-cpu counter and always decrement it in d_free instead of doing various batching operations to reduce lock hold times in the callers. Based on an earlier patch from Nick Piggin <npiggin@suse.de>.

Christoph Hellwig committed 16 years ago

Commit 312d3ca1 parent 9c82ab93 files · +35 −20Browse files

Unified · Split

3 files
  1. fs/dcache.c +32 −19
  2. include/linux/fs.h +2 −0
  3. kernel/sysctl.c +1 −1
fs/dcache.c+32−19
@@ -67,6 +67,19 @@ struct dentry_stat_t dentry_stat = { · 66 unmodified lines
67 .age_limit = 45,67 .age_limit = 45,
68};68};
69 69
70static struct percpu_counter nr_dentry __cacheline_aligned_in_smp;
71static struct percpu_counter nr_dentry_unused __cacheline_aligned_in_smp;
72
73#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
74int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
75 size_t *lenp, loff_t *ppos)
76{
77 dentry_stat.nr_dentry = percpu_counter_sum_positive(&nr_dentry);
78 dentry_stat.nr_unused = percpu_counter_sum_positive(&nr_dentry_unused);
79 return proc_dointvec(table, write, buffer, lenp, ppos);
80}
81#endif
82
70static void __d_free(struct rcu_head *head)83static void __d_free(struct rcu_head *head)
71{84{
72 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);85 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
@@ -78,13 +91,14 @@ static void __d_free(struct rcu_head *head) · 5 unmodified lines
78}91}
79 92
80/*93/*
81 * no dcache_lock, please. The caller must decrement dentry_stat.nr_dentry94 * no dcache_lock, please.
82 * inside dcache_lock.
83 */95 */
84static void d_free(struct dentry *dentry)96static void d_free(struct dentry *dentry)
85{97{
98 percpu_counter_dec(&nr_dentry);
86 if (dentry->d_op && dentry->d_op->d_release)99 if (dentry->d_op && dentry->d_op->d_release)
87 dentry->d_op->d_release(dentry);100 dentry->d_op->d_release(dentry);
101
88 /* if dentry was never inserted into hash, immediate free is OK */102 /* if dentry was never inserted into hash, immediate free is OK */
89 if (hlist_unhashed(&dentry->d_hash))103 if (hlist_unhashed(&dentry->d_hash))
90 __d_free(&dentry->d_u.d_rcu);104 __d_free(&dentry->d_u.d_rcu);
@@ -125,14 +139,14 @@ static void dentry_lru_add(struct dentry *dentry) · 34 unmodified lines
125{139{
126 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);140 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
127 dentry->d_sb->s_nr_dentry_unused++;141 dentry->d_sb->s_nr_dentry_unused++;
128 dentry_stat.nr_unused++;142 percpu_counter_inc(&nr_dentry_unused);
129}143}
130 144
131static void dentry_lru_add_tail(struct dentry *dentry)145static void dentry_lru_add_tail(struct dentry *dentry)
132{146{
133 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);147 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
134 dentry->d_sb->s_nr_dentry_unused++;148 dentry->d_sb->s_nr_dentry_unused++;
135 dentry_stat.nr_unused++;149 percpu_counter_inc(&nr_dentry_unused);
136}150}
137 151
138static void dentry_lru_del(struct dentry *dentry)152static void dentry_lru_del(struct dentry *dentry)
@@ -140,7 +154,7 @@ static void dentry_lru_del(struct dentry *dentry) · 1 unmodified line
140 if (!list_empty(&dentry->d_lru)) {154 if (!list_empty(&dentry->d_lru)) {
141 list_del(&dentry->d_lru);155 list_del(&dentry->d_lru);
142 dentry->d_sb->s_nr_dentry_unused--;156 dentry->d_sb->s_nr_dentry_unused--;
143 dentry_stat.nr_unused--;157 percpu_counter_dec(&nr_dentry_unused);
144 }158 }
145}159}
146 160
@@ -149,7 +163,7 @@ static void dentry_lru_del_init(struct dentry *dentry) · 2 unmodified lines
149 if (likely(!list_empty(&dentry->d_lru))) {163 if (likely(!list_empty(&dentry->d_lru))) {
150 list_del_init(&dentry->d_lru);164 list_del_init(&dentry->d_lru);
151 dentry->d_sb->s_nr_dentry_unused--;165 dentry->d_sb->s_nr_dentry_unused--;
152 dentry_stat.nr_unused--;166 percpu_counter_dec(&nr_dentry_unused);
153 }167 }
154}168}
155 169
@@ -168,7 +182,6 @@ static struct dentry *d_kill(struct dentry *dentry) · 12 unmodified lines
168 struct dentry *parent;182 struct dentry *parent;
169 183
170 list_del(&dentry->d_u.d_child);184 list_del(&dentry->d_u.d_child);
171 dentry_stat.nr_dentry--; /* For d_free, below */
172 /*drops the locks, at that point nobody can reach this dentry */185 /*drops the locks, at that point nobody can reach this dentry */
173 dentry_iput(dentry);186 dentry_iput(dentry);
174 if (IS_ROOT(dentry))187 if (IS_ROOT(dentry))
@@ -314,7 +327,6 @@ int d_invalidate(struct dentry * dentry) · 139 unmodified lines
314EXPORT_SYMBOL(d_invalidate);327EXPORT_SYMBOL(d_invalidate);
315 328
316/* This should be called _only_ with dcache_lock held */329/* This should be called _only_ with dcache_lock held */
317
318static inline struct dentry * __dget_locked(struct dentry *dentry)330static inline struct dentry * __dget_locked(struct dentry *dentry)
319{331{
320 atomic_inc(&dentry->d_count);332 atomic_inc(&dentry->d_count);
@@ -534,7 +546,7 @@ static void prune_dcache(int count) · 213 unmodified lines
534{546{
535 struct super_block *sb, *p = NULL;547 struct super_block *sb, *p = NULL;
536 int w_count;548 int w_count;
537 int unused = dentry_stat.nr_unused;549 int unused = percpu_counter_sum_positive(&nr_dentry_unused);
538 int prune_ratio;550 int prune_ratio;
539 int pruned;551 int pruned;
540 552
@@ -699,20 +711,13 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry) · 158 unmodified lines
699 * otherwise we ascend to the parent and move to the711 * otherwise we ascend to the parent and move to the
700 * next sibling if there is one */712 * next sibling if there is one */
701 if (!parent)713 if (!parent)
702 goto out;714 return;
703
704 dentry = parent;715 dentry = parent;
705
706 } while (list_empty(&dentry->d_subdirs));716 } while (list_empty(&dentry->d_subdirs));
707 717
708 dentry = list_entry(dentry->d_subdirs.next,718 dentry = list_entry(dentry->d_subdirs.next,
709 struct dentry, d_u.d_child);719 struct dentry, d_u.d_child);
710 }720 }
711out:
712 /* several dentries were freed, need to correct nr_dentry */
713 spin_lock(&dcache_lock);
714 dentry_stat.nr_dentry -= detached;
715 spin_unlock(&dcache_lock);
716}721}
717 722
718/*723/*
@@ -896,12 +901,16 @@ EXPORT_SYMBOL(shrink_dcache_parent); · 177 unmodified lines
896 */901 */
897static int shrink_dcache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)902static int shrink_dcache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)
898{903{
904 int nr_unused;
905
899 if (nr) {906 if (nr) {
900 if (!(gfp_mask & __GFP_FS))907 if (!(gfp_mask & __GFP_FS))
901 return -1;908 return -1;
902 prune_dcache(nr);909 prune_dcache(nr);
903 }910 }
904 return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;911
912 nr_unused = percpu_counter_sum_positive(&nr_dentry_unused);
913 return (nr_unused / 100) * sysctl_vfs_cache_pressure;
905}914}
906 915
907static struct shrinker dcache_shrinker = {916static struct shrinker dcache_shrinker = {
@@ -968,9 +977,10 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) · 60 unmodified lines
968 spin_lock(&dcache_lock);977 spin_lock(&dcache_lock);
969 if (parent)978 if (parent)
970 list_add(&dentry->d_u.d_child, &parent->d_subdirs);979 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
971 dentry_stat.nr_dentry++;
972 spin_unlock(&dcache_lock);980 spin_unlock(&dcache_lock);
973 981
982 percpu_counter_inc(&nr_dentry);
983
974 return dentry;984 return dentry;
975}985}
976EXPORT_SYMBOL(d_alloc);986EXPORT_SYMBOL(d_alloc);
@@ -2417,6 +2427,9 @@ static void __init dcache_init(void) · 1440 unmodified lines
2417{2427{
2418 int loop;2428 int loop;
2419 2429
2430 percpu_counter_init(&nr_dentry, 0);
2431 percpu_counter_init(&nr_dentry_unused, 0);
2432
2420 /* 2433 /*
2421 * A constructor could be added for stable state like the lists,2434 * A constructor could be added for stable state like the lists,
2422 * but it is probably not worth it because of the cache nature2435 * but it is probably not worth it because of the cache nature
include/linux/fs.h+2−0
@@ -2490,6 +2490,8 @@ ssize_t simple_attr_write(struct file *file, const char __user *buf, · 2489 unmodified lines
2490struct ctl_table;2490struct ctl_table;
2491int proc_nr_files(struct ctl_table *table, int write,2491int proc_nr_files(struct ctl_table *table, int write,
2492 void __user *buffer, size_t *lenp, loff_t *ppos);2492 void __user *buffer, size_t *lenp, loff_t *ppos);
2493int proc_nr_dentry(struct ctl_table *table, int write,
2494 void __user *buffer, size_t *lenp, loff_t *ppos);
2493int proc_nr_inodes(struct ctl_table *table, int write,2495int proc_nr_inodes(struct ctl_table *table, int write,
2494 void __user *buffer, size_t *lenp, loff_t *ppos);2496 void __user *buffer, size_t *lenp, loff_t *ppos);
2495int __init get_filesystem_list(char *buf);2497int __init get_filesystem_list(char *buf);
kernel/sysctl.c+1−1
@@ -1377,7 +1377,7 @@ static struct ctl_table fs_table[] = { · 1376 unmodified lines
1377 .data = &dentry_stat,1377 .data = &dentry_stat,
1378 .maxlen = 6*sizeof(int),1378 .maxlen = 6*sizeof(int),
1379 .mode = 0444,1379 .mode = 0444,
1380 .proc_handler = proc_dointvec,1380 .proc_handler = proc_nr_dentry,
1381 },1381 },
1382 {1382 {
1383 .procname = "overflowuid",1383 .procname = "overflowuid",