| @@ -84,6 +84,61 @@ int dcache_dir_close(struct inode *inode, struct file *file) · 83 unmodified lines |
| 84 | } | 84 | } |
| 85 | EXPORT_SYMBOL(dcache_dir_close); | 85 | EXPORT_SYMBOL(dcache_dir_close); |
| 86 | | 86 | |
| | 87 | /* parent is locked at least shared */ |
| | 88 | static struct dentry *next_positive(struct dentry *parent, |
| | 89 | struct list_head *from, |
| | 90 | int count) |
| | 91 | { |
| | 92 | unsigned *seq = &parent->d_inode->i_dir_seq, n; |
| | 93 | struct dentry *res; |
| | 94 | struct list_head *p; |
| | 95 | bool skipped; |
| | 96 | int i; |
| | 97 | |
| | 98 | retry: |
| | 99 | i = count; |
| | 100 | skipped = false; |
| | 101 | n = smp_load_acquire(seq) & ~1; |
| | 102 | res = NULL; |
| | 103 | rcu_read_lock(); |
| | 104 | for (p = from->next; p != &parent->d_subdirs; p = p->next) { |
| | 105 | struct dentry *d = list_entry(p, struct dentry, d_child); |
| | 106 | if (!simple_positive(d)) { |
| | 107 | skipped = true; |
| | 108 | } else if (!--i) { |
| | 109 | res = d; |
| | 110 | break; |
| | 111 | } |
| | 112 | } |
| | 113 | rcu_read_unlock(); |
| | 114 | if (skipped) { |
| | 115 | smp_rmb(); |
| | 116 | if (unlikely(*seq != n)) |
| | 117 | goto retry; |
| | 118 | } |
| | 119 | return res; |
| | 120 | } |
| | 121 | |
| | 122 | static void move_cursor(struct dentry *cursor, struct list_head *after) |
| | 123 | { |
| | 124 | struct dentry *parent = cursor->d_parent; |
| | 125 | unsigned n, *seq = &parent->d_inode->i_dir_seq; |
| | 126 | spin_lock(&parent->d_lock); |
| | 127 | for (;;) { |
| | 128 | n = *seq; |
| | 129 | if (!(n & 1) && cmpxchg(seq, n, n + 1) == n) |
| | 130 | break; |
| | 131 | cpu_relax(); |
| | 132 | } |
| | 133 | __list_del(cursor->d_child.prev, cursor->d_child.next); |
| | 134 | if (after) |
| | 135 | list_add(&cursor->d_child, after); |
| | 136 | else |
| | 137 | list_add_tail(&cursor->d_child, &parent->d_subdirs); |
| | 138 | smp_store_release(seq, n + 2); |
| | 139 | spin_unlock(&parent->d_lock); |
| | 140 | } |
| | 141 | |
| 87 | loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) | 142 | loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) |
| 88 | { | 143 | { |
| 89 | struct dentry *dentry = file->f_path.dentry; | 144 | struct dentry *dentry = file->f_path.dentry; |
| @@ -99,25 +154,14 @@ loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) · 9 unmodified lines |
| 99 | if (offset != file->f_pos) { | 154 | if (offset != file->f_pos) { |
| 100 | file->f_pos = offset; | 155 | file->f_pos = offset; |
| 101 | if (file->f_pos >= 2) { | 156 | if (file->f_pos >= 2) { |
| 102 | struct list_head *p; | | |
| 103 | struct dentry *cursor = file->private_data; | 157 | struct dentry *cursor = file->private_data; |
| | 158 | struct dentry *to; |
| 104 | loff_t n = file->f_pos - 2; | 159 | loff_t n = file->f_pos - 2; |
| 105 | | 160 | |
| 106 | spin_lock(&dentry->d_lock); | 161 | inode_lock_shared(dentry->d_inode); |
| 107 | /* d_lock not required for cursor */ | 162 | to = next_positive(dentry, &dentry->d_subdirs, n); |
| 108 | list_del(&cursor->d_child); | 163 | move_cursor(cursor, to ? &to->d_child : NULL); |
| 109 | p = dentry->d_subdirs.next; | 164 | inode_unlock_shared(dentry->d_inode); |
| 110 | while (n && p != &dentry->d_subdirs) { | | |
| 111 | struct dentry *next; | | |
| 112 | next = list_entry(p, struct dentry, d_child); | | |
| 113 | spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); | | |
| 114 | if (simple_positive(next)) | | |
| 115 | n--; | | |
| 116 | spin_unlock(&next->d_lock); | | |
| 117 | p = p->next; | | |
| 118 | } | | |
| 119 | list_add_tail(&cursor->d_child, p); | | |
| 120 | spin_unlock(&dentry->d_lock); | | |
| 121 | } | 165 | } |
| 122 | } | 166 | } |
| 123 | return offset; | 167 | return offset; |
| @@ -140,36 +184,25 @@ int dcache_readdir(struct file *file, struct dir_context *ctx) · 16 unmodified lines |
| 140 | { | 184 | { |
| 141 | struct dentry *dentry = file->f_path.dentry; | 185 | struct dentry *dentry = file->f_path.dentry; |
| 142 | struct dentry *cursor = file->private_data; | 186 | struct dentry *cursor = file->private_data; |
| 143 | struct list_head *p, *q = &cursor->d_child; | 187 | struct list_head *p = &cursor->d_child; |
| | 188 | struct dentry *next; |
| | 189 | bool moved = false; |
| 144 | | 190 | |
| 145 | if (!dir_emit_dots(file, ctx)) | 191 | if (!dir_emit_dots(file, ctx)) |
| 146 | return 0; | 192 | return 0; |
| 147 | spin_lock(&dentry->d_lock); | | |
| 148 | if (ctx->pos == 2) | | |
| 149 | list_move(q, &dentry->d_subdirs); | | |
| 150 | | 193 | |
| 151 | for (p = q->next; p != &dentry->d_subdirs; p = p->next) { | 194 | if (ctx->pos == 2) |
| 152 | struct dentry *next = list_entry(p, struct dentry, d_child); | 195 | p = &dentry->d_subdirs; |
| 153 | spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); | 196 | while ((next = next_positive(dentry, p, 1)) != NULL) { |
| 154 | if (!simple_positive(next)) { | | |
| 155 | spin_unlock(&next->d_lock); | | |
| 156 | continue; | | |
| 157 | } | | |
| 158 | | | |
| 159 | spin_unlock(&next->d_lock); | | |
| 160 | spin_unlock(&dentry->d_lock); | | |
| 161 | if (!dir_emit(ctx, next->d_name.name, next->d_name.len, | 197 | if (!dir_emit(ctx, next->d_name.name, next->d_name.len, |
| 162 | d_inode(next)->i_ino, dt_type(d_inode(next)))) | 198 | d_inode(next)->i_ino, dt_type(d_inode(next)))) |
| 163 | return 0; | 199 | break; |
| 164 | spin_lock(&dentry->d_lock); | 200 | moved = true; |
| 165 | spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); | 201 | p = &next->d_child; |
| 166 | /* next is still alive */ | | |
| 167 | list_move(q, p); | | |
| 168 | spin_unlock(&next->d_lock); | | |
| 169 | p = q; | | |
| 170 | ctx->pos++; | 202 | ctx->pos++; |
| 171 | } | 203 | } |
| 172 | spin_unlock(&dentry->d_lock); | 204 | if (moved) |
| | 205 | move_cursor(cursor, p); |
| 173 | return 0; | 206 | return 0; |
| 174 | } | 207 | } |
| 175 | EXPORT_SYMBOL(dcache_readdir); | 208 | EXPORT_SYMBOL(dcache_readdir); |