| @@ -84,6 +84,61 @@ int dcache_dir_close(struct inode *inode, struct file *file) · 83 unmodified lines |
| 84 | 84 | } |
| 85 | 85 | EXPORT_SYMBOL(dcache_dir_close); |
| 86 | 86 | |
| 87 | +/* parent is locked at least shared */ |
| 88 | +static struct dentry *next_positive(struct dentry *parent, |
| 89 | + struct list_head *from, |
| 90 | + int count) |
| 91 | +{ |
| 92 | + unsigned *seq = &parent->d_inode->i_dir_seq, n; |
| 93 | + struct dentry *res; |
| 94 | + struct list_head *p; |
| 95 | + bool skipped; |
| 96 | + int i; |
| 97 | + |
| 98 | +retry: |
| 99 | + i = count; |
| 100 | + skipped = false; |
| 101 | + n = smp_load_acquire(seq) & ~1; |
| 102 | + res = NULL; |
| 103 | + rcu_read_lock(); |
| 104 | + for (p = from->next; p != &parent->d_subdirs; p = p->next) { |
| 105 | + struct dentry *d = list_entry(p, struct dentry, d_child); |
| 106 | + if (!simple_positive(d)) { |
| 107 | + skipped = true; |
| 108 | + } else if (!--i) { |
| 109 | + res = d; |
| 110 | + break; |
| 111 | + } |
| 112 | + } |
| 113 | + rcu_read_unlock(); |
| 114 | + if (skipped) { |
| 115 | + smp_rmb(); |
| 116 | + if (unlikely(*seq != n)) |
| 117 | + goto retry; |
| 118 | + } |
| 119 | + return res; |
| 120 | +} |
| 121 | + |
| 122 | +static void move_cursor(struct dentry *cursor, struct list_head *after) |
| 123 | +{ |
| 124 | + struct dentry *parent = cursor->d_parent; |
| 125 | + unsigned n, *seq = &parent->d_inode->i_dir_seq; |
| 126 | + spin_lock(&parent->d_lock); |
| 127 | + for (;;) { |
| 128 | + n = *seq; |
| 129 | + if (!(n & 1) && cmpxchg(seq, n, n + 1) == n) |
| 130 | + break; |
| 131 | + cpu_relax(); |
| 132 | + } |
| 133 | + __list_del(cursor->d_child.prev, cursor->d_child.next); |
| 134 | + if (after) |
| 135 | + list_add(&cursor->d_child, after); |
| 136 | + else |
| 137 | + list_add_tail(&cursor->d_child, &parent->d_subdirs); |
| 138 | + smp_store_release(seq, n + 2); |
| 139 | + spin_unlock(&parent->d_lock); |
| 140 | +} |
| 141 | + |
| 87 | 142 | loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) |
| 88 | 143 | { |
| 89 | 144 | struct dentry *dentry = file->f_path.dentry; |
| @@ -99,25 +154,14 @@ loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) · 9 unmodified lines |
| 99 | 154 | if (offset != file->f_pos) { |
| 100 | 155 | file->f_pos = offset; |
| 101 | 156 | if (file->f_pos >= 2) { |
| 102 | | − struct list_head *p; |
| 103 | 157 | struct dentry *cursor = file->private_data; |
| 158 | + struct dentry *to; |
| 104 | 159 | loff_t n = file->f_pos - 2; |
| 105 | 160 | |
| 106 | | − spin_lock(&dentry->d_lock); |
| 107 | | − /* d_lock not required for cursor */ |
| 108 | | − list_del(&cursor->d_child); |
| 109 | | − p = dentry->d_subdirs.next; |
| 110 | | − while (n && p != &dentry->d_subdirs) { |
| 111 | | − struct dentry *next; |
| 112 | | − next = list_entry(p, struct dentry, d_child); |
| 113 | | − spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); |
| 114 | | − if (simple_positive(next)) |
| 115 | | − n--; |
| 116 | | − spin_unlock(&next->d_lock); |
| 117 | | − p = p->next; |
| 118 | | − } |
| 119 | | − list_add_tail(&cursor->d_child, p); |
| 120 | | − spin_unlock(&dentry->d_lock); |
| 161 | + inode_lock_shared(dentry->d_inode); |
| 162 | + to = next_positive(dentry, &dentry->d_subdirs, n); |
| 163 | + move_cursor(cursor, to ? &to->d_child : NULL); |
| 164 | + inode_unlock_shared(dentry->d_inode); |
| 121 | 165 | } |
| 122 | 166 | } |
| 123 | 167 | return offset; |
| @@ -140,36 +184,25 @@ int dcache_readdir(struct file *file, struct dir_context *ctx) · 16 unmodified lines |
| 140 | 184 | { |
| 141 | 185 | struct dentry *dentry = file->f_path.dentry; |
| 142 | 186 | struct dentry *cursor = file->private_data; |
| 143 | | − struct list_head *p, *q = &cursor->d_child; |
| 187 | + struct list_head *p = &cursor->d_child; |
| 188 | + struct dentry *next; |
| 189 | + bool moved = false; |
| 144 | 190 | |
| 145 | 191 | if (!dir_emit_dots(file, ctx)) |
| 146 | 192 | return 0; |
| 147 | | − spin_lock(&dentry->d_lock); |
| 148 | | − if (ctx->pos == 2) |
| 149 | | − list_move(q, &dentry->d_subdirs); |
| 150 | 193 | |
| 151 | | − for (p = q->next; p != &dentry->d_subdirs; p = p->next) { |
| 152 | | − struct dentry *next = list_entry(p, struct dentry, d_child); |
| 153 | | − spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); |
| 154 | | − if (!simple_positive(next)) { |
| 155 | | − spin_unlock(&next->d_lock); |
| 156 | | − continue; |
| 157 | | − } |
| 158 | | − |
| 159 | | − spin_unlock(&next->d_lock); |
| 160 | | − spin_unlock(&dentry->d_lock); |
| 194 | + if (ctx->pos == 2) |
| 195 | + p = &dentry->d_subdirs; |
| 196 | + while ((next = next_positive(dentry, p, 1)) != NULL) { |
| 161 | 197 | if (!dir_emit(ctx, next->d_name.name, next->d_name.len, |
| 162 | 198 | d_inode(next)->i_ino, dt_type(d_inode(next)))) |
| 163 | | − return 0; |
| 164 | | − spin_lock(&dentry->d_lock); |
| 165 | | − spin_lock_nested(&next->d_lock, DENTRY_D_LOCK_NESTED); |
| 166 | | − /* next is still alive */ |
| 167 | | − list_move(q, p); |
| 168 | | − spin_unlock(&next->d_lock); |
| 169 | | − p = q; |
| 199 | + break; |
| 200 | + moved = true; |
| 201 | + p = &next->d_child; |
| 170 | 202 | ctx->pos++; |
| 171 | 203 | } |
| 172 | | − spin_unlock(&dentry->d_lock); |
| 204 | + if (moved) |
| 205 | + move_cursor(cursor, p); |
| 173 | 206 | return 0; |
| 174 | 207 | } |
| 175 | 208 | EXPORT_SYMBOL(dcache_readdir); |