Rather than returning a pointer to a static nulls, rht_bucket_var() now returns NULL if the bucket doesn't exist. This will make the next patch, which stores a bitlock in the bucket pointer, somewhat cleaner. This change involves introducing __rht_bucket_nested() which is like rht_bucket_nested(), but doesn't provide the static nulls, and changing rht_bucket_nested() to call this and possible provide a static nulls - as is still needed for the non-var case.
Unified · Split
| @@ -265,6 +265,8 @@ void rhashtable_destroy(struct rhashtable *ht); · 264 unmodified lines | ||
| 265 | 265 | |
| 266 | 266 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, |
| 267 | 267 | unsigned int hash); |
| 268 | +struct rhash_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, | |
| 269 | + unsigned int hash); | |
| 268 | 270 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, |
| 269 | 271 | struct bucket_table *tbl, |
| 270 | 272 | unsigned int hash); |
| @@ -294,7 +296,7 @@ static inline struct rhash_head __rcu *const *rht_bucket( · 23 unmodified lines | ||
| 294 | 296 | static inline struct rhash_head __rcu **rht_bucket_var( |
| 295 | 297 | struct bucket_table *tbl, unsigned int hash) |
| 296 | 298 | { |
| 297 | − return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : | |
| 299 | + return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) : | |
| 298 | 300 | &tbl->buckets[hash]; |
| 299 | 301 | } |
| 300 | 302 | |
| @@ -890,6 +892,8 @@ static inline int __rhashtable_remove_fast_one( · 589 unmodified lines | ||
| 890 | 892 | spin_lock_bh(lock); |
| 891 | 893 | |
| 892 | 894 | pprev = rht_bucket_var(tbl, hash); |
| 895 | + if (!pprev) | |
| 896 | + goto out; | |
| 893 | 897 | rht_for_each_from(he, *pprev, tbl, hash) { |
| 894 | 898 | struct rhlist_head *list; |
| 895 | 899 | |
| @@ -934,6 +938,7 @@ static inline int __rhashtable_remove_fast_one( · 38 unmodified lines | ||
| 934 | 938 | break; |
| 935 | 939 | } |
| 936 | 940 | |
| 941 | +out: | |
| 937 | 942 | spin_unlock_bh(lock); |
| 938 | 943 | |
| 939 | 944 | if (err > 0) { |
| @@ -1042,6 +1047,8 @@ static inline int __rhashtable_replace_fast( · 102 unmodified lines | ||
| 1042 | 1047 | spin_lock_bh(lock); |
| 1043 | 1048 | |
| 1044 | 1049 | pprev = rht_bucket_var(tbl, hash); |
| 1050 | + if (!pprev) | |
| 1051 | + goto out; | |
| 1045 | 1052 | rht_for_each_from(he, *pprev, tbl, hash) { |
| 1046 | 1053 | if (he != obj_old) { |
| 1047 | 1054 | pprev = &he->next; |
| @@ -1053,7 +1060,7 @@ static inline int __rhashtable_replace_fast( · 5 unmodified lines | ||
| 1053 | 1060 | err = 0; |
| 1054 | 1061 | break; |
| 1055 | 1062 | } |
| 1056 | − | |
| 1063 | +out: | |
| 1057 | 1064 | spin_unlock_bh(lock); |
| 1058 | 1065 | |
| 1059 | 1066 | return err; |
| @@ -237,8 +237,10 @@ static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash) · 236 unmodified lines | ||
| 237 | 237 | goto out; |
| 238 | 238 | |
| 239 | 239 | err = -ENOENT; |
| 240 | + if (!pprev) | |
| 241 | + goto out; | |
| 240 | 242 | |
| 241 | − rht_for_each(entry, old_tbl, old_hash) { | |
| 243 | + rht_for_each_from(entry, *pprev, old_tbl, old_hash) { | |
| 242 | 244 | err = 0; |
| 243 | 245 | next = rht_dereference_bucket(entry->next, old_tbl, old_hash); |
| 244 | 246 | |
| @@ -496,6 +498,8 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, · 251 unmodified lines | ||
| 496 | 498 | |
| 497 | 499 | elasticity = RHT_ELASTICITY; |
| 498 | 500 | pprev = rht_bucket_var(tbl, hash); |
| 501 | + if (!pprev) | |
| 502 | + return ERR_PTR(-ENOENT); | |
| 499 | 503 | rht_for_each_from(head, *pprev, tbl, hash) { |
| 500 | 504 | struct rhlist_head *list; |
| 501 | 505 | struct rhlist_head *plist; |
| @@ -1161,11 +1165,10 @@ void rhashtable_destroy(struct rhashtable *ht) · 659 unmodified lines | ||
| 1161 | 1165 | } |
| 1162 | 1166 | EXPORT_SYMBOL_GPL(rhashtable_destroy); |
| 1163 | 1167 | |
| 1164 | −struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | |
| 1165 | − unsigned int hash) | |
| 1168 | +struct rhash_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, | |
| 1169 | + unsigned int hash) | |
| 1166 | 1170 | { |
| 1167 | 1171 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); |
| 1168 | − static struct rhash_head __rcu *rhnull; | |
| 1169 | 1172 | unsigned int index = hash & ((1 << tbl->nest) - 1); |
| 1170 | 1173 | unsigned int size = tbl->size >> tbl->nest; |
| 1171 | 1174 | unsigned int subhash = hash; |
| @@ -1183,15 +1186,23 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, · 11 unmodified lines | ||
| 1183 | 1186 | subhash >>= shift; |
| 1184 | 1187 | } |
| 1185 | 1188 | |
| 1186 | − if (!ntbl) { | |
| 1187 | − if (!rhnull) | |
| 1188 | − INIT_RHT_NULLS_HEAD(rhnull); | |
| 1189 | − return &rhnull; | |
| 1190 | − } | |
| 1189 | + if (!ntbl) | |
| 1190 | + return NULL; | |
| 1191 | 1191 | |
| 1192 | 1192 | return &ntbl[subhash].bucket; |
| 1193 | 1193 | |
| 1194 | 1194 | } |
| 1195 | +EXPORT_SYMBOL_GPL(__rht_bucket_nested); | |
| 1196 | + | |
| 1197 | +struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | |
| 1198 | + unsigned int hash) | |
| 1199 | +{ | |
| 1200 | + static struct rhash_head __rcu *rhnull; | |
| 1201 | + | |
| 1202 | + if (!rhnull) | |
| 1203 | + INIT_RHT_NULLS_HEAD(rhnull); | |
| 1204 | + return __rht_bucket_nested(tbl, hash) ?: &rhnull; | |
| 1205 | +} | |
| 1195 | 1206 | EXPORT_SYMBOL_GPL(rht_bucket_nested); |
| 1196 | 1207 | |
| 1197 | 1208 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, |