Rather than returning a pointer to a static nulls, rht_bucket_var() now returns NULL if the bucket doesn't exist. This will make the next patch, which stores a bitlock in the bucket pointer, somewhat cleaner. This change involves introducing __rht_bucket_nested() which is like rht_bucket_nested(), but doesn't provide the static nulls, and changing rht_bucket_nested() to call this and possible provide a static nulls - as is still needed for the non-var case.
Unified · Split
| @@ -265,6 +265,8 @@ void rhashtable_destroy(struct rhashtable *ht); · 264 unmodified lines | |||
| 265 | 265 | ||
| 266 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | 266 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, |
| 267 | unsigned int hash); | 267 | unsigned int hash); |
| 268 | struct rhash_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, | ||
| 269 | unsigned int hash); | ||
| 268 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, | 270 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, |
| 269 | struct bucket_table *tbl, | 271 | struct bucket_table *tbl, |
| 270 | unsigned int hash); | 272 | unsigned int hash); |
| @@ -294,7 +296,7 @@ static inline struct rhash_head __rcu *const *rht_bucket( · 23 unmodified lines | |||
| 294 | static inline struct rhash_head __rcu **rht_bucket_var( | 296 | static inline struct rhash_head __rcu **rht_bucket_var( |
| 295 | struct bucket_table *tbl, unsigned int hash) | 297 | struct bucket_table *tbl, unsigned int hash) |
| 296 | { | 298 | { |
| 297 | return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : | 299 | return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) : |
| 298 | &tbl->buckets[hash]; | 300 | &tbl->buckets[hash]; |
| 299 | } | 301 | } |
| 300 | 302 | ||
| @@ -890,6 +892,8 @@ static inline int __rhashtable_remove_fast_one( · 589 unmodified lines | |||
| 890 | spin_lock_bh(lock); | 892 | spin_lock_bh(lock); |
| 891 | 893 | ||
| 892 | pprev = rht_bucket_var(tbl, hash); | 894 | pprev = rht_bucket_var(tbl, hash); |
| 895 | if (!pprev) | ||
| 896 | goto out; | ||
| 893 | rht_for_each_from(he, *pprev, tbl, hash) { | 897 | rht_for_each_from(he, *pprev, tbl, hash) { |
| 894 | struct rhlist_head *list; | 898 | struct rhlist_head *list; |
| 895 | 899 | ||
| @@ -934,6 +938,7 @@ static inline int __rhashtable_remove_fast_one( · 38 unmodified lines | |||
| 934 | break; | 938 | break; |
| 935 | } | 939 | } |
| 936 | 940 | ||
| 941 | out: | ||
| 937 | spin_unlock_bh(lock); | 942 | spin_unlock_bh(lock); |
| 938 | 943 | ||
| 939 | if (err > 0) { | 944 | if (err > 0) { |
| @@ -1042,6 +1047,8 @@ static inline int __rhashtable_replace_fast( · 102 unmodified lines | |||
| 1042 | spin_lock_bh(lock); | 1047 | spin_lock_bh(lock); |
| 1043 | 1048 | ||
| 1044 | pprev = rht_bucket_var(tbl, hash); | 1049 | pprev = rht_bucket_var(tbl, hash); |
| 1050 | if (!pprev) | ||
| 1051 | goto out; | ||
| 1045 | rht_for_each_from(he, *pprev, tbl, hash) { | 1052 | rht_for_each_from(he, *pprev, tbl, hash) { |
| 1046 | if (he != obj_old) { | 1053 | if (he != obj_old) { |
| 1047 | pprev = &he->next; | 1054 | pprev = &he->next; |
| @@ -1053,7 +1060,7 @@ static inline int __rhashtable_replace_fast( · 5 unmodified lines | |||
| 1053 | err = 0; | 1060 | err = 0; |
| 1054 | break; | 1061 | break; |
| 1055 | } | 1062 | } |
| 1056 | 1063 | out: | |
| 1057 | spin_unlock_bh(lock); | 1064 | spin_unlock_bh(lock); |
| 1058 | 1065 | ||
| 1059 | return err; | 1066 | return err; |
| @@ -237,8 +237,10 @@ static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash) · 236 unmodified lines | |||
| 237 | goto out; | 237 | goto out; |
| 238 | 238 | ||
| 239 | err = -ENOENT; | 239 | err = -ENOENT; |
| 240 | if (!pprev) | ||
| 241 | goto out; | ||
| 240 | 242 | ||
| 241 | rht_for_each(entry, old_tbl, old_hash) { | 243 | rht_for_each_from(entry, *pprev, old_tbl, old_hash) { |
| 242 | err = 0; | 244 | err = 0; |
| 243 | next = rht_dereference_bucket(entry->next, old_tbl, old_hash); | 245 | next = rht_dereference_bucket(entry->next, old_tbl, old_hash); |
| 244 | 246 | ||
| @@ -496,6 +498,8 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, · 251 unmodified lines | |||
| 496 | 498 | ||
| 497 | elasticity = RHT_ELASTICITY; | 499 | elasticity = RHT_ELASTICITY; |
| 498 | pprev = rht_bucket_var(tbl, hash); | 500 | pprev = rht_bucket_var(tbl, hash); |
| 501 | if (!pprev) | ||
| 502 | return ERR_PTR(-ENOENT); | ||
| 499 | rht_for_each_from(head, *pprev, tbl, hash) { | 503 | rht_for_each_from(head, *pprev, tbl, hash) { |
| 500 | struct rhlist_head *list; | 504 | struct rhlist_head *list; |
| 501 | struct rhlist_head *plist; | 505 | struct rhlist_head *plist; |
| @@ -1161,11 +1165,10 @@ void rhashtable_destroy(struct rhashtable *ht) · 659 unmodified lines | |||
| 1161 | } | 1165 | } |
| 1162 | EXPORT_SYMBOL_GPL(rhashtable_destroy); | 1166 | EXPORT_SYMBOL_GPL(rhashtable_destroy); |
| 1163 | 1167 | ||
| 1164 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | 1168 | struct rhash_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, |
| 1165 | unsigned int hash) | 1169 | unsigned int hash) |
| 1166 | { | 1170 | { |
| 1167 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); | 1171 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); |
| 1168 | static struct rhash_head __rcu *rhnull; | ||
| 1169 | unsigned int index = hash & ((1 << tbl->nest) - 1); | 1172 | unsigned int index = hash & ((1 << tbl->nest) - 1); |
| 1170 | unsigned int size = tbl->size >> tbl->nest; | 1173 | unsigned int size = tbl->size >> tbl->nest; |
| 1171 | unsigned int subhash = hash; | 1174 | unsigned int subhash = hash; |
| @@ -1183,15 +1186,23 @@ struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, · 11 unmodified lines | |||
| 1183 | subhash >>= shift; | 1186 | subhash >>= shift; |
| 1184 | } | 1187 | } |
| 1185 | 1188 | ||
| 1186 | if (!ntbl) { | 1189 | if (!ntbl) |
| 1187 | if (!rhnull) | 1190 | return NULL; |
| 1188 | INIT_RHT_NULLS_HEAD(rhnull); | ||
| 1189 | return &rhnull; | ||
| 1190 | } | ||
| 1191 | 1191 | ||
| 1192 | return &ntbl[subhash].bucket; | 1192 | return &ntbl[subhash].bucket; |
| 1193 | 1193 | ||
| 1194 | } | 1194 | } |
| 1195 | EXPORT_SYMBOL_GPL(__rht_bucket_nested); | ||
| 1196 | |||
| 1197 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | ||
| 1198 | unsigned int hash) | ||
| 1199 | { | ||
| 1200 | static struct rhash_head __rcu *rhnull; | ||
| 1201 | |||
| 1202 | if (!rhnull) | ||
| 1203 | INIT_RHT_NULLS_HEAD(rhnull); | ||
| 1204 | return __rht_bucket_nested(tbl, hash) ?: &rhnull; | ||
| 1205 | } | ||
| 1195 | EXPORT_SYMBOL_GPL(rht_bucket_nested); | 1206 | EXPORT_SYMBOL_GPL(rht_bucket_nested); |
| 1196 | 1207 | ||
| 1197 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, | 1208 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, |