History

rhashtable: don't hold lock on first table throughout insertion.

rhashtable_try_insert() currently holds a lock on the bucket in the first table, while also locking buckets in subsequent tables. This is unnecessary and looks like a hold-over from some earlier version of the implementation. As insert and remove always lock a bucket in each table in turn, and as insert only inserts in the final table, there cannot be any races that are not covered by simply locking a bucket in each table in turn. When an insert call reaches that last table it can be sure that there is no matchinf entry in any other table as it has searched them all, and insertion never happens anywhere but in the last table. The fact that code tests for the existence of future_tbl while holding a lock on the relevant bucket ensures that two threads inserting the same key will make compatible decisions about which is the "last" table. This simplifies the code and allows the ->rehash field to be discarded. We still need a way to ensure that a dead bucket_table is never re-linked by rhashtable_walk_stop(). This can be achieved by calling call_rcu() inside the locked region, and checking with rcu_head_after_call_rcu() in rhashtable_walk_stop() to see if the bucket table is empty and dead.

NeilBrown committed 7 years ago

Commit 4feb7c71 parent 83b038d2 files · +16 −49Browse files

Unified · Split

2 files
  1. include/linux/rhashtable.h +0 −13
  2. lib/rhashtable.c +16 −36
include/linux/rhashtable.h+0−13
@@ -63,7 +63,6 @@ · 62 unmodified lines
6363 struct bucket_table {
6464 unsigned int size;
6565 unsigned int nest;
66− unsigned int rehash;
6766 u32 hash_rnd;
6867 unsigned int locks_mask;
6968 spinlock_t *locks;
@@ -776,12 +775,6 @@ static inline int rhltable_insert( · 706 unmodified lines
776775 * @obj: pointer to hash head inside object
777776 * @params: hash table parameters
778777 *
779− * Locks down the bucket chain in both the old and new table if a resize
780− * is in progress to ensure that writers can't remove from the old table
781− * and can't insert to the new table during the atomic operation of search
782− * and insertion. Searches for duplicates in both the old and new table if
783− * a resize is in progress.
784− *
785778 * This lookup function may only be used for fixed key hash table (key_len
786779 * parameter set). It will BUG() if used inappropriately.
787780 *
@@ -837,12 +830,6 @@ static inline void *rhashtable_lookup_get_insert_fast( · 49 unmodified lines
837830 * @obj: pointer to hash head inside object
838831 * @params: hash table parameters
839832 *
840− * Locks down the bucket chain in both the old and new table if a resize
841− * is in progress to ensure that writers can't remove from the old table
842− * and can't insert to the new table during the atomic operation of search
843− * and insertion. Searches for duplicates in both the old and new table if
844− * a resize is in progress.
845− *
846833 * Lookups may occur in parallel with hashtable mutations and resizing.
847834 *
848835 * Will trigger an automatic deferred table resizing if residency in the
lib/rhashtable.c+16−36
@@ -197,6 +197,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, · 196 unmodified lines
197197 return NULL;
198198 }
199199
200+ rcu_head_init(&tbl->rcu);
200201 INIT_LIST_HEAD(&tbl->walkers);
201202
202203 tbl->hash_rnd = get_random_u32();
@@ -280,10 +281,9 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, · 77 unmodified lines
280281 while (!(err = rhashtable_rehash_one(ht, old_hash)))
281282 ;
282283
283− if (err == -ENOENT) {
284− old_tbl->rehash++;
284+ if (err == -ENOENT)
285285 err = 0;
286− }
286+
287287 spin_unlock_bh(old_bucket_lock);
288288
289289 return err;
@@ -330,13 +330,16 @@ static int rhashtable_rehash_table(struct rhashtable *ht) · 40 unmodified lines
330330 spin_lock(&ht->lock);
331331 list_for_each_entry(walker, &old_tbl->walkers, list)
332332 walker->tbl = NULL;
333− spin_unlock(&ht->lock);
334333
335334 /* Wait for readers. All new readers will see the new
336335 * table, and thus no references to the old table will
337336 * remain.
337+ * We do this inside the locked region so that
338+ * rhashtable_walk_stop() can use rcu_head_after_call_rcu()
339+ * to check if it should not re-link the table.
338340 */
339341 call_rcu(&old_tbl->rcu, bucket_table_free_rcu);
342+ spin_unlock(&ht->lock);
340343
341344 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0;
342345 }
@@ -578,46 +581,22 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, · 235 unmodified lines
578581 struct bucket_table *new_tbl;
579582 struct bucket_table *tbl;
580583 unsigned int hash;
581− spinlock_t *lock;
582584 void *data;
583585
584− tbl = rcu_dereference(ht->tbl);
585−
586− /* All insertions must grab the oldest table containing
587− * the hashed bucket that is yet to be rehashed.
588− */
589− for (;;) {
590− hash = rht_head_hashfn(ht, tbl, obj, ht->p);
591− lock = rht_bucket_lock(tbl, hash);
592− spin_lock_bh(lock);
593−
594− if (tbl->rehash <= hash)
595− break;
596−
597− spin_unlock_bh(lock);
598− tbl = rht_dereference_rcu(tbl->future_tbl, ht);
599− }
600−
601− data = rhashtable_lookup_one(ht, tbl, hash, key, obj);
602− new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data);
603− if (PTR_ERR(new_tbl) != -EEXIST)
604− data = ERR_CAST(new_tbl);
586+ new_tbl = rcu_dereference(ht->tbl);
605587
606− while (!IS_ERR_OR_NULL(new_tbl)) {
588+ do {
607589 tbl = new_tbl;
608590 hash = rht_head_hashfn(ht, tbl, obj, ht->p);
609− spin_lock_nested(rht_bucket_lock(tbl, hash),
610− SINGLE_DEPTH_NESTING);
591+ spin_lock_bh(rht_bucket_lock(tbl, hash));
611592
612593 data = rhashtable_lookup_one(ht, tbl, hash, key, obj);
613594 new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data);
614595 if (PTR_ERR(new_tbl) != -EEXIST)
615596 data = ERR_CAST(new_tbl);
616597
617− spin_unlock(rht_bucket_lock(tbl, hash));
618− }
619−
620− spin_unlock_bh(lock);
598+ spin_unlock_bh(rht_bucket_lock(tbl, hash));
599+ } while (!IS_ERR_OR_NULL(new_tbl));
621600
622601 if (PTR_ERR(data) == -EAGAIN)
623602 data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?:
@@ -939,10 +918,11 @@ void rhashtable_walk_stop(struct rhashtable_iter *iter) · 315 unmodified lines
939918 ht = iter->ht;
940919
941920 spin_lock(&ht->lock);
942− if (tbl->rehash < tbl->size)
943− list_add(&iter->walker.list, &tbl->walkers);
944− else
921+ if (rcu_head_after_call_rcu(&tbl->rcu, bucket_table_free_rcu))
922+ /* This bucket table is being freed, don't re-link it. */
945923 iter->walker.tbl = NULL;
924+ else
925+ list_add(&iter->walker.list, &tbl->walkers);
946926 spin_unlock(&ht->lock);
947927
948928 out:

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t