History

rhashtable: don't hold lock on first table throughout insertion.

rhashtable_try_insert() currently holds a lock on the bucket in the first table, while also locking buckets in subsequent tables. This is unnecessary and looks like a hold-over from some earlier version of the implementation. As insert and remove always lock a bucket in each table in turn, and as insert only inserts in the final table, there cannot be any races that are not covered by simply locking a bucket in each table in turn. When an insert call reaches that last table it can be sure that there is no matchinf entry in any other table as it has searched them all, and insertion never happens anywhere but in the last table. The fact that code tests for the existence of future_tbl while holding a lock on the relevant bucket ensures that two threads inserting the same key will make compatible decisions about which is the "last" table. This simplifies the code and allows the ->rehash field to be discarded. We still need a way to ensure that a dead bucket_table is never re-linked by rhashtable_walk_stop(). This can be achieved by calling call_rcu() inside the locked region, and checking with rcu_head_after_call_rcu() in rhashtable_walk_stop() to see if the bucket table is empty and dead.

NeilBrown committed 7 years ago

Commit 4feb7c71 parent 83b038d2 files · +16 −49Browse files

Unified · Split

2 files
  1. include/linux/rhashtable.h +0 −13
  2. lib/rhashtable.c +16 −36
include/linux/rhashtable.h+0−13
@@ -63,7 +63,6 @@ · 62 unmodified lines
63struct bucket_table {63struct bucket_table {
64 unsigned int size;64 unsigned int size;
65 unsigned int nest;65 unsigned int nest;
66 unsigned int rehash;
67 u32 hash_rnd;66 u32 hash_rnd;
68 unsigned int locks_mask;67 unsigned int locks_mask;
69 spinlock_t *locks;68 spinlock_t *locks;
@@ -776,12 +775,6 @@ static inline int rhltable_insert( · 706 unmodified lines
776 * @obj: pointer to hash head inside object775 * @obj: pointer to hash head inside object
777 * @params: hash table parameters776 * @params: hash table parameters
778 *777 *
779 * Locks down the bucket chain in both the old and new table if a resize
780 * is in progress to ensure that writers can't remove from the old table
781 * and can't insert to the new table during the atomic operation of search
782 * and insertion. Searches for duplicates in both the old and new table if
783 * a resize is in progress.
784 *
785 * This lookup function may only be used for fixed key hash table (key_len778 * This lookup function may only be used for fixed key hash table (key_len
786 * parameter set). It will BUG() if used inappropriately.779 * parameter set). It will BUG() if used inappropriately.
787 *780 *
@@ -837,12 +830,6 @@ static inline void *rhashtable_lookup_get_insert_fast( · 49 unmodified lines
837 * @obj: pointer to hash head inside object830 * @obj: pointer to hash head inside object
838 * @params: hash table parameters831 * @params: hash table parameters
839 *832 *
840 * Locks down the bucket chain in both the old and new table if a resize
841 * is in progress to ensure that writers can't remove from the old table
842 * and can't insert to the new table during the atomic operation of search
843 * and insertion. Searches for duplicates in both the old and new table if
844 * a resize is in progress.
845 *
846 * Lookups may occur in parallel with hashtable mutations and resizing.833 * Lookups may occur in parallel with hashtable mutations and resizing.
847 *834 *
848 * Will trigger an automatic deferred table resizing if residency in the835 * Will trigger an automatic deferred table resizing if residency in the
lib/rhashtable.c+16−36
@@ -197,6 +197,7 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, · 196 unmodified lines
197 return NULL;197 return NULL;
198 }198 }
199 199
200 rcu_head_init(&tbl->rcu);
200 INIT_LIST_HEAD(&tbl->walkers);201 INIT_LIST_HEAD(&tbl->walkers);
201 202
202 tbl->hash_rnd = get_random_u32();203 tbl->hash_rnd = get_random_u32();
@@ -280,10 +281,9 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, · 77 unmodified lines
280 while (!(err = rhashtable_rehash_one(ht, old_hash)))281 while (!(err = rhashtable_rehash_one(ht, old_hash)))
281 ;282 ;
282 283
283 if (err == -ENOENT) {284 if (err == -ENOENT)
284 old_tbl->rehash++;
285 err = 0;285 err = 0;
286 }286
287 spin_unlock_bh(old_bucket_lock);287 spin_unlock_bh(old_bucket_lock);
288 288
289 return err;289 return err;
@@ -330,13 +330,16 @@ static int rhashtable_rehash_table(struct rhashtable *ht) · 40 unmodified lines
330 spin_lock(&ht->lock);330 spin_lock(&ht->lock);
331 list_for_each_entry(walker, &old_tbl->walkers, list)331 list_for_each_entry(walker, &old_tbl->walkers, list)
332 walker->tbl = NULL;332 walker->tbl = NULL;
333 spin_unlock(&ht->lock);
334 333
335 /* Wait for readers. All new readers will see the new334 /* Wait for readers. All new readers will see the new
336 * table, and thus no references to the old table will335 * table, and thus no references to the old table will
337 * remain.336 * remain.
337 * We do this inside the locked region so that
338 * rhashtable_walk_stop() can use rcu_head_after_call_rcu()
339 * to check if it should not re-link the table.
338 */340 */
339 call_rcu(&old_tbl->rcu, bucket_table_free_rcu);341 call_rcu(&old_tbl->rcu, bucket_table_free_rcu);
342 spin_unlock(&ht->lock);
340 343
341 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0;344 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0;
342}345}
@@ -578,46 +581,22 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, · 235 unmodified lines
578 struct bucket_table *new_tbl;581 struct bucket_table *new_tbl;
579 struct bucket_table *tbl;582 struct bucket_table *tbl;
580 unsigned int hash;583 unsigned int hash;
581 spinlock_t *lock;
582 void *data;584 void *data;
583 585
584 tbl = rcu_dereference(ht->tbl);586 new_tbl = rcu_dereference(ht->tbl);
585
586 /* All insertions must grab the oldest table containing
587 * the hashed bucket that is yet to be rehashed.
588 */
589 for (;;) {
590 hash = rht_head_hashfn(ht, tbl, obj, ht->p);
591 lock = rht_bucket_lock(tbl, hash);
592 spin_lock_bh(lock);
593
594 if (tbl->rehash <= hash)
595 break;
596
597 spin_unlock_bh(lock);
598 tbl = rht_dereference_rcu(tbl->future_tbl, ht);
599 }
600
601 data = rhashtable_lookup_one(ht, tbl, hash, key, obj);
602 new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data);
603 if (PTR_ERR(new_tbl) != -EEXIST)
604 data = ERR_CAST(new_tbl);
605 587
606 while (!IS_ERR_OR_NULL(new_tbl)) {588 do {
607 tbl = new_tbl;589 tbl = new_tbl;
608 hash = rht_head_hashfn(ht, tbl, obj, ht->p);590 hash = rht_head_hashfn(ht, tbl, obj, ht->p);
609 spin_lock_nested(rht_bucket_lock(tbl, hash),591 spin_lock_bh(rht_bucket_lock(tbl, hash));
610 SINGLE_DEPTH_NESTING);
611 592
612 data = rhashtable_lookup_one(ht, tbl, hash, key, obj);593 data = rhashtable_lookup_one(ht, tbl, hash, key, obj);
613 new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data);594 new_tbl = rhashtable_insert_one(ht, tbl, hash, obj, data);
614 if (PTR_ERR(new_tbl) != -EEXIST)595 if (PTR_ERR(new_tbl) != -EEXIST)
615 data = ERR_CAST(new_tbl);596 data = ERR_CAST(new_tbl);
616 597
617 spin_unlock(rht_bucket_lock(tbl, hash));598 spin_unlock_bh(rht_bucket_lock(tbl, hash));
618 }599 } while (!IS_ERR_OR_NULL(new_tbl));
619
620 spin_unlock_bh(lock);
621 600
622 if (PTR_ERR(data) == -EAGAIN)601 if (PTR_ERR(data) == -EAGAIN)
623 data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?:602 data = ERR_PTR(rhashtable_insert_rehash(ht, tbl) ?:
@@ -939,10 +918,11 @@ void rhashtable_walk_stop(struct rhashtable_iter *iter) · 315 unmodified lines
939 ht = iter->ht;918 ht = iter->ht;
940 919
941 spin_lock(&ht->lock);920 spin_lock(&ht->lock);
942 if (tbl->rehash < tbl->size)921 if (rcu_head_after_call_rcu(&tbl->rcu, bucket_table_free_rcu))
943 list_add(&iter->walker.list, &tbl->walkers);922 /* This bucket table is being freed, don't re-link it. */
944 else
945 iter->walker.tbl = NULL;923 iter->walker.tbl = NULL;
924 else
925 list_add(&iter->walker.list, &tbl->walkers);
946 spin_unlock(&ht->lock);926 spin_unlock(&ht->lock);
947 927
948out:928out:

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t