2 * Resizable, Scalable, Concurrent Hash Table
4 * Copyright (c) 2014 Thomas Graf <tgraf@suug.ch>
5 * Copyright (c) 2008-2014 Patrick McHardy <kaber@trash.net>
7 * Based on the following paper:
8 * https://www.usenix.org/legacy/event/atc11/tech/final_files/Triplett.pdf
10 * Code partially derived from nft_hash
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
17 #include <linux/kernel.h>
18 #include <linux/init.h>
19 #include <linux/log2.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
23 #include <linux/hash.h>
24 #include <linux/random.h>
25 #include <linux/rhashtable.h>
27 #define HASH_DEFAULT_SIZE 64UL
28 #define HASH_MIN_SIZE 4UL
30 #define ASSERT_RHT_MUTEX(HT) BUG_ON(!lockdep_rht_mutex_is_held(HT))
32 #ifdef CONFIG_PROVE_LOCKING
33 int lockdep_rht_mutex_is_held(const struct rhashtable
*ht
)
35 return ht
->p
.mutex_is_held();
37 EXPORT_SYMBOL_GPL(lockdep_rht_mutex_is_held
);
40 static void *rht_obj(const struct rhashtable
*ht
, const struct rhash_head
*he
)
42 return (void *) he
- ht
->p
.head_offset
;
45 static u32
__hashfn(const struct rhashtable
*ht
, const void *key
,
50 h
= ht
->p
.hashfn(key
, len
, ht
->p
.hash_rnd
);
52 return h
& (hsize
- 1);
56 * rhashtable_hashfn - compute hash for key of given length
57 * @ht: hash table to compute for
58 * @key: pointer to key
61 * Computes the hash value using the hash function provided in the 'hashfn'
62 * of struct rhashtable_params. The returned value is guaranteed to be
63 * smaller than the number of buckets in the hash table.
65 u32
rhashtable_hashfn(const struct rhashtable
*ht
, const void *key
, u32 len
)
67 struct bucket_table
*tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
69 return __hashfn(ht
, key
, len
, tbl
->size
);
71 EXPORT_SYMBOL_GPL(rhashtable_hashfn
);
73 static u32
obj_hashfn(const struct rhashtable
*ht
, const void *ptr
, u32 hsize
)
75 if (unlikely(!ht
->p
.key_len
)) {
78 h
= ht
->p
.obj_hashfn(ptr
, ht
->p
.hash_rnd
);
80 return h
& (hsize
- 1);
83 return __hashfn(ht
, ptr
+ ht
->p
.key_offset
, ht
->p
.key_len
, hsize
);
87 * rhashtable_obj_hashfn - compute hash for hashed object
88 * @ht: hash table to compute for
89 * @ptr: pointer to hashed object
91 * Computes the hash value using the hash function `hashfn` respectively
92 * 'obj_hashfn' depending on whether the hash table is set up to work with
93 * a fixed length key. The returned value is guaranteed to be smaller than
94 * the number of buckets in the hash table.
96 u32
rhashtable_obj_hashfn(const struct rhashtable
*ht
, void *ptr
)
98 struct bucket_table
*tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
100 return obj_hashfn(ht
, ptr
, tbl
->size
);
102 EXPORT_SYMBOL_GPL(rhashtable_obj_hashfn
);
104 static u32
head_hashfn(const struct rhashtable
*ht
,
105 const struct rhash_head
*he
, u32 hsize
)
107 return obj_hashfn(ht
, rht_obj(ht
, he
), hsize
);
110 static struct bucket_table
*bucket_table_alloc(size_t nbuckets
, gfp_t flags
)
112 struct bucket_table
*tbl
;
115 size
= sizeof(*tbl
) + nbuckets
* sizeof(tbl
->buckets
[0]);
116 tbl
= kzalloc(size
, flags
);
123 tbl
->size
= nbuckets
;
128 static void bucket_table_free(const struct bucket_table
*tbl
)
134 * rht_grow_above_75 - returns true if nelems > 0.75 * table-size
136 * @new_size: new table size
138 bool rht_grow_above_75(const struct rhashtable
*ht
, size_t new_size
)
140 /* Expand table when exceeding 75% load */
141 return ht
->nelems
> (new_size
/ 4 * 3);
143 EXPORT_SYMBOL_GPL(rht_grow_above_75
);
146 * rht_shrink_below_30 - returns true if nelems < 0.3 * table-size
148 * @new_size: new table size
150 bool rht_shrink_below_30(const struct rhashtable
*ht
, size_t new_size
)
152 /* Shrink table beneath 30% load */
153 return ht
->nelems
< (new_size
* 3 / 10);
155 EXPORT_SYMBOL_GPL(rht_shrink_below_30
);
157 static void hashtable_chain_unzip(const struct rhashtable
*ht
,
158 const struct bucket_table
*new_tbl
,
159 struct bucket_table
*old_tbl
, size_t n
)
161 struct rhash_head
*he
, *p
, *next
;
164 /* Old bucket empty, no work needed. */
165 p
= rht_dereference(old_tbl
->buckets
[n
], ht
);
169 /* Advance the old bucket pointer one or more times until it
170 * reaches a node that doesn't hash to the same bucket as the
171 * previous node p. Call the previous node p;
173 h
= head_hashfn(ht
, p
, new_tbl
->size
);
174 rht_for_each(he
, p
->next
, ht
) {
175 if (head_hashfn(ht
, he
, new_tbl
->size
) != h
)
179 RCU_INIT_POINTER(old_tbl
->buckets
[n
], p
->next
);
181 /* Find the subsequent node which does hash to the same
182 * bucket as node P, or NULL if no such node exists.
186 rht_for_each(he
, he
->next
, ht
) {
187 if (head_hashfn(ht
, he
, new_tbl
->size
) == h
) {
194 /* Set p's next pointer to that subsequent node pointer,
195 * bypassing the nodes which do not hash to p's bucket
197 RCU_INIT_POINTER(p
->next
, next
);
201 * rhashtable_expand - Expand hash table while allowing concurrent lookups
202 * @ht: the hash table to expand
203 * @flags: allocation flags
205 * A secondary bucket array is allocated and the hash entries are migrated
206 * while keeping them on both lists until the end of the RCU grace period.
208 * This function may only be called in a context where it is safe to call
209 * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
211 * The caller must ensure that no concurrent table mutations take place.
212 * It is however valid to have concurrent lookups if they are RCU protected.
214 int rhashtable_expand(struct rhashtable
*ht
, gfp_t flags
)
216 struct bucket_table
*new_tbl
, *old_tbl
= rht_dereference(ht
->tbl
, ht
);
217 struct rhash_head
*he
;
221 ASSERT_RHT_MUTEX(ht
);
223 if (ht
->p
.max_shift
&& ht
->shift
>= ht
->p
.max_shift
)
226 new_tbl
= bucket_table_alloc(old_tbl
->size
* 2, flags
);
232 /* For each new bucket, search the corresponding old bucket
233 * for the first entry that hashes to the new bucket, and
234 * link the new bucket to that entry. Since all the entries
235 * which will end up in the new bucket appear in the same
236 * old bucket, this constructs an entirely valid new hash
237 * table, but with multiple buckets "zipped" together into a
238 * single imprecise chain.
240 for (i
= 0; i
< new_tbl
->size
; i
++) {
241 h
= i
& (old_tbl
->size
- 1);
242 rht_for_each(he
, old_tbl
->buckets
[h
], ht
) {
243 if (head_hashfn(ht
, he
, new_tbl
->size
) == i
) {
244 RCU_INIT_POINTER(new_tbl
->buckets
[i
], he
);
250 /* Publish the new table pointer. Lookups may now traverse
251 * the new table, but they will not benefit from any
252 * additional efficiency until later steps unzip the buckets.
254 rcu_assign_pointer(ht
->tbl
, new_tbl
);
256 /* Unzip interleaved hash chains */
258 /* Wait for readers. All new readers will see the new
259 * table, and thus no references to the old table will
264 /* For each bucket in the old table (each of which
265 * contains items from multiple buckets of the new
269 for (i
= 0; i
< old_tbl
->size
; i
++) {
270 hashtable_chain_unzip(ht
, new_tbl
, old_tbl
, i
);
271 if (old_tbl
->buckets
[i
] != NULL
)
276 bucket_table_free(old_tbl
);
279 EXPORT_SYMBOL_GPL(rhashtable_expand
);
282 * rhashtable_shrink - Shrink hash table while allowing concurrent lookups
283 * @ht: the hash table to shrink
284 * @flags: allocation flags
286 * This function may only be called in a context where it is safe to call
287 * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
289 * The caller must ensure that no concurrent table mutations take place.
290 * It is however valid to have concurrent lookups if they are RCU protected.
292 int rhashtable_shrink(struct rhashtable
*ht
, gfp_t flags
)
294 struct bucket_table
*ntbl
, *tbl
= rht_dereference(ht
->tbl
, ht
);
295 struct rhash_head __rcu
**pprev
;
298 ASSERT_RHT_MUTEX(ht
);
300 if (ht
->shift
<= ht
->p
.min_shift
)
303 ntbl
= bucket_table_alloc(tbl
->size
/ 2, flags
);
309 /* Link each bucket in the new table to the first bucket
310 * in the old table that contains entries which will hash
313 for (i
= 0; i
< ntbl
->size
; i
++) {
314 ntbl
->buckets
[i
] = tbl
->buckets
[i
];
316 /* Link each bucket in the new table to the first bucket
317 * in the old table that contains entries which will hash
320 for (pprev
= &ntbl
->buckets
[i
]; *pprev
!= NULL
;
321 pprev
= &rht_dereference(*pprev
, ht
)->next
)
323 RCU_INIT_POINTER(*pprev
, tbl
->buckets
[i
+ ntbl
->size
]);
326 /* Publish the new, valid hash table */
327 rcu_assign_pointer(ht
->tbl
, ntbl
);
329 /* Wait for readers. No new readers will have references to the
334 bucket_table_free(tbl
);
338 EXPORT_SYMBOL_GPL(rhashtable_shrink
);
341 * rhashtable_insert - insert object into hash hash table
343 * @obj: pointer to hash head inside object
344 * @flags: allocation flags (table expansion)
346 * Will automatically grow the table via rhashtable_expand() if the the
347 * grow_decision function specified at rhashtable_init() returns true.
349 * The caller must ensure that no concurrent table mutations occur. It is
350 * however valid to have concurrent lookups if they are RCU protected.
352 void rhashtable_insert(struct rhashtable
*ht
, struct rhash_head
*obj
,
355 struct bucket_table
*tbl
= rht_dereference(ht
->tbl
, ht
);
358 ASSERT_RHT_MUTEX(ht
);
360 hash
= head_hashfn(ht
, obj
, tbl
->size
);
361 RCU_INIT_POINTER(obj
->next
, tbl
->buckets
[hash
]);
362 rcu_assign_pointer(tbl
->buckets
[hash
], obj
);
365 if (ht
->p
.grow_decision
&& ht
->p
.grow_decision(ht
, tbl
->size
))
366 rhashtable_expand(ht
, flags
);
368 EXPORT_SYMBOL_GPL(rhashtable_insert
);
371 * rhashtable_remove_pprev - remove object from hash table given previous element
373 * @obj: pointer to hash head inside object
374 * @pprev: pointer to previous element
375 * @flags: allocation flags (table expansion)
377 * Identical to rhashtable_remove() but caller is alreayd aware of the element
378 * in front of the element to be deleted. This is in particular useful for
379 * deletion when combined with walking or lookup.
381 void rhashtable_remove_pprev(struct rhashtable
*ht
, struct rhash_head
*obj
,
382 struct rhash_head __rcu
**pprev
, gfp_t flags
)
384 struct bucket_table
*tbl
= rht_dereference(ht
->tbl
, ht
);
386 ASSERT_RHT_MUTEX(ht
);
388 RCU_INIT_POINTER(*pprev
, obj
->next
);
391 if (ht
->p
.shrink_decision
&&
392 ht
->p
.shrink_decision(ht
, tbl
->size
))
393 rhashtable_shrink(ht
, flags
);
395 EXPORT_SYMBOL_GPL(rhashtable_remove_pprev
);
398 * rhashtable_remove - remove object from hash table
400 * @obj: pointer to hash head inside object
401 * @flags: allocation flags (table expansion)
403 * Since the hash chain is single linked, the removal operation needs to
404 * walk the bucket chain upon removal. The removal operation is thus
405 * considerable slow if the hash table is not correctly sized.
407 * Will automatically shrink the table via rhashtable_expand() if the the
408 * shrink_decision function specified at rhashtable_init() returns true.
410 * The caller must ensure that no concurrent table mutations occur. It is
411 * however valid to have concurrent lookups if they are RCU protected.
413 bool rhashtable_remove(struct rhashtable
*ht
, struct rhash_head
*obj
,
416 struct bucket_table
*tbl
= rht_dereference(ht
->tbl
, ht
);
417 struct rhash_head __rcu
**pprev
;
418 struct rhash_head
*he
;
421 ASSERT_RHT_MUTEX(ht
);
423 h
= head_hashfn(ht
, obj
, tbl
->size
);
425 pprev
= &tbl
->buckets
[h
];
426 rht_for_each(he
, tbl
->buckets
[h
], ht
) {
432 rhashtable_remove_pprev(ht
, he
, pprev
, flags
);
438 EXPORT_SYMBOL_GPL(rhashtable_remove
);
441 * rhashtable_lookup - lookup key in hash table
443 * @key: pointer to key
445 * Computes the hash value for the key and traverses the bucket chain looking
446 * for a entry with an identical key. The first matching entry is returned.
448 * This lookup function may only be used for fixed key hash table (key_len
449 * paramter set). It will BUG() if used inappropriately.
451 * Lookups may occur in parallel with hash mutations as long as the lookup is
452 * guarded by rcu_read_lock(). The caller must take care of this.
454 void *rhashtable_lookup(const struct rhashtable
*ht
, const void *key
)
456 const struct bucket_table
*tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
457 struct rhash_head
*he
;
460 BUG_ON(!ht
->p
.key_len
);
462 h
= __hashfn(ht
, key
, ht
->p
.key_len
, tbl
->size
);
463 rht_for_each_rcu(he
, tbl
->buckets
[h
], ht
) {
464 if (memcmp(rht_obj(ht
, he
) + ht
->p
.key_offset
, key
,
467 return (void *) he
- ht
->p
.head_offset
;
472 EXPORT_SYMBOL_GPL(rhashtable_lookup
);
475 * rhashtable_lookup_compare - search hash table with compare function
477 * @hash: hash value of desired entry
478 * @compare: compare function, must return true on match
479 * @arg: argument passed on to compare function
481 * Traverses the bucket chain behind the provided hash value and calls the
482 * specified compare function for each entry.
484 * Lookups may occur in parallel with hash mutations as long as the lookup is
485 * guarded by rcu_read_lock(). The caller must take care of this.
487 * Returns the first entry on which the compare function returned true.
489 void *rhashtable_lookup_compare(const struct rhashtable
*ht
, u32 hash
,
490 bool (*compare
)(void *, void *), void *arg
)
492 const struct bucket_table
*tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
493 struct rhash_head
*he
;
495 if (unlikely(hash
>= tbl
->size
))
498 rht_for_each_rcu(he
, tbl
->buckets
[hash
], ht
) {
499 if (!compare(rht_obj(ht
, he
), arg
))
501 return (void *) he
- ht
->p
.head_offset
;
506 EXPORT_SYMBOL_GPL(rhashtable_lookup_compare
);
508 static size_t rounded_hashtable_size(struct rhashtable_params
*params
)
510 return max(roundup_pow_of_two(params
->nelem_hint
* 4 / 3),
511 1UL << params
->min_shift
);
515 * rhashtable_init - initialize a new hash table
516 * @ht: hash table to be initialized
517 * @params: configuration parameters
519 * Initializes a new hash table based on the provided configuration
520 * parameters. A table can be configured either with a variable or
523 * Configuration Example 1: Fixed length keys
527 * struct rhash_head node;
530 * struct rhashtable_params params = {
531 * .head_offset = offsetof(struct test_obj, node),
532 * .key_offset = offsetof(struct test_obj, key),
533 * .key_len = sizeof(int),
534 * .hashfn = arch_fast_hash,
535 * .mutex_is_held = &my_mutex_is_held,
538 * Configuration Example 2: Variable length keys
541 * struct rhash_head node;
544 * u32 my_hash_fn(const void *data, u32 seed)
546 * struct test_obj *obj = data;
548 * return [... hash ...];
551 * struct rhashtable_params params = {
552 * .head_offset = offsetof(struct test_obj, node),
553 * .hashfn = arch_fast_hash,
554 * .obj_hashfn = my_hash_fn,
555 * .mutex_is_held = &my_mutex_is_held,
558 int rhashtable_init(struct rhashtable
*ht
, struct rhashtable_params
*params
)
560 struct bucket_table
*tbl
;
563 size
= HASH_DEFAULT_SIZE
;
565 if ((params
->key_len
&& !params
->hashfn
) ||
566 (!params
->key_len
&& !params
->obj_hashfn
))
569 params
->min_shift
= max_t(size_t, params
->min_shift
,
570 ilog2(HASH_MIN_SIZE
));
572 if (params
->nelem_hint
)
573 size
= rounded_hashtable_size(params
);
575 tbl
= bucket_table_alloc(size
, GFP_KERNEL
);
579 memset(ht
, 0, sizeof(*ht
));
580 ht
->shift
= ilog2(tbl
->size
);
581 memcpy(&ht
->p
, params
, sizeof(*params
));
582 RCU_INIT_POINTER(ht
->tbl
, tbl
);
585 get_random_bytes(&ht
->p
.hash_rnd
, sizeof(ht
->p
.hash_rnd
));
589 EXPORT_SYMBOL_GPL(rhashtable_init
);
592 * rhashtable_destroy - destroy hash table
593 * @ht: the hash table to destroy
595 * Frees the bucket array. This function is not rcu safe, therefore the caller
596 * has to make sure that no resizing may happen by unpublishing the hashtable
597 * and waiting for the quiescent cycle before releasing the bucket array.
599 void rhashtable_destroy(const struct rhashtable
*ht
)
601 bucket_table_free(ht
->tbl
);
603 EXPORT_SYMBOL_GPL(rhashtable_destroy
);
605 /**************************************************************************
607 **************************************************************************/
609 #ifdef CONFIG_TEST_RHASHTABLE
611 #define TEST_HT_SIZE 8
612 #define TEST_ENTRIES 2048
613 #define TEST_PTR ((void *) 0xdeadbeef)
614 #define TEST_NEXPANDS 4
616 static int test_mutex_is_held(void)
624 struct rhash_head node
;
627 static int __init
test_rht_lookup(struct rhashtable
*ht
)
631 for (i
= 0; i
< TEST_ENTRIES
* 2; i
++) {
632 struct test_obj
*obj
;
633 bool expected
= !(i
% 2);
636 obj
= rhashtable_lookup(ht
, &key
);
638 if (expected
&& !obj
) {
639 pr_warn("Test failed: Could not find key %u\n", key
);
641 } else if (!expected
&& obj
) {
642 pr_warn("Test failed: Unexpected entry found for key %u\n",
645 } else if (expected
&& obj
) {
646 if (obj
->ptr
!= TEST_PTR
|| obj
->value
!= i
) {
647 pr_warn("Test failed: Lookup value mismatch %p!=%p, %u!=%u\n",
648 obj
->ptr
, TEST_PTR
, obj
->value
, i
);
657 static void test_bucket_stats(struct rhashtable
*ht
,
658 struct bucket_table
*tbl
,
661 unsigned int cnt
, i
, total
= 0;
662 struct test_obj
*obj
;
664 for (i
= 0; i
< tbl
->size
; i
++) {
668 pr_info(" [%#4x/%zu]", i
, tbl
->size
);
670 rht_for_each_entry_rcu(obj
, tbl
->buckets
[i
], node
) {
674 pr_cont(" [%p],", obj
);
678 pr_cont("\n [%#x] first element: %p, chain length: %u\n",
679 i
, tbl
->buckets
[i
], cnt
);
682 pr_info(" Traversal complete: counted=%u, nelems=%zu, entries=%d\n",
683 total
, ht
->nelems
, TEST_ENTRIES
);
686 static int __init
test_rhashtable(struct rhashtable
*ht
)
688 struct bucket_table
*tbl
;
689 struct test_obj
*obj
, *next
;
695 * Insert TEST_ENTRIES into table with all keys even numbers
697 pr_info(" Adding %d keys\n", TEST_ENTRIES
);
698 for (i
= 0; i
< TEST_ENTRIES
; i
++) {
699 struct test_obj
*obj
;
701 obj
= kzalloc(sizeof(*obj
), GFP_KERNEL
);
710 rhashtable_insert(ht
, &obj
->node
, GFP_KERNEL
);
714 tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
715 test_bucket_stats(ht
, tbl
, true);
719 for (i
= 0; i
< TEST_NEXPANDS
; i
++) {
720 pr_info(" Table expansion iteration %u...\n", i
);
721 rhashtable_expand(ht
, GFP_KERNEL
);
724 pr_info(" Verifying lookups...\n");
729 for (i
= 0; i
< TEST_NEXPANDS
; i
++) {
730 pr_info(" Table shrinkage iteration %u...\n", i
);
731 rhashtable_shrink(ht
, GFP_KERNEL
);
734 pr_info(" Verifying lookups...\n");
739 pr_info(" Deleting %d keys\n", TEST_ENTRIES
);
740 for (i
= 0; i
< TEST_ENTRIES
; i
++) {
743 obj
= rhashtable_lookup(ht
, &key
);
746 rhashtable_remove(ht
, &obj
->node
, GFP_KERNEL
);
753 tbl
= rht_dereference_rcu(ht
->tbl
, ht
);
754 for (i
= 0; i
< tbl
->size
; i
++)
755 rht_for_each_entry_safe(obj
, next
, tbl
->buckets
[i
], ht
, node
)
761 static int __init
test_rht_init(void)
763 struct rhashtable ht
;
764 struct rhashtable_params params
= {
765 .nelem_hint
= TEST_HT_SIZE
,
766 .head_offset
= offsetof(struct test_obj
, node
),
767 .key_offset
= offsetof(struct test_obj
, value
),
768 .key_len
= sizeof(int),
769 .hashfn
= arch_fast_hash
,
770 .mutex_is_held
= &test_mutex_is_held
,
771 .grow_decision
= rht_grow_above_75
,
772 .shrink_decision
= rht_shrink_below_30
,
776 pr_info("Running resizable hashtable tests...\n");
778 err
= rhashtable_init(&ht
, ¶ms
);
780 pr_warn("Test failed: Unable to initialize hashtable: %d\n",
785 err
= test_rhashtable(&ht
);
787 rhashtable_destroy(&ht
);
792 subsys_initcall(test_rht_init
);
794 #endif /* CONFIG_TEST_RHASHTABLE */