Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / tools / testing / radix-tree / multiorder.c
blob59245b3d587c35c039c77e673bdd706319368167
1 /*
2 * multiorder.c: Multi-order radix tree entry testing
3 * Copyright (c) 2016 Intel Corporation
4 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
5 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
16 #include <linux/radix-tree.h>
17 #include <linux/slab.h>
18 #include <linux/errno.h>
20 #include "test.h"
22 #define for_each_index(i, base, order) \
23 for (i = base; i < base + (1 << order); i++)
25 static void __multiorder_tag_test(int index, int order)
27 RADIX_TREE(tree, GFP_KERNEL);
28 int base, err, i;
30 /* our canonical entry */
31 base = index & ~((1 << order) - 1);
33 printv(2, "Multiorder tag test with index %d, canonical entry %d\n",
34 index, base);
36 err = item_insert_order(&tree, index, order);
37 assert(!err);
40 * Verify we get collisions for covered indices. We try and fail to
41 * insert an exceptional entry so we don't leak memory via
42 * item_insert_order().
44 for_each_index(i, base, order) {
45 err = __radix_tree_insert(&tree, i, order,
46 (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY));
47 assert(err == -EEXIST);
50 for_each_index(i, base, order) {
51 assert(!radix_tree_tag_get(&tree, i, 0));
52 assert(!radix_tree_tag_get(&tree, i, 1));
55 assert(radix_tree_tag_set(&tree, index, 0));
57 for_each_index(i, base, order) {
58 assert(radix_tree_tag_get(&tree, i, 0));
59 assert(!radix_tree_tag_get(&tree, i, 1));
62 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1);
63 assert(radix_tree_tag_clear(&tree, index, 0));
65 for_each_index(i, base, order) {
66 assert(!radix_tree_tag_get(&tree, i, 0));
67 assert(radix_tree_tag_get(&tree, i, 1));
70 assert(radix_tree_tag_clear(&tree, index, 1));
72 assert(!radix_tree_tagged(&tree, 0));
73 assert(!radix_tree_tagged(&tree, 1));
75 item_kill_tree(&tree);
78 static void __multiorder_tag_test2(unsigned order, unsigned long index2)
80 RADIX_TREE(tree, GFP_KERNEL);
81 unsigned long index = (1 << order);
82 index2 += index;
84 assert(item_insert_order(&tree, 0, order) == 0);
85 assert(item_insert(&tree, index2) == 0);
87 assert(radix_tree_tag_set(&tree, 0, 0));
88 assert(radix_tree_tag_set(&tree, index2, 0));
90 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2);
92 item_kill_tree(&tree);
95 static void multiorder_tag_tests(void)
97 int i, j;
99 /* test multi-order entry for indices 0-7 with no sibling pointers */
100 __multiorder_tag_test(0, 3);
101 __multiorder_tag_test(5, 3);
103 /* test multi-order entry for indices 8-15 with no sibling pointers */
104 __multiorder_tag_test(8, 3);
105 __multiorder_tag_test(15, 3);
108 * Our order 5 entry covers indices 0-31 in a tree with height=2.
109 * This is broken up as follows:
110 * 0-7: canonical entry
111 * 8-15: sibling 1
112 * 16-23: sibling 2
113 * 24-31: sibling 3
115 __multiorder_tag_test(0, 5);
116 __multiorder_tag_test(29, 5);
118 /* same test, but with indices 32-63 */
119 __multiorder_tag_test(32, 5);
120 __multiorder_tag_test(44, 5);
123 * Our order 8 entry covers indices 0-255 in a tree with height=3.
124 * This is broken up as follows:
125 * 0-63: canonical entry
126 * 64-127: sibling 1
127 * 128-191: sibling 2
128 * 192-255: sibling 3
130 __multiorder_tag_test(0, 8);
131 __multiorder_tag_test(190, 8);
133 /* same test, but with indices 256-511 */
134 __multiorder_tag_test(256, 8);
135 __multiorder_tag_test(300, 8);
137 __multiorder_tag_test(0x12345678UL, 8);
139 for (i = 1; i < 10; i++)
140 for (j = 0; j < (10 << i); j++)
141 __multiorder_tag_test2(i, j);
144 static void multiorder_check(unsigned long index, int order)
146 unsigned long i;
147 unsigned long min = index & ~((1UL << order) - 1);
148 unsigned long max = min + (1UL << order);
149 void **slot;
150 struct item *item2 = item_create(min, order);
151 RADIX_TREE(tree, GFP_KERNEL);
153 printv(2, "Multiorder index %ld, order %d\n", index, order);
155 assert(item_insert_order(&tree, index, order) == 0);
157 for (i = min; i < max; i++) {
158 struct item *item = item_lookup(&tree, i);
159 assert(item != 0);
160 assert(item->index == index);
162 for (i = 0; i < min; i++)
163 item_check_absent(&tree, i);
164 for (i = max; i < 2*max; i++)
165 item_check_absent(&tree, i);
166 for (i = min; i < max; i++)
167 assert(radix_tree_insert(&tree, i, item2) == -EEXIST);
169 slot = radix_tree_lookup_slot(&tree, index);
170 free(*slot);
171 radix_tree_replace_slot(&tree, slot, item2);
172 for (i = min; i < max; i++) {
173 struct item *item = item_lookup(&tree, i);
174 assert(item != 0);
175 assert(item->index == min);
178 assert(item_delete(&tree, min) != 0);
180 for (i = 0; i < 2*max; i++)
181 item_check_absent(&tree, i);
184 static void multiorder_shrink(unsigned long index, int order)
186 unsigned long i;
187 unsigned long max = 1 << order;
188 RADIX_TREE(tree, GFP_KERNEL);
189 struct radix_tree_node *node;
191 printv(2, "Multiorder shrink index %ld, order %d\n", index, order);
193 assert(item_insert_order(&tree, 0, order) == 0);
195 node = tree.rnode;
197 assert(item_insert(&tree, index) == 0);
198 assert(node != tree.rnode);
200 assert(item_delete(&tree, index) != 0);
201 assert(node == tree.rnode);
203 for (i = 0; i < max; i++) {
204 struct item *item = item_lookup(&tree, i);
205 assert(item != 0);
206 assert(item->index == 0);
208 for (i = max; i < 2*max; i++)
209 item_check_absent(&tree, i);
211 if (!item_delete(&tree, 0)) {
212 printv(2, "failed to delete index %ld (order %d)\n", index, order);
213 abort();
216 for (i = 0; i < 2*max; i++)
217 item_check_absent(&tree, i);
220 static void multiorder_insert_bug(void)
222 RADIX_TREE(tree, GFP_KERNEL);
224 item_insert(&tree, 0);
225 radix_tree_tag_set(&tree, 0, 0);
226 item_insert_order(&tree, 3 << 6, 6);
228 item_kill_tree(&tree);
231 void multiorder_iteration(void)
233 RADIX_TREE(tree, GFP_KERNEL);
234 struct radix_tree_iter iter;
235 void **slot;
236 int i, j, err;
238 printv(1, "Multiorder iteration test\n");
240 #define NUM_ENTRIES 11
241 int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128};
242 int order[NUM_ENTRIES] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7};
244 for (i = 0; i < NUM_ENTRIES; i++) {
245 err = item_insert_order(&tree, index[i], order[i]);
246 assert(!err);
249 for (j = 0; j < 256; j++) {
250 for (i = 0; i < NUM_ENTRIES; i++)
251 if (j <= (index[i] | ((1 << order[i]) - 1)))
252 break;
254 radix_tree_for_each_slot(slot, &tree, &iter, j) {
255 int height = order[i] / RADIX_TREE_MAP_SHIFT;
256 int shift = height * RADIX_TREE_MAP_SHIFT;
257 unsigned long mask = (1UL << order[i]) - 1;
258 struct item *item = *slot;
260 assert((iter.index | mask) == (index[i] | mask));
261 assert(iter.shift == shift);
262 assert(!radix_tree_is_internal_node(item));
263 assert((item->index | mask) == (index[i] | mask));
264 assert(item->order == order[i]);
265 i++;
269 item_kill_tree(&tree);
272 void multiorder_tagged_iteration(void)
274 RADIX_TREE(tree, GFP_KERNEL);
275 struct radix_tree_iter iter;
276 void **slot;
277 int i, j;
279 printv(1, "Multiorder tagged iteration test\n");
281 #define MT_NUM_ENTRIES 9
282 int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128};
283 int order[MT_NUM_ENTRIES] = {1, 0, 2, 4, 3, 1, 3, 0, 7};
285 #define TAG_ENTRIES 7
286 int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128};
288 for (i = 0; i < MT_NUM_ENTRIES; i++)
289 assert(!item_insert_order(&tree, index[i], order[i]));
291 assert(!radix_tree_tagged(&tree, 1));
293 for (i = 0; i < TAG_ENTRIES; i++)
294 assert(radix_tree_tag_set(&tree, tag_index[i], 1));
296 for (j = 0; j < 256; j++) {
297 int k;
299 for (i = 0; i < TAG_ENTRIES; i++) {
300 for (k = i; index[k] < tag_index[i]; k++)
302 if (j <= (index[k] | ((1 << order[k]) - 1)))
303 break;
306 radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) {
307 unsigned long mask;
308 struct item *item = *slot;
309 for (k = i; index[k] < tag_index[i]; k++)
311 mask = (1UL << order[k]) - 1;
313 assert((iter.index | mask) == (tag_index[i] | mask));
314 assert(!radix_tree_is_internal_node(item));
315 assert((item->index | mask) == (tag_index[i] | mask));
316 assert(item->order == order[k]);
317 i++;
321 assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) ==
322 TAG_ENTRIES);
324 for (j = 0; j < 256; j++) {
325 int mask, k;
327 for (i = 0; i < TAG_ENTRIES; i++) {
328 for (k = i; index[k] < tag_index[i]; k++)
330 if (j <= (index[k] | ((1 << order[k]) - 1)))
331 break;
334 radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) {
335 struct item *item = *slot;
336 for (k = i; index[k] < tag_index[i]; k++)
338 mask = (1 << order[k]) - 1;
340 assert((iter.index | mask) == (tag_index[i] | mask));
341 assert(!radix_tree_is_internal_node(item));
342 assert((item->index | mask) == (tag_index[i] | mask));
343 assert(item->order == order[k]);
344 i++;
348 assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0)
349 == TAG_ENTRIES);
350 i = 0;
351 radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) {
352 assert(iter.index == tag_index[i]);
353 i++;
356 item_kill_tree(&tree);
360 * Basic join checks: make sure we can't find an entry in the tree after
361 * a larger entry has replaced it
363 static void multiorder_join1(unsigned long index,
364 unsigned order1, unsigned order2)
366 unsigned long loc;
367 void *item, *item2 = item_create(index + 1, order1);
368 RADIX_TREE(tree, GFP_KERNEL);
370 item_insert_order(&tree, index, order2);
371 item = radix_tree_lookup(&tree, index);
372 radix_tree_join(&tree, index + 1, order1, item2);
373 loc = find_item(&tree, item);
374 if (loc == -1)
375 free(item);
376 item = radix_tree_lookup(&tree, index + 1);
377 assert(item == item2);
378 item_kill_tree(&tree);
382 * Check that the accounting of exceptional entries is handled correctly
383 * by joining an exceptional entry to a normal pointer.
385 static void multiorder_join2(unsigned order1, unsigned order2)
387 RADIX_TREE(tree, GFP_KERNEL);
388 struct radix_tree_node *node;
389 void *item1 = item_create(0, order1);
390 void *item2;
392 item_insert_order(&tree, 0, order2);
393 radix_tree_insert(&tree, 1 << order2, (void *)0x12UL);
394 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
395 assert(item2 == (void *)0x12UL);
396 assert(node->exceptional == 1);
398 item2 = radix_tree_lookup(&tree, 0);
399 free(item2);
401 radix_tree_join(&tree, 0, order1, item1);
402 item2 = __radix_tree_lookup(&tree, 1 << order2, &node, NULL);
403 assert(item2 == item1);
404 assert(node->exceptional == 0);
405 item_kill_tree(&tree);
409 * This test revealed an accounting bug for exceptional entries at one point.
410 * Nodes were being freed back into the pool with an elevated exception count
411 * by radix_tree_join() and then radix_tree_split() was failing to zero the
412 * count of exceptional entries.
414 static void multiorder_join3(unsigned int order)
416 RADIX_TREE(tree, GFP_KERNEL);
417 struct radix_tree_node *node;
418 void **slot;
419 struct radix_tree_iter iter;
420 unsigned long i;
422 for (i = 0; i < (1 << order); i++) {
423 radix_tree_insert(&tree, i, (void *)0x12UL);
426 radix_tree_join(&tree, 0, order, (void *)0x16UL);
427 rcu_barrier();
429 radix_tree_split(&tree, 0, 0);
431 radix_tree_for_each_slot(slot, &tree, &iter, 0) {
432 radix_tree_iter_replace(&tree, &iter, slot, (void *)0x12UL);
435 __radix_tree_lookup(&tree, 0, &node, NULL);
436 assert(node->exceptional == node->count);
438 item_kill_tree(&tree);
441 static void multiorder_join(void)
443 int i, j, idx;
445 for (idx = 0; idx < 1024; idx = idx * 2 + 3) {
446 for (i = 1; i < 15; i++) {
447 for (j = 0; j < i; j++) {
448 multiorder_join1(idx, i, j);
453 for (i = 1; i < 15; i++) {
454 for (j = 0; j < i; j++) {
455 multiorder_join2(i, j);
459 for (i = 3; i < 10; i++) {
460 multiorder_join3(i);
464 static void check_mem(unsigned old_order, unsigned new_order, unsigned alloc)
466 struct radix_tree_preload *rtp = &radix_tree_preloads;
467 if (rtp->nr != 0)
468 printv(2, "split(%u %u) remaining %u\n", old_order, new_order,
469 rtp->nr);
471 * Can't check for equality here as some nodes may have been
472 * RCU-freed while we ran. But we should never finish with more
473 * nodes allocated since they should have all been preloaded.
475 if (nr_allocated > alloc)
476 printv(2, "split(%u %u) allocated %u %u\n", old_order, new_order,
477 alloc, nr_allocated);
480 static void __multiorder_split(int old_order, int new_order)
482 RADIX_TREE(tree, GFP_ATOMIC);
483 void **slot;
484 struct radix_tree_iter iter;
485 unsigned alloc;
486 struct item *item;
488 radix_tree_preload(GFP_KERNEL);
489 assert(item_insert_order(&tree, 0, old_order) == 0);
490 radix_tree_preload_end();
492 /* Wipe out the preloaded cache or it'll confuse check_mem() */
493 radix_tree_cpu_dead(0);
495 item = radix_tree_tag_set(&tree, 0, 2);
497 radix_tree_split_preload(old_order, new_order, GFP_KERNEL);
498 alloc = nr_allocated;
499 radix_tree_split(&tree, 0, new_order);
500 check_mem(old_order, new_order, alloc);
501 radix_tree_for_each_slot(slot, &tree, &iter, 0) {
502 radix_tree_iter_replace(&tree, &iter, slot,
503 item_create(iter.index, new_order));
505 radix_tree_preload_end();
507 item_kill_tree(&tree);
508 free(item);
511 static void __multiorder_split2(int old_order, int new_order)
513 RADIX_TREE(tree, GFP_KERNEL);
514 void **slot;
515 struct radix_tree_iter iter;
516 struct radix_tree_node *node;
517 void *item;
519 __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
521 item = __radix_tree_lookup(&tree, 0, &node, NULL);
522 assert(item == (void *)0x12);
523 assert(node->exceptional > 0);
525 radix_tree_split(&tree, 0, new_order);
526 radix_tree_for_each_slot(slot, &tree, &iter, 0) {
527 radix_tree_iter_replace(&tree, &iter, slot,
528 item_create(iter.index, new_order));
531 item = __radix_tree_lookup(&tree, 0, &node, NULL);
532 assert(item != (void *)0x12);
533 assert(node->exceptional == 0);
535 item_kill_tree(&tree);
538 static void __multiorder_split3(int old_order, int new_order)
540 RADIX_TREE(tree, GFP_KERNEL);
541 void **slot;
542 struct radix_tree_iter iter;
543 struct radix_tree_node *node;
544 void *item;
546 __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
548 item = __radix_tree_lookup(&tree, 0, &node, NULL);
549 assert(item == (void *)0x12);
550 assert(node->exceptional > 0);
552 radix_tree_split(&tree, 0, new_order);
553 radix_tree_for_each_slot(slot, &tree, &iter, 0) {
554 radix_tree_iter_replace(&tree, &iter, slot, (void *)0x16);
557 item = __radix_tree_lookup(&tree, 0, &node, NULL);
558 assert(item == (void *)0x16);
559 assert(node->exceptional > 0);
561 item_kill_tree(&tree);
563 __radix_tree_insert(&tree, 0, old_order, (void *)0x12);
565 item = __radix_tree_lookup(&tree, 0, &node, NULL);
566 assert(item == (void *)0x12);
567 assert(node->exceptional > 0);
569 radix_tree_split(&tree, 0, new_order);
570 radix_tree_for_each_slot(slot, &tree, &iter, 0) {
571 if (iter.index == (1 << new_order))
572 radix_tree_iter_replace(&tree, &iter, slot,
573 (void *)0x16);
574 else
575 radix_tree_iter_replace(&tree, &iter, slot, NULL);
578 item = __radix_tree_lookup(&tree, 1 << new_order, &node, NULL);
579 assert(item == (void *)0x16);
580 assert(node->count == node->exceptional);
581 do {
582 node = node->parent;
583 if (!node)
584 break;
585 assert(node->count == 1);
586 assert(node->exceptional == 0);
587 } while (1);
589 item_kill_tree(&tree);
592 static void multiorder_split(void)
594 int i, j;
596 for (i = 3; i < 11; i++)
597 for (j = 0; j < i; j++) {
598 __multiorder_split(i, j);
599 __multiorder_split2(i, j);
600 __multiorder_split3(i, j);
604 static void multiorder_account(void)
606 RADIX_TREE(tree, GFP_KERNEL);
607 struct radix_tree_node *node;
608 void **slot;
610 item_insert_order(&tree, 0, 5);
612 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
613 __radix_tree_lookup(&tree, 0, &node, NULL);
614 assert(node->count == node->exceptional * 2);
615 radix_tree_delete(&tree, 1 << 5);
616 assert(node->exceptional == 0);
618 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
619 __radix_tree_lookup(&tree, 1 << 5, &node, &slot);
620 assert(node->count == node->exceptional * 2);
621 __radix_tree_replace(&tree, node, slot, NULL, NULL);
622 assert(node->exceptional == 0);
624 item_kill_tree(&tree);
627 void multiorder_checks(void)
629 int i;
631 for (i = 0; i < 20; i++) {
632 multiorder_check(200, i);
633 multiorder_check(0, i);
634 multiorder_check((1UL << i) + 1, i);
637 for (i = 0; i < 15; i++)
638 multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i);
640 multiorder_insert_bug();
641 multiorder_tag_tests();
642 multiorder_iteration();
643 multiorder_tagged_iteration();
644 multiorder_join();
645 multiorder_split();
646 multiorder_account();
648 radix_tree_cpu_dead(0);
651 int __weak main(void)
653 radix_tree_init();
654 multiorder_checks();
655 return 0;