History

radix tree: Remove multiorder support

All users have now been converted to the XArray. Removing the support reduces code size and ensures new users will use the XArray instead.

Matthew Wilcox committed 8 years ago

Commit 3a08cd51 parent 542980a5 files · +19 −245Browse files

Unified · Split

5 files
  1. include/linux/radix-tree.h +4 −36
  2. lib/Kconfig +0 −4
  3. lib/radix-tree.c +13 −202
  4. mm/Kconfig +2 −2
  5. tools/testing/radix-tree/generated/autoconf.h +0 −1
include/linux/radix-tree.h+4−36
@@ -96,7 +96,6 @@ static inline bool radix_tree_empty(const struct radix_tree_root *root) · 95 unmodified lines
9696 * @next_index: one beyond the last index for this chunk
9797 * @tags: bit-mask for tag-iterating
9898 * @node: node that contains current slot
99− * @shift: shift for the node that holds our slots
10099 *
101100 * This radix tree iterator works in terms of "chunks" of slots. A chunk is a
102101 * subinterval of slots contained within one radix tree leaf node. It is
@@ -110,20 +109,8 @@ struct radix_tree_iter { · 7 unmodified lines
110109 unsigned long next_index;
111110 unsigned long tags;
112111 struct radix_tree_node *node;
113−#ifdef CONFIG_RADIX_TREE_MULTIORDER
114− unsigned int shift;
115−#endif
116112 };
117113
118−static inline unsigned int iter_shift(const struct radix_tree_iter *iter)
119−{
120−#ifdef CONFIG_RADIX_TREE_MULTIORDER
121− return iter->shift;
122−#else
123− return 0;
124−#endif
125−}
126−
127114 /**
128115 * Radix-tree synchronization
129116 *
@@ -230,13 +217,8 @@ static inline int radix_tree_exception(void *arg) · 100 unmodified lines
230217 return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK);
231218 }
232219
233−int __radix_tree_insert(struct radix_tree_root *, unsigned long index,
234− unsigned order, void *);
235−static inline int radix_tree_insert(struct radix_tree_root *root,
236− unsigned long index, void *entry)
237−{
238− return __radix_tree_insert(root, index, 0, entry);
239−}
220+int radix_tree_insert(struct radix_tree_root *, unsigned long index,
221+ void *);
240222 void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,
241223 struct radix_tree_node **nodep, void __rcu ***slotp);
242224 void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);
@@ -384,7 +366,7 @@ void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter) · 141 unmodified lines
384366 static inline unsigned long
385367 __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
386368 {
387− return iter->index + (slots << iter_shift(iter));
369+ return iter->index + slots;
388370 }
389371
390372 /**
@@ -409,20 +391,8 @@ void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot, · 18 unmodified lines
409391 static __always_inline long
410392 radix_tree_chunk_size(struct radix_tree_iter *iter)
411393 {
412− return (iter->next_index - iter->index) >> iter_shift(iter);
413−}
414−
415−#ifdef CONFIG_RADIX_TREE_MULTIORDER
416−void __rcu **__radix_tree_next_slot(void __rcu **slot,
417− struct radix_tree_iter *iter, unsigned flags);
418−#else
419−/* Can't happen without sibling entries, but the compiler can't tell that */
420−static inline void __rcu **__radix_tree_next_slot(void __rcu **slot,
421− struct radix_tree_iter *iter, unsigned flags)
422−{
423− return slot;
394+ return iter->next_index - iter->index;
424395 }
425−#endif
426396
427397 /**
428398 * radix_tree_next_slot - find next slot in chunk
@@ -482,8 +452,6 @@ static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot, · 53 unmodified lines
482452 return NULL;
483453
484454 found:
485− if (unlikely(radix_tree_is_internal_node(rcu_dereference_raw(*slot))))
486− return __radix_tree_next_slot(slot, iter, flags);
487455 return slot;
488456 }
489457
lib/Kconfig+0−4
@@ -405,10 +405,6 @@ config XARRAY_MULTI · 404 unmodified lines
405405 Support entries which occupy multiple consecutive indices in the
406406 XArray.
407407
408−config RADIX_TREE_MULTIORDER
409− bool
410− select XARRAY_MULTI
411−
412408 config ASSOCIATIVE_ARRAY
413409 bool
414410 help
lib/radix-tree.c+13−202
@@ -110,11 +110,6 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, · 109 unmodified lines
110110 unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK;
111111 void __rcu **entry = rcu_dereference_raw(parent->slots[offset]);
112112
113− if (xa_is_sibling(entry)) {
114− offset = xa_to_sibling(entry);
115− entry = rcu_dereference_raw(parent->slots[offset]);
116− }
117−
118113 *nodep = (void *)entry;
119114 return offset;
120115 }
@@ -229,7 +224,7 @@ radix_tree_find_next_bit(struct radix_tree_node *node, unsigned int tag, · 108 unmodified lines
229224
230225 static unsigned int iter_offset(const struct radix_tree_iter *iter)
231226 {
232− return (iter->index >> iter_shift(iter)) & RADIX_TREE_MAP_MASK;
227+ return iter->index & RADIX_TREE_MAP_MASK;
233228 }
234229
235230 /*
@@ -506,16 +501,13 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root) · 270 unmodified lines
506501
507502 /*
508503 * The candidate node has more than one child, or its child
509− * is not at the leftmost slot, or the child is a multiorder
510− * entry, we cannot shrink.
504+ * is not at the leftmost slot, we cannot shrink.
511505 */
512506 if (node->count != 1)
513507 break;
514508 child = rcu_dereference_raw(node->slots[0]);
515509 if (!child)
516510 break;
517− if (!radix_tree_is_internal_node(child) && node->shift)
518− break;
519511
520512 /*
521513 * For an IDR, we must not shrink entry 0 into the root in
@@ -613,7 +605,6 @@ static bool delete_node(struct radix_tree_root *root, · 91 unmodified lines
613605 * __radix_tree_create - create a slot in a radix tree
614606 * @root: radix tree root
615607 * @index: index key
616− * @order: index occupies 2^order aligned slots
617608 * @nodep: returns node
618609 * @slotp: returns slot
619610 *
@@ -627,21 +618,19 @@ static bool delete_node(struct radix_tree_root *root, · 7 unmodified lines
627618 * Returns -ENOMEM, or 0 for success.
628619 */
629620 static int __radix_tree_create(struct radix_tree_root *root,
630− unsigned long index, unsigned order,
631− struct radix_tree_node **nodep, void __rcu ***slotp)
621+ unsigned long index, struct radix_tree_node **nodep,
622+ void __rcu ***slotp)
632623 {
633624 struct radix_tree_node *node = NULL, *child;
634625 void __rcu **slot = (void __rcu **)&root->xa_head;
635626 unsigned long maxindex;
636627 unsigned int shift, offset = 0;
637− unsigned long max = index | ((1UL << order) - 1);
628+ unsigned long max = index;
638629 gfp_t gfp = root_gfp_mask(root);
639630
640631 shift = radix_tree_load_root(root, &child, &maxindex);
641632
642633 /* Make sure the tree is high enough. */
643− if (order > 0 && max == ((1UL << order) - 1))
644− max++;
645634 if (max > maxindex) {
646635 int error = radix_tree_extend(root, gfp, max, shift);
647636 if (error < 0)
@@ -650,7 +639,7 @@ static int __radix_tree_create(struct radix_tree_root *root, · 2 unmodified lines
650639 child = rcu_dereference_raw(root->xa_head);
651640 }
652641
653− while (shift > order) {
642+ while (shift > 0) {
654643 shift -= RADIX_TREE_MAP_SHIFT;
655644 if (child == NULL) {
656645 /* Have to add a child node. */
@@ -711,70 +700,8 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) · 54 unmodified lines
711700 }
712701 }
713702
714−#ifdef CONFIG_RADIX_TREE_MULTIORDER
715−static inline int insert_entries(struct radix_tree_node *node,
716− void __rcu **slot, void *item, unsigned order, bool replace)
717−{
718− void *sibling;
719− unsigned i, n, tag, offset, tags = 0;
720−
721− if (node) {
722− if (order > node->shift)
723− n = 1 << (order - node->shift);
724− else
725− n = 1;
726− offset = get_slot_offset(node, slot);
727− } else {
728− n = 1;
729− offset = 0;
730− }
731−
732− if (n > 1) {
733− offset = offset & ~(n - 1);
734− slot = &node->slots[offset];
735− }
736− sibling = xa_mk_sibling(offset);
737−
738− for (i = 0; i < n; i++) {
739− if (slot[i]) {
740− if (replace) {
741− node->count--;
742− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
743− if (tag_get(node, tag, offset + i))
744− tags |= 1 << tag;
745− } else
746− return -EEXIST;
747− }
748− }
749−
750− for (i = 0; i < n; i++) {
751− struct radix_tree_node *old = rcu_dereference_raw(slot[i]);
752− if (i) {
753− rcu_assign_pointer(slot[i], sibling);
754− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
755− if (tags & (1 << tag))
756− tag_clear(node, tag, offset + i);
757− } else {
758− rcu_assign_pointer(slot[i], item);
759− for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++)
760− if (tags & (1 << tag))
761− tag_set(node, tag, offset);
762− }
763− if (xa_is_node(old))
764− radix_tree_free_nodes(old);
765− if (xa_is_value(old))
766− node->nr_values--;
767− }
768− if (node) {
769− node->count += n;
770− if (xa_is_value(item))
771− node->nr_values += n;
772− }
773− return n;
774−}
775−#else
776703 static inline int insert_entries(struct radix_tree_node *node,
777− void __rcu **slot, void *item, unsigned order, bool replace)
704+ void __rcu **slot, void *item, bool replace)
778705 {
779706 if (*slot)
780707 return -EEXIST;
@@ -786,19 +713,17 @@ static inline int insert_entries(struct radix_tree_node *node, · 5 unmodified lines
786713 }
787714 return 1;
788715 }
789−#endif
790716
791717 /**
792718 * __radix_tree_insert - insert into a radix tree
793719 * @root: radix tree root
794720 * @index: index key
795− * @order: key covers the 2^order indices around index
796721 * @item: item to insert
797722 *
798723 * Insert an item into the radix tree at position @index.
799724 */
800−int __radix_tree_insert(struct radix_tree_root *root, unsigned long index,
801− unsigned order, void *item)
725+int radix_tree_insert(struct radix_tree_root *root, unsigned long index,
726+ void *item)
802727 {
803728 struct radix_tree_node *node;
804729 void __rcu **slot;
@@ -806,11 +731,11 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, · 1 unmodified line
806731
807732 BUG_ON(radix_tree_is_internal_node(item));
808733
809− error = __radix_tree_create(root, index, order, &node, &slot);
734+ error = __radix_tree_create(root, index, &node, &slot);
810735 if (error)
811736 return error;
812737
813− error = insert_entries(node, slot, item, order, false);
738+ error = insert_entries(node, slot, item, false);
814739 if (error < 0)
815740 return error;
816741
@@ -825,7 +750,7 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, · 8 unmodified lines
825750
826751 return 0;
827752 }
828−EXPORT_SYMBOL(__radix_tree_insert);
753+EXPORT_SYMBOL(radix_tree_insert);
829754
830755 /**
831756 * __radix_tree_lookup - lookup an item in a radix tree
@@ -917,32 +842,12 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) · 85 unmodified lines
917842 }
918843 EXPORT_SYMBOL(radix_tree_lookup);
919844
920−static inline void replace_sibling_entries(struct radix_tree_node *node,
921− void __rcu **slot, int count, int values)
922−{
923−#ifdef CONFIG_RADIX_TREE_MULTIORDER
924− unsigned offset = get_slot_offset(node, slot);
925− void *ptr = xa_mk_sibling(offset);
926−
927− while (++offset < RADIX_TREE_MAP_SIZE) {
928− if (rcu_dereference_raw(node->slots[offset]) != ptr)
929− break;
930− if (count < 0) {
931− node->slots[offset] = NULL;
932− node->count--;
933− }
934− node->nr_values += values;
935− }
936−#endif
937−}
938−
939845 static void replace_slot(void __rcu **slot, void *item,
940846 struct radix_tree_node *node, int count, int values)
941847 {
942848 if (node && (count || values)) {
943849 node->count += count;
944850 node->nr_values += values;
945− replace_sibling_entries(node, slot, count, values);
946851 }
947852
948853 rcu_assign_pointer(*slot, item);
@@ -1223,14 +1128,6 @@ int radix_tree_tag_get(const struct radix_tree_root *root, · 274 unmodified lines
12231128 }
12241129 EXPORT_SYMBOL(radix_tree_tag_get);
12251130
1226−static inline void __set_iter_shift(struct radix_tree_iter *iter,
1227− unsigned int shift)
1228−{
1229−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1230− iter->shift = shift;
1231−#endif
1232−}
1233−
12341131 /* Construct iter->tags bit-mask from node->tags[tag] array */
12351132 static void set_iter_tags(struct radix_tree_iter *iter,
12361133 struct radix_tree_node *node, unsigned offset,
@@ -1257,92 +1154,11 @@ static void set_iter_tags(struct radix_tree_iter *iter, · 20 unmodified lines
12571154 }
12581155 }
12591156
1260−#ifdef CONFIG_RADIX_TREE_MULTIORDER
1261−static void __rcu **skip_siblings(struct radix_tree_node **nodep,
1262− void __rcu **slot, struct radix_tree_iter *iter)
1263−{
1264− while (iter->index < iter->next_index) {
1265− *nodep = rcu_dereference_raw(*slot);
1266− if (*nodep && !xa_is_sibling(*nodep))
1267− return slot;
1268− slot++;
1269− iter->index = __radix_tree_iter_add(iter, 1);
1270− iter->tags >>= 1;
1271− }
1272−
1273− *nodep = NULL;
1274− return NULL;
1275−}
1276−
1277−void __rcu **__radix_tree_next_slot(void __rcu **slot,
1278− struct radix_tree_iter *iter, unsigned flags)
1279−{
1280− unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
1281− struct radix_tree_node *node;
1282−
1283− slot = skip_siblings(&node, slot, iter);
1284−
1285− while (radix_tree_is_internal_node(node)) {
1286− unsigned offset;
1287− unsigned long next_index;
1288−
1289− if (node == RADIX_TREE_RETRY)
1290− return slot;
1291− node = entry_to_node(node);
1292− iter->node = node;
1293− iter->shift = node->shift;
1294−
1295− if (flags & RADIX_TREE_ITER_TAGGED) {
1296− offset = radix_tree_find_next_bit(node, tag, 0);
1297− if (offset == RADIX_TREE_MAP_SIZE)
1298− return NULL;
1299− slot = &node->slots[offset];
1300− iter->index = __radix_tree_iter_add(iter, offset);
1301− set_iter_tags(iter, node, offset, tag);
1302− node = rcu_dereference_raw(*slot);
1303− } else {
1304− offset = 0;
1305− slot = &node->slots[0];
1306− for (;;) {
1307− node = rcu_dereference_raw(*slot);
1308− if (node)
1309− break;
1310− slot++;
1311− offset++;
1312− if (offset == RADIX_TREE_MAP_SIZE)
1313− return NULL;
1314− }
1315− iter->index = __radix_tree_iter_add(iter, offset);
1316− }
1317− if ((flags & RADIX_TREE_ITER_CONTIG) && (offset > 0))
1318− goto none;
1319− next_index = (iter->index | shift_maxindex(iter->shift)) + 1;
1320− if (next_index < iter->next_index)
1321− iter->next_index = next_index;
1322− }
1323−
1324− return slot;
1325− none:
1326− iter->next_index = 0;
1327− return NULL;
1328−}
1329−EXPORT_SYMBOL(__radix_tree_next_slot);
1330−#else
1331−static void __rcu **skip_siblings(struct radix_tree_node **nodep,
1332− void __rcu **slot, struct radix_tree_iter *iter)
1333−{
1334− return slot;
1335−}
1336−#endif
1337−
13381157 void __rcu **radix_tree_iter_resume(void __rcu **slot,
13391158 struct radix_tree_iter *iter)
13401159 {
1341− struct radix_tree_node *node;
1342−
13431160 slot++;
13441161 iter->index = __radix_tree_iter_add(iter, 1);
1345− skip_siblings(&node, slot, iter);
13461162 iter->next_index = iter->index;
13471163 iter->tags = 0;
13481164 return NULL;
@@ -1393,7 +1209,6 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 44 unmodified lines
13931209 iter->next_index = maxindex + 1;
13941210 iter->tags = 1;
13951211 iter->node = NULL;
1396− __set_iter_shift(iter, 0);
13971212 return (void __rcu **)&root->xa_head;
13981213 }
13991214
@@ -1414,8 +1229,6 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 14 unmodified lines
14141229 while (++offset < RADIX_TREE_MAP_SIZE) {
14151230 void *slot = rcu_dereference_raw(
14161231 node->slots[offset]);
1417− if (xa_is_sibling(slot))
1418− continue;
14191232 if (slot)
14201233 break;
14211234 }
@@ -1436,10 +1249,9 @@ void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, · 14 unmodified lines
14361249 } while (node->shift && radix_tree_is_internal_node(child));
14371250
14381251 /* Update the iterator state */
1439− iter->index = (index &~ node_maxindex(node)) | (offset << node->shift);
1252+ iter->index = (index &~ node_maxindex(node)) | offset;
14401253 iter->next_index = (index | node_maxindex(node)) + 1;
14411254 iter->node = node;
1442− __set_iter_shift(iter, node->shift);
14431255
14441256 if (flags & RADIX_TREE_ITER_TAGGED)
14451257 set_iter_tags(iter, node, offset, tag);
@@ -1750,7 +1562,6 @@ void __rcu **idr_get_free(struct radix_tree_root *root, · 304 unmodified lines
17501562 else
17511563 iter->next_index = 1;
17521564 iter->node = node;
1753− __set_iter_shift(iter, shift);
17541565 set_iter_tags(iter, node, offset, IDR_FREE);
17551566
17561567 return slot;
mm/Kconfig+2−2
@@ -379,7 +379,7 @@ config TRANSPARENT_HUGEPAGE · 378 unmodified lines
379379 bool "Transparent Hugepage Support"
380380 depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE
381381 select COMPACTION
382− select RADIX_TREE_MULTIORDER
382+ select XARRAY_MULTI
383383 help
384384 Transparent Hugepages allows the kernel to use huge pages and
385385 huge tlb transparently to the applications whenever possible.
@@ -671,7 +671,7 @@ config ZONE_DEVICE · 285 unmodified lines
671671 depends on MEMORY_HOTREMOVE
672672 depends on SPARSEMEM_VMEMMAP
673673 depends on ARCH_HAS_ZONE_DEVICE
674− select RADIX_TREE_MULTIORDER
674+ select XARRAY_MULTI
675675
676676 help
677677 Device memory hotplug support allows for establishing pmem,
tools/testing/radix-tree/generated/autoconf.h+0−1
@@ -1,2 +1 @@
1−#define CONFIG_RADIX_TREE_MULTIORDER 1
21 #define CONFIG_XARRAY_MULTI 1